Refactor the pytest test summary info parsing script (#4619)

* refactor the short test summary info parsing script

* update the call of the test summary parse script

* move the I/O to the topmost function

* preprocess the lines before passing them to the extraction function

Author: keewis

.github/workflows/parse_logs.py

@@ -1,21 +1,53 @@
 # type: ignore
+import argparse
+import itertools
 import pathlib
+import textwrap
 
-files = pathlib.Path("logs").rglob("**/*-log")
-files = sorted(filter(lambda x: x.is_file(), files))
+parser = argparse.ArgumentParser()
+parser.add_argument("filepaths", nargs="+", type=pathlib.Path)
+args = parser.parse_args()
+
+filepaths = sorted(p for p in args.filepaths if p.is_file())
+
+
+def extract_short_test_summary_info(lines):
+    up_to_start_of_section = itertools.dropwhile(
+        lambda l: "=== short test summary info ===" not in l,
+        lines,
+    )
+    up_to_section_content = itertools.islice(up_to_start_of_section, 1, None)
+    section_content = itertools.takewhile(
+        lambda l: l.startswith("FAILED"), up_to_section_content
+    )
+    content = "\n".join(section_content)
+
+    return content
+
+
+def format_log_message(path):
+    py_version = path.name.split("-")[1]
+    summary = f"Python {py_version} Test Summary Info"
+    with open(path) as f:
+        data = extract_short_test_summary_info(line.rstrip() for line in f)
+    message = textwrap.dedent(
+        f"""\
+        <details><summary>{summary}</summary>
+
+        ```
+        {data}
+        ```
+
+        </details>
+        """
+    )
+
+    return message
 
-message = "\n"
 
 print("Parsing logs ...")
-for file in files:
-    with open(file) as fpt:
-        print(f"Parsing {file.absolute()}")
-        data = fpt.read().split("test summary info")[-1].splitlines()[1:-1]
-        data = "\n".join(data)
-        py_version = file.name.split("-")[1]
-        message = f"{message}\n<details>\n<summary>\nPython {py_version} Test Summary Info\n</summary>\n\n```bash\n{data}\n```\n</details>\n"
+message = "\n\n".join(format_log_message(path) for path in filepaths)
 
 output_file = pathlib.Path("pytest-logs.txt")
-with open(output_file, "w") as fpt:
-    print(f"Writing output file to: {output_file.absolute()} ")
-    fpt.write(message)
+print(f"Writing output file to: {output_file.absolute()}")
+output_file.write_text(message)

.github/workflows/upstream-dev-ci.yaml

@@ -74,7 +74,8 @@ jobs:
           ls -R ./logs
       - name: Parse logs
         run: |
-          python .github/workflows/parse_logs.py
+          shopt -s globstar
+          python .github/workflows/parse_logs.py logs/**/*-log
       - name: Report failures
         uses: actions/github-script@v3
         with: