Use a better / more detailed reporter on Github PRs

.github/workflows/test-devel.yml

@@ -346,14 +346,32 @@ jobs:
     - test-unrealircd-atheme
     runs-on: ubuntu-latest
     steps:
+    - uses: actions/checkout@v2
     - name: Download Artifacts
       uses: actions/download-artifact@v2
       with:
         path: artifacts
-    - name: Publish Unit Test Results
+    - if: github.event_name == 'pull_request'
-      uses: EnricoMi/publish-unit-test-result-action@v1
+      name: Publish Unit Test Results
+      uses: actions/github-script@v4
       with:
-        files: artifacts/**/*.xml
+        result-encoding: string
+        script: |
+          let body = '';
+          const options = {};
+          options.listeners = {
+              stdout: (data) => {
+                  body += data.toString();
+              }
+          };
+          await exec.exec('bash', ['-c', 'shopt -s globstar; python3 report.py artifacts/**/*.xml'], options);
+          github.issues.createComment({
+            issue_number: context.issue.number,
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            body: body,
+          });
+          return body;
   test-bahamut:
     needs:
     - build-bahamut

.github/workflows/test-devel_release.yml

@@ -78,14 +78,32 @@ jobs:
     - test-inspircd-atheme
     runs-on: ubuntu-latest
     steps:
+    - uses: actions/checkout@v2
     - name: Download Artifacts
       uses: actions/download-artifact@v2
       with:
         path: artifacts
-    - name: Publish Unit Test Results
+    - if: github.event_name == 'pull_request'
-      uses: EnricoMi/publish-unit-test-result-action@v1
+      name: Publish Unit Test Results
+      uses: actions/github-script@v4
       with:
-        files: artifacts/**/*.xml
+        result-encoding: string
+        script: |
+          let body = '';
+          const options = {};
+          options.listeners = {
+              stdout: (data) => {
+                  body += data.toString();
+              }
+          };
+          await exec.exec('bash', ['-c', 'shopt -s globstar; python3 report.py artifacts/**/*.xml'], options);
+          github.issues.createComment({
+            issue_number: context.issue.number,
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            body: body,
+          });
+          return body;
   test-inspircd:
     needs:
     - build-inspircd

.github/workflows/test-stable.yml

@@ -389,14 +389,32 @@ jobs:
     - test-unrealircd-atheme
     runs-on: ubuntu-latest
     steps:
+    - uses: actions/checkout@v2
     - name: Download Artifacts
       uses: actions/download-artifact@v2
       with:
         path: artifacts
-    - name: Publish Unit Test Results
+    - if: github.event_name == 'pull_request'
-      uses: EnricoMi/publish-unit-test-result-action@v1
+      name: Publish Unit Test Results
+      uses: actions/github-script@v4
       with:
-        files: artifacts/**/*.xml
+        result-encoding: string
+        script: |
+          let body = '';
+          const options = {};
+          options.listeners = {
+              stdout: (data) => {
+                  body += data.toString();
+              }
+          };
+          await exec.exec('bash', ['-c', 'shopt -s globstar; python3 report.py artifacts/**/*.xml'], options);
+          github.issues.createComment({
+            issue_number: context.issue.number,
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            body: body,
+          });
+          return body;
   test-bahamut:
     needs:
     - build-bahamut

make_workflows.py

@@ -10,6 +10,7 @@ and keep them in sync.
 
 import enum
 import pathlib
+import textwrap
 
 import yaml
 
@@ -357,6 +358,7 @@ def generate_workflow(config: dict, version_flavor: VersionFlavor):
         # this job then
         "if": "success() || failure()",
         "steps": [
+            {"uses": "actions/checkout@v2"},
             {
                 "name": "Download Artifacts",
                 "uses": "actions/download-artifact@v2",
@@ -364,8 +366,32 @@ def generate_workflow(config: dict, version_flavor: VersionFlavor):
             },
             {
                 "name": "Publish Unit Test Results",
-                "uses": "EnricoMi/publish-unit-test-result-action@v1",
+                "uses": "actions/github-script@v4",
-                "with": {"files": "artifacts/**/*.xml"},
+                "if": "github.event_name == 'pull_request'",
+                "with": {
+                    "result-encoding": "string",
+                    "script": script(
+                        textwrap.dedent(
+                            """\
+                            let body = '';
+                            const options = {};
+                            options.listeners = {
+                                stdout: (data) => {
+                                    body += data.toString();
+                                }
+                            };
+                            await exec.exec('bash', ['-c', 'shopt -s globstar; python3 report.py artifacts/**/*.xml'], options);
+                            github.issues.createComment({
+                              issue_number: context.issue.number,
+                              owner: context.repo.owner,
+                              repo: context.repo.repo,
+                              body: body,
+                            });
+                            return body;
+                        """
+                        )
+                    ),
+                },
             },
         ],
     }

report.py

@@ -0,0 +1,179 @@
+"""
+Reads pytest's XML output, and produces a report of the test run(s).
+"""
+
+import dataclasses
+import functools
+import itertools
+import re
+import sys
+import textwrap
+from typing import Optional, Set
+import xml.etree.ElementTree as ET
+
+
+def visit_bottomup(f, d):
+    """Visits and rewrites a nested-dict ``d`` from the bottom to the top,
+    using the ``f`` predicate."""
+    if isinstance(d, dict):
+        return f({k: visit_bottomup(f, v) for (k, v) in d.items()})
+    else:
+        return f(d)
+
+
+@dataclasses.dataclass
+class CaseResult:
+    success: bool
+    skipped: bool
+    type: Optional[str] = None
+    message: Optional[str] = None
+
+
+@dataclasses.dataclass
+class CompactedResult:
+    success: bool
+    count: int
+    nb_skipped: int
+    messages: Set[str]
+
+
+def partial_compaction(d):
+    # Group all the perfect successes together, but keep those with skipped
+    # tests separate
+    compacted_d = {}
+    successes = []
+    for (k, v) in d.items():
+        if isinstance(v, CompactedResult) and v.success and v.nb_skipped == 0:
+            successes.append((k, v))
+        else:
+            compacted_d[k] = v
+    if len(successes) == 0:
+        pass
+    elif len(successes) == 1:
+        ((k, v),) = successes
+        compacted_d[k] = v
+    else:
+        compacted_d["(others)"] = CompactedResult(
+            success=True,
+            count=sum(res.count for (_, res) in successes),
+            nb_skipped=0,
+            messages=set(),
+        )
+    return compacted_d
+
+
+def compact_results(d):
+    """Rewrite the nested dict ``d`` of CaseResult in a more compact form;
+    by folding successful subtrees."""
+    if isinstance(d, dict):
+        if set(d) == {None}:
+            return d[None]
+        while len(d) == 1 and all(isinstance(v, dict) for v in d.values()):
+            (key,) = d
+            d = {f"{key}::{k}": v for (k, v) in d[key].items()}
+        if not all(isinstance(v, CompactedResult) for v in d.values()):
+            # Some children are not compactable, so this subtree isn't either
+            return partial_compaction(d)
+        statuses = {v.success for v in d.values()}
+        if len(statuses) == 1:
+            (status,) = statuses
+            return CompactedResult(
+                success=status,
+                count=sum(v.count for v in d.values()),
+                nb_skipped=sum(v.nb_skipped for v in d.values()),
+                messages=set(
+                    itertools.chain.from_iterable(v.messages for v in d.values())
+                ),
+            )
+        else:
+            return partial_compaction(d)
+    elif isinstance(d, CaseResult):
+        return CompactedResult(
+            success=d.success,
+            count=1,
+            nb_skipped=int(d.skipped),
+            messages={d.message} if d.message else set(),
+        )
+    else:
+        assert False, repr(d)
+
+
+def format_results(d) -> str:
+    """Using the nested dict ``d`` of CaseResult, formats a string report."""
+    if isinstance(d, dict):
+        items = [f"<li>{k}: {v}</li>" for (k, v) in d.items()]
+        items_str = textwrap.indent("\n".join(items), prefix="  ")
+        return f"\n<ul>\n{items_str}\n</ul>"
+    elif isinstance(d, CompactedResult):
+        if d.success:
+            if d.nb_skipped:
+                return f"✔️ {d.count} successful ({d.nb_skipped} skipped)"
+            else:
+                return f"✔️ {d.count} successful"
+        else:
+            assert d.nb_skipped == 0
+            reason = "".join(f"<li>{msg}</li>" for msg in d.messages)
+            return f"❌ {d.count} failed:\n<ul>{reason}</ul>"
+    else:
+        assert False, d
+
+
+def main(filenames):
+    print("<ul>")
+    for filename in filenames:
+        results = {}
+        job = ET.parse(filename).getroot()
+        (suite,) = job
+        for case in suite:
+            if "name" not in case.attrib:
+                continue
+            path = case.attrib["classname"].split(".")
+            class_results = functools.reduce(
+                lambda d, name: d.setdefault(name, {}), path, results
+            )
+
+            if len(case):
+                (case_result,) = case
+                if case_result.tag == "skipped":
+                    leaf = CaseResult(success=True, skipped=True, **case_result.attrib)
+                elif case_result.tag in ("failure", "error"):
+                    leaf = CaseResult(
+                        success=False, skipped=False, **case_result.attrib
+                    )
+                else:
+                    assert False, case_result.tag
+            else:
+                leaf = CaseResult(success=True, skipped=False)
+
+            name = case.attrib["name"]
+            m = re.match(r"^(?P<name>.*?)(?P<param>\[.*\])$", name)
+            if m:
+                d = class_results.setdefault(m.group("name"), {})
+                assert m.group("param") not in d
+                d[m.group("param")] = leaf
+            else:
+                d = class_results.setdefault(name, {})
+                assert None not in d
+                d[None] = leaf
+
+        results = visit_bottomup(compact_results, results)
+        result = visit_bottomup(format_results, results)
+        if "\n" in result:
+            (summary, details) = result.split("\n", 1)
+            summary.rstrip(":")
+            print(
+                f"<li>\n"
+                f"  <details>\n"
+                f"    <summary>{filename}: {summary}</summary>\n"
+                f"    {details}\n"
+                f"  </details>\n"
+                f"</li>"
+            )
+        else:
+            print(f"<li>{filename}: {result}</li>")
+    print("</ul>")
+
+
+if __name__ == "__main__":
+    (_, *filenames) = sys.argv
+    exit(main(filenames))