|
| 1 | +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. |
| 2 | +# SPDX-License-Identifier: Apache-2.0 |
| 3 | + |
| 4 | +"""Write a plain-English pull-request comment from the JUnit files of a CI run. |
| 5 | +
|
| 6 | +Reads every ``*.xml`` under the test-results directory that the final CI job |
| 7 | +downloads, one subdirectory per test job, and writes a short Markdown comment: |
| 8 | +how many configurations ran, which tests did not pass and where, the execution |
| 9 | +totals, and a link to the run's Summary page where the full reports live. |
| 10 | +
|
| 11 | +Usage:: |
| 12 | +
|
| 13 | + python ci/tools/junit_comment.py --results test-results \\ |
| 14 | + --commit <sha> --run-url <url> --output comment.md |
| 15 | +""" |
| 16 | + |
| 17 | +from __future__ import annotations |
| 18 | + |
| 19 | +import argparse |
| 20 | +import sys |
| 21 | +import xml.etree.ElementTree as ET |
| 22 | +from dataclasses import dataclass, field |
| 23 | +from pathlib import Path |
| 24 | + |
| 25 | +ARTIFACT_PREFIX = "test-results-" |
| 26 | +STANDARD_MODE_PREFIX = "standard-" |
| 27 | +MAX_LISTED = 25 |
| 28 | + |
| 29 | + |
| 30 | +@dataclass |
| 31 | +class Totals: |
| 32 | + passed: int = 0 |
| 33 | + failed: int = 0 |
| 34 | + errors: int = 0 |
| 35 | + skipped: int = 0 |
| 36 | + |
| 37 | + |
| 38 | +@dataclass |
| 39 | +class Results: |
| 40 | + files: int = 0 |
| 41 | + configurations: dict[str, Totals] = field(default_factory=dict) |
| 42 | + # test id -> {configuration: "failed" | "error"} for tests that did not pass somewhere |
| 43 | + not_passed: dict[str, dict[str, str]] = field(default_factory=dict) |
| 44 | + |
| 45 | + |
| 46 | +def configuration_label(results_dir: Path, xml_path: Path) -> str: |
| 47 | + """The test job's configuration, taken from the artifact directory name.""" |
| 48 | + parts = xml_path.relative_to(results_dir).parts |
| 49 | + name = parts[0] if len(parts) > 1 else "" |
| 50 | + if name.startswith(ARTIFACT_PREFIX): |
| 51 | + name = name[len(ARTIFACT_PREFIX) :] |
| 52 | + if name.startswith(STANDARD_MODE_PREFIX): |
| 53 | + name = name[len(STANDARD_MODE_PREFIX) :] |
| 54 | + return name or "(unknown configuration)" |
| 55 | + |
| 56 | + |
| 57 | +def outcome(case: ET.Element) -> str: |
| 58 | + """JUnit: <error> is an exception outside the test body, <failure> one inside it.""" |
| 59 | + tags = {child.tag for child in case} |
| 60 | + if "error" in tags: |
| 61 | + return "error" |
| 62 | + if "failure" in tags: |
| 63 | + return "failed" |
| 64 | + if "skipped" in tags: |
| 65 | + return "skipped" |
| 66 | + return "passed" |
| 67 | + |
| 68 | + |
| 69 | +def test_id(case: ET.Element) -> str: |
| 70 | + classname = case.get("classname") or "" |
| 71 | + name = case.get("name") or "(unnamed test)" |
| 72 | + return f"{classname}::{name}" if classname else name |
| 73 | + |
| 74 | + |
| 75 | +def collect(results_dir: Path) -> Results: |
| 76 | + results = Results() |
| 77 | + for xml_path in sorted(results_dir.rglob("*.xml")): |
| 78 | + label = configuration_label(results_dir, xml_path) |
| 79 | + totals = results.configurations.setdefault(label, Totals()) |
| 80 | + results.files += 1 |
| 81 | + # The files are pytest's own output from this run, not untrusted input. |
| 82 | + for case in ET.parse(xml_path).getroot().iter("testcase"): # noqa: S314 |
| 83 | + state = outcome(case) |
| 84 | + if state == "passed": |
| 85 | + totals.passed += 1 |
| 86 | + elif state == "failed": |
| 87 | + totals.failed += 1 |
| 88 | + elif state == "error": |
| 89 | + totals.errors += 1 |
| 90 | + else: |
| 91 | + totals.skipped += 1 |
| 92 | + if state in ("failed", "error"): |
| 93 | + results.not_passed.setdefault(test_id(case), {})[label] = state |
| 94 | + return results |
| 95 | + |
| 96 | + |
| 97 | +def plural(count: int, word: str) -> str: |
| 98 | + return f"{count:,} {word}" if count == 1 else f"{count:,} {word}s" |
| 99 | + |
| 100 | + |
| 101 | +def render(results: Results, commit: str, run_url: str) -> str: |
| 102 | + lines = [f"## Test results for commit {commit[:7]}", ""] |
| 103 | + if results.files == 0: |
| 104 | + lines += ["No test result files were found for this run.", "", f"Run: {run_url}"] |
| 105 | + return "\n".join(lines) + "\n" |
| 106 | + |
| 107 | + n_conf = len(results.configurations) |
| 108 | + configurations = sorted(results.configurations) |
| 109 | + lines.append(f"{plural(n_conf, 'configuration')} ran the test suites: {'; '.join(configurations)}.") |
| 110 | + lines.append("") |
| 111 | + |
| 112 | + passed = sum(t.passed for t in results.configurations.values()) |
| 113 | + failed = sum(t.failed for t in results.configurations.values()) |
| 114 | + errors = sum(t.errors for t in results.configurations.values()) |
| 115 | + skipped = sum(t.skipped for t in results.configurations.values()) |
| 116 | + |
| 117 | + if not results.not_passed: |
| 118 | + lines.append( |
| 119 | + f"No test failed. Counting every test execution across all configurations: " |
| 120 | + f"{passed:,} passed, {skipped:,} skipped." |
| 121 | + ) |
| 122 | + else: |
| 123 | + errored = {t for t, where in results.not_passed.items() if "error" in where.values()} |
| 124 | + only_failed = len(results.not_passed) - len(errored) |
| 125 | + lines.append( |
| 126 | + f"{plural(len(results.not_passed), 'test')} did not pass in at least one configuration. " |
| 127 | + f"{only_failed} failed inside the test body (an assertion or an exception). " |
| 128 | + f"{len(errored)} hit an error outside the test body (fixture setup or teardown, or collection)." |
| 129 | + ) |
| 130 | + lines.append("") |
| 131 | + lines.append( |
| 132 | + f"Counting every test execution across all configurations: " |
| 133 | + f"{passed:,} passed, {failed:,} failed, {errors:,} errored, {skipped:,} skipped." |
| 134 | + ) |
| 135 | + lines.append("") |
| 136 | + lines.append("Tests that did not pass:") |
| 137 | + lines.append("") |
| 138 | + listed = sorted(results.not_passed) |
| 139 | + for name in listed[:MAX_LISTED]: |
| 140 | + where = results.not_passed[name] |
| 141 | + state = "error" if "error" in where.values() else "failed" |
| 142 | + if n_conf > 1 and len(where) == n_conf: |
| 143 | + place = f"in all {n_conf} configurations" |
| 144 | + else: |
| 145 | + place = "in " + ", ".join(sorted(where)) |
| 146 | + lines.append(f"- `{name}`: {state} {place}") |
| 147 | + if len(listed) > MAX_LISTED: |
| 148 | + lines.append(f"- and {len(listed) - MAX_LISTED:,} more") |
| 149 | + |
| 150 | + lines.append("") |
| 151 | + lines.append(f"The per-configuration tables and the failing-test details are on the run's Summary page: {run_url}") |
| 152 | + lines.append("") |
| 153 | + lines.append( |
| 154 | + "A test *fails* when an assertion or exception happens inside the test body. " |
| 155 | + "A test *errors* when the exception happens outside it: fixture setup or teardown, or collection." |
| 156 | + ) |
| 157 | + return "\n".join(lines) + "\n" |
| 158 | + |
| 159 | + |
| 160 | +def main(argv: list[str] | None = None) -> int: |
| 161 | + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) |
| 162 | + parser.add_argument( |
| 163 | + "--results", required=True, type=Path, help="directory holding the downloaded test-results-* artifacts" |
| 164 | + ) |
| 165 | + parser.add_argument("--commit", required=True, help="commit SHA the results belong to") |
| 166 | + parser.add_argument("--run-url", required=True, help="URL of the workflow run whose Summary page holds the reports") |
| 167 | + parser.add_argument("--output", required=True, type=Path, help="file to write the Markdown comment to") |
| 168 | + args = parser.parse_args(argv) |
| 169 | + |
| 170 | + text = render(collect(args.results), args.commit, args.run_url) |
| 171 | + args.output.write_text(text, encoding="utf-8") |
| 172 | + sys.stdout.write(text) |
| 173 | + return 0 |
| 174 | + |
| 175 | + |
| 176 | +if __name__ == "__main__": |
| 177 | + raise SystemExit(main()) |
0 commit comments