|
| 1 | +import { appendFile, readFile } from "node:fs/promises"; |
| 2 | + |
| 3 | +// Playwright's `retries: 2` (desktop/playwright.config.ts) lets a test fail |
| 4 | +// then pass on retry with no durable signal beyond a one-line "N flaky" in |
| 5 | +// the console log — the exact gap that hid the stream.spec.ts membership |
| 6 | +// race (#1798) for months. This walks the JSON reporter's suite tree |
| 7 | +// (recursive: `describe` blocks nest as child `suites`) and appends any |
| 8 | +// `status === "flaky"` test to the job's GitHub Actions summary so retried |
| 9 | +// failures stay visible even when the shard ultimately goes green. |
| 10 | +// |
| 11 | +// Usage: node scripts/summarize-flaky-tests.mjs <report.json> <run-label> |
| 12 | + |
| 13 | +function collectFlakyTests(suite, out) { |
| 14 | + for (const spec of suite.specs ?? []) { |
| 15 | + for (const test of spec.tests ?? []) { |
| 16 | + if (test.status !== "flaky") continue; |
| 17 | + out.push({ |
| 18 | + title: `${suite.file} › ${spec.title}`, |
| 19 | + project: test.projectName, |
| 20 | + attempts: test.results?.length ?? 0, |
| 21 | + }); |
| 22 | + } |
| 23 | + } |
| 24 | + for (const child of suite.suites ?? []) { |
| 25 | + collectFlakyTests(child, out); |
| 26 | + } |
| 27 | +} |
| 28 | + |
| 29 | +const [reportPath, runLabel] = process.argv.slice(2); |
| 30 | +if (!reportPath || !runLabel) { |
| 31 | + console.error( |
| 32 | + "Usage: node scripts/summarize-flaky-tests.mjs <report.json> <run-label>", |
| 33 | + ); |
| 34 | + process.exit(1); |
| 35 | +} |
| 36 | + |
| 37 | +// This step runs `if: !cancelled()` purely to surface flaky tests — it must |
| 38 | +// never fail the job on its own, so a malformed/unexpected report (from a |
| 39 | +// Playwright version bump or a crashed run) is swallowed, not thrown. |
| 40 | +try { |
| 41 | + const report = JSON.parse(await readFile(reportPath, "utf8")); |
| 42 | + |
| 43 | + const flaky = []; |
| 44 | + for (const suite of report.suites ?? []) { |
| 45 | + collectFlakyTests(suite, flaky); |
| 46 | + } |
| 47 | + |
| 48 | + if (flaky.length > 0) { |
| 49 | + const escapeCell = (value) => String(value).replaceAll("|", "\\|"); |
| 50 | + const rows = flaky |
| 51 | + .map( |
| 52 | + (t) => |
| 53 | + `| ${escapeCell(t.title)} | ${escapeCell(t.project)} | ${t.attempts} |`, |
| 54 | + ) |
| 55 | + .join("\n"); |
| 56 | + const summary = |
| 57 | + `### Flaky tests — ${runLabel}\n\n` + |
| 58 | + `${flaky.length} test(s) failed at least once before passing on retry:\n\n` + |
| 59 | + "| Test | Project | Attempts |\n| --- | --- | --- |\n" + |
| 60 | + `${rows}\n`; |
| 61 | + |
| 62 | + console.log(summary); |
| 63 | + |
| 64 | + const summaryFile = process.env.GITHUB_STEP_SUMMARY; |
| 65 | + if (summaryFile) { |
| 66 | + await appendFile(summaryFile, `${summary}\n`); |
| 67 | + } |
| 68 | + } |
| 69 | +} catch (error) { |
| 70 | + console.log(`Skipping flaky-test summary: ${error.message}`); |
| 71 | +} |
0 commit comments