Skip to content

Commit a653406

Browse files
wpfleger96npub1mn7jgtj4w2pd0g0zeuhxsa6jy6p0rewxz4kujt98my82ahfmp72sxjexk7
andauthored
ci(desktop): surface flaky E2E tests instead of retry-masking them (#1838)
Signed-off-by: Will Pfleger <[email protected]> Co-authored-by: npub1mn7jgtj4w2pd0g0zeuhxsa6jy6p0rewxz4kujt98my82ahfmp72sxjexk7 <dcfd242e557282d7a1e2cf2e6877522682f1e5c6156dc92ca7d90eaedd3b0f95@sprout-oss.stage.blox.sqprod.co>
1 parent 7e62a25 commit a653406

4 files changed

Lines changed: 87 additions & 2 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -234,15 +234,21 @@ jobs:
234234
run: just desktop-build
235235
- name: Desktop smoke e2e
236236
run: cd desktop && pnpm exec playwright test --project=smoke --shard=${{ matrix.shard }}/4
237+
- name: Summarize flaky tests
238+
if: ${{ !cancelled() }}
239+
run: node scripts/summarize-flaky-tests.mjs playwright-report.json "Desktop Smoke E2E (${{ matrix.shard }})"
240+
working-directory: desktop
237241
- name: Upload desktop smoke e2e artifacts
238-
if: failure()
242+
if: ${{ !cancelled() }}
239243
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
240244
with:
241245
name: desktop-smoke-e2e-artifacts-${{ matrix.shard }}
242246
path: |
243247
desktop/playwright-report
248+
desktop/playwright-report.json
244249
desktop/test-results
245250
if-no-files-found: ignore
251+
retention-days: 7
246252

247253
desktop:
248254
name: Desktop
@@ -443,16 +449,22 @@ jobs:
443449
run: bash scripts/setup-desktop-test-data.sh
444450
- name: Desktop relay-backed e2e
445451
run: cd desktop && pnpm exec playwright test --project=integration --shard=${{ matrix.shard }}/2
452+
- name: Summarize flaky tests
453+
if: ${{ !cancelled() }}
454+
run: node scripts/summarize-flaky-tests.mjs playwright-report.json "Desktop E2E Integration (${{ matrix.shard }}/2)"
455+
working-directory: desktop
446456
- name: Upload desktop integration artifacts
447-
if: failure()
457+
if: ${{ !cancelled() }}
448458
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
449459
with:
450460
name: desktop-e2e-integration-artifacts-${{ matrix.shard }}
451461
path: |
452462
desktop/playwright-report
463+
desktop/playwright-report.json
453464
desktop/test-results
454465
/tmp/buzz-relay.log
455466
if-no-files-found: ignore
467+
retention-days: 7
456468
- name: Save pnpm store cache
457469
if: github.event_name == 'push'
458470
uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5

‎desktop/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ node_modules
1212
dist
1313
dist-ssr
1414
playwright-report
15+
playwright-report.json
1516
test-results
1617
*.local
1718
playwright-report

‎desktop/playwright.config.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ export default defineConfig({
88
reporter: [
99
["list"],
1010
["html", { open: "never", outputFolder: "playwright-report" }],
11+
["json", { outputFile: "playwright-report.json" }],
1112
],
1213
use: {
1314
baseURL: "http://127.0.0.1:4173",
Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
import { appendFile, readFile } from "node:fs/promises";
2+
3+
// Playwright's `retries: 2` (desktop/playwright.config.ts) lets a test fail
4+
// then pass on retry with no durable signal beyond a one-line "N flaky" in
5+
// the console log — the exact gap that hid the stream.spec.ts membership
6+
// race (#1798) for months. This walks the JSON reporter's suite tree
7+
// (recursive: `describe` blocks nest as child `suites`) and appends any
8+
// `status === "flaky"` test to the job's GitHub Actions summary so retried
9+
// failures stay visible even when the shard ultimately goes green.
10+
//
11+
// Usage: node scripts/summarize-flaky-tests.mjs <report.json> <run-label>
12+
13+
function collectFlakyTests(suite, out) {
14+
for (const spec of suite.specs ?? []) {
15+
for (const test of spec.tests ?? []) {
16+
if (test.status !== "flaky") continue;
17+
out.push({
18+
title: `${suite.file} › ${spec.title}`,
19+
project: test.projectName,
20+
attempts: test.results?.length ?? 0,
21+
});
22+
}
23+
}
24+
for (const child of suite.suites ?? []) {
25+
collectFlakyTests(child, out);
26+
}
27+
}
28+
29+
const [reportPath, runLabel] = process.argv.slice(2);
30+
if (!reportPath || !runLabel) {
31+
console.error(
32+
"Usage: node scripts/summarize-flaky-tests.mjs <report.json> <run-label>",
33+
);
34+
process.exit(1);
35+
}
36+
37+
// This step runs `if: !cancelled()` purely to surface flaky tests — it must
38+
// never fail the job on its own, so a malformed/unexpected report (from a
39+
// Playwright version bump or a crashed run) is swallowed, not thrown.
40+
try {
41+
const report = JSON.parse(await readFile(reportPath, "utf8"));
42+
43+
const flaky = [];
44+
for (const suite of report.suites ?? []) {
45+
collectFlakyTests(suite, flaky);
46+
}
47+
48+
if (flaky.length > 0) {
49+
const escapeCell = (value) => String(value).replaceAll("|", "\\|");
50+
const rows = flaky
51+
.map(
52+
(t) =>
53+
`| ${escapeCell(t.title)} | ${escapeCell(t.project)} | ${t.attempts} |`,
54+
)
55+
.join("\n");
56+
const summary =
57+
`### Flaky tests — ${runLabel}\n\n` +
58+
`${flaky.length} test(s) failed at least once before passing on retry:\n\n` +
59+
"| Test | Project | Attempts |\n| --- | --- | --- |\n" +
60+
`${rows}\n`;
61+
62+
console.log(summary);
63+
64+
const summaryFile = process.env.GITHUB_STEP_SUMMARY;
65+
if (summaryFile) {
66+
await appendFile(summaryFile, `${summary}\n`);
67+
}
68+
}
69+
} catch (error) {
70+
console.log(`Skipping flaky-test summary: ${error.message}`);
71+
}

0 commit comments

Comments
 (0)