Skip to content

Commit 0776683

Browse files
committed
fix(ci): key the Bun crash retry on the signature that is actually stable
The Windows retry added for #2152 grepped for `panic(thread`. This repository already learned that is the wrong anchor: Bun emits BOTH `panic(thread 2852)` and `panic(main thread)` for the same class of failure, and devlog/_fin/260731_pr_issue_triage_round/050_windows_ci_flake_rca.md names `Internal assertion failure` as the stable fingerprint. Verified by literal probe -- panic(thread 3960) matched, panic(main thread) did not. The shard would have failed on exactly the crash the retry exists for. All three signature lists -- the macOS inline grep, the new Windows one, and is_bun_runtime_crash in run-bun-test-batches.sh -- now carry the same alternatives. The workflow comment already required them to stay in sync; nothing enforced it, so three copies drifted into two shapes. The contract test now pins the sync itself rather than the text, and pins that no list keys on the thread-numbered form. hasShellCommandHead is added because the existing exact-line matcher rejected the `| tee` the retry requires, while still rejecting an echoed or commented-out copy.
1 parent 8f04c9a commit 0776683

3 files changed

Lines changed: 55 additions & 5 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -516,7 +516,7 @@ jobs:
516516
if [ "$suite_status" -eq 0 ]; then
517517
exit 0
518518
fi
519-
if ! grep -Eqi 'oh no: Bun has crashed|Segmentation fault at address|Illegal instruction|Bus error|Aborted \(core dumped\)' "$suite_log"; then
519+
if ! grep -Eqi 'oh no: Bun has crashed|Internal assertion failure|Segmentation fault at address|Illegal instruction|Bus error|Aborted \(core dumped\)' "$suite_log"; then
520520
echo "::error::macOS suite failed on attempt ${attempt} (exit ${suite_status}); assertion failures are not retried."
521521
exit "$suite_status"
522522
fi
@@ -628,7 +628,7 @@ jobs:
628628
if [ "$suite_status" -eq 0 ]; then
629629
exit 0
630630
fi
631-
if ! grep -Eqi 'oh no: Bun has crashed|Segmentation fault at address|Illegal instruction|Bus error|panic\(thread' "$suite_log"; then
631+
if ! grep -Eqi 'oh no: Bun has crashed|Internal assertion failure|Segmentation fault at address|Illegal instruction|Bus error|Aborted \(core dumped\)' "$suite_log"; then
632632
echo "::error::Windows shard ${{ matrix.shard }}/4 failed on attempt ${attempt} (exit ${suite_status}); assertion failures are not retried."
633633
exit "$suite_status"
634634
fi

‎scripts/ci/run-bun-test-batches.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ is_bun_runtime_crash() {
7878
fi
7979

8080
grep -Eqi \
81-
'oh no: Bun has crashed|Segmentation fault at address|Illegal instruction|Bus error|Aborted \(core dumped\)' \
81+
'oh no: Bun has crashed|Internal assertion failure|Segmentation fault at address|Illegal instruction|Bus error|Aborted \(core dumped\)' \
8282
"$log_file"
8383
}
8484

‎tests/ci-workflows.test.ts‎

Lines changed: 52 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,21 @@ function hasExactShellCommand(run: string | undefined, expected: string): boolea
4848
.includes(expected);
4949
}
5050

51+
/**
52+
* Same intent as {@link hasExactShellCommand}, but for a command that is the HEAD of a
53+
* pipeline. The retry loops capture the suite with `… 2>&1 | tee "$suite_log"`, so an exact
54+
* whole-line match would reject the very shape the retry requires. Anchoring at the start of
55+
* the line still rejects an `echo` of the command or a commented-out copy, which is what the
56+
* exact match was protecting against.
57+
*/
58+
function hasShellCommandHead(run: string | undefined, expected: string): boolean {
59+
return (run ?? "")
60+
.split(/\r?\n/)
61+
.map(line => line.trim())
62+
.filter(line => line.length > 0 && !line.startsWith("#"))
63+
.some(line => line === expected || line.startsWith(`${expected} `));
64+
}
65+
5166
function expectSecureLinuxKeyringBootstrap(workflow: string): void {
5267
const smokeStep = workflow
5368
.split("- name: OS keyring create/read/delete smoke")[1]
@@ -224,17 +239,52 @@ describe("GitHub Actions hardening", () => {
224239
// Three composed-acceptance failures were that default firing on tests still working
225240
// at 41s. Pin the flag so the leg cannot silently drift back to the default.
226241
const windowsTestCommand = `bun test --isolate --timeout 60000 tests --shard=\${{ matrix.shard }}/${windowsShards.length}`;
227-
expect(hasExactShellCommand(`echo ${windowsTestCommand}`, windowsTestCommand)).toBe(false);
242+
expect(hasShellCommandHead(`echo ${windowsTestCommand}`, windowsTestCommand)).toBe(false);
228243
// Binding the assertion to an executable line is only half the guarantee: a
229244
// step carrying the exact command still runs nothing under `if: false`, and
230245
// the suite would stay green against a Windows leg that never tests. Require
231246
// the matching step to be unconditional.
232-
const windowsTestSteps = winSteps.filter(step => hasExactShellCommand(step.run, windowsTestCommand));
247+
const windowsTestSteps = winSteps.filter(step => hasShellCommandHead(step.run, windowsTestCommand));
233248
expect(windowsTestSteps.length).toBeGreaterThan(0);
234249
expect(windowsTestSteps.every(step => step.if === undefined)).toBe(true);
235250
expect(winSteps.some(step => step.if === "runner.environment == 'self-hosted'"
236251
&& step.run?.includes("git clean -xffd"))).toBe(true);
237252

253+
// The three crash-signature lists must stay identical, and they must not key on
254+
// `panic(thread`.
255+
//
256+
// Bun emits BOTH `panic(thread 2852)` and `panic(main thread)` for the same class of
257+
// failure, so a grep anchored on the numbered form silently misses half of them and the
258+
// shard fails on a crash it was supposed to retry. This repository already learned that
259+
// once — `devlog/_fin/260731_pr_issue_triage_round/050_windows_ci_flake_rca.md` names
260+
// `Internal assertion failure` as the stable fingerprint — and #2152 reintroduced it.
261+
// Three copies of one list is the real hazard, so pin the sync rather than the text.
262+
const crashSignatures = [
263+
"oh no: Bun has crashed",
264+
"Internal assertion failure",
265+
"Segmentation fault at address",
266+
"Illegal instruction",
267+
"Bus error",
268+
];
269+
const windowsTestRun = windowsTestSteps[0]?.run ?? "";
270+
const batchScript = await readText("scripts/ci/run-bun-test-batches.sh");
271+
for (const signature of crashSignatures) {
272+
expect(`macos:${signature}:${macosTestRun.includes(signature)}`).toBe(`macos:${signature}:true`);
273+
expect(`windows:${signature}:${windowsTestRun.includes(signature)}`).toBe(`windows:${signature}:true`);
274+
expect(`script:${signature}:${batchScript.includes(signature)}`).toBe(`script:${signature}:true`);
275+
}
276+
// The thread-numbered form must not be the anchor anywhere.
277+
expect(macosTestRun).not.toContain("panic\\(thread");
278+
expect(windowsTestRun).not.toContain("panic\\(thread");
279+
expect(batchScript).not.toContain("panic\\(thread");
280+
281+
// Windows carries the same bounded retry as macOS: one attempt, crash-only.
282+
expect(hasExactShellCommand(windowsTestRun, "set +e")).toBe(true);
283+
expect(windowsTestRun).toContain("for attempt in 1 2");
284+
expect(windowsTestRun).not.toContain("while true");
285+
expect(windowsTestRun).toContain("assertion failures are not retried");
286+
expect(windowsTestRun).toContain("failing after one retry");
287+
238288
// Every job that runs the root suite must build the GUI first, unconditionally.
239289
// Tests that fetch the served dashboard read their session bootstrap out of
240290
// `gui/dist/index.html`; with no build the server has no index to serve and the

0 commit comments

Comments
 (0)