Skip to content

Commit da5c939

Browse files
authored
Merge pull request #79 from radroid/t3x/win-exit-diagnostics
fix(t3x): the Windows transient-crash retry could never fire; classify on the code vp prints (#47)
2 parents 5e1bf19 + abbbfcd commit da5c939

4 files changed

Lines changed: 375 additions & 3 deletions

File tree

‎.github/workflows/t3x-release.yml‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -296,10 +296,21 @@ jobs:
296296
# Classified on output, not on the exit code. Bash truncates a child's status to 8
297297
# bits, so 0xC0000142 can never appear in `$?` to be matched, and vp reports its own
298298
# status rather than the dead grandchild's anyway.
299-
if grep -qiE '0xC0000142|0xC0000005|STATUS_DLL_INIT_FAILED|Access violation' "$log"; then
300-
echo "::warning::Attempt $attempt hit a Windows process-startup failure. Retrying."
299+
#
300+
# The classifier moved out of an inline grep and into scripts/t3x/windows-exit-codes.mjs
301+
# because the grep was wrong (#47). It matched `0xC0000142|0xC0000005|
302+
# STATUS_DLL_INIT_FAILED|Access violation`, and the observed failure contains none of
303+
# them — vp prints the SIGNED DECIMAL Node reports, `FAILED (exit code: -1073741502)`.
304+
# The retry could never have fired. Matching every spelling is now covered by tests
305+
# built on the log from the issue, and the classifier also prints what the status
306+
# actually means, so a recurrence is diagnosed from this log rather than from a bare
307+
# negative integer.
308+
#
309+
# Exit 0 from it means retry; 1 means this is a real failure and stopping now is right.
310+
if node scripts/t3x/windows-exit-codes.mjs --classify "$log"; then
311+
echo "::warning::Attempt $attempt hit a transient Windows process-startup failure (see the classification above). Retrying."
301312
else
302-
echo "::error::build:desktop failed with exit $status (not a known transient)." >&2
313+
echo "::error::build:desktop failed with exit $status, and this is not a transient process-startup failure." >&2
303314
exit "$status"
304315
fi
305316
done
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
/*
2+
* Types for windows-exit-codes.mjs. Hand-written for the same reason as
3+
* render-release-notes.d.mts: the implementation is `.mjs` so the release workflow can run it on
4+
* the runner image's Node, and this package's tsconfig does not set `allowJs`.
5+
*/
6+
7+
export interface WindowsStatus {
8+
/** NTSTATUS symbol, as documented by Microsoft. */
9+
readonly name: string;
10+
/** The value as an unsigned 32-bit integer — the hex spelling Windows itself uses. */
11+
readonly unsigned: number;
12+
/** The same value as Node reports a child's status: signed 32-bit. */
13+
readonly signed: number;
14+
/** Whether another attempt is reasonable. */
15+
readonly transient: boolean;
16+
readonly meaning: string;
17+
readonly advice: string;
18+
}
19+
20+
export const WINDOWS_FATAL_STATUSES: readonly WindowsStatus[];
21+
22+
export function describeWindowsExitCode(code: number | string): WindowsStatus | undefined;
23+
24+
export function findWindowsFatalStatuses(logText: string): readonly WindowsStatus[];
25+
26+
export function isTransientProcessStartFailure(logText: string): boolean;
27+
28+
export function explainWindowsFailure(logText: string): string;

‎scripts/t3x/windows-exit-codes.mjs‎

Lines changed: 207 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,207 @@
1+
/*
2+
* Turns a Windows fatal exit status into something a human can act on, and decides whether it is
3+
* worth retrying.
4+
*
5+
* Issue #47: `pnpm dist:desktop:win:x64` intermittently died with
6+
*
7+
* [3] @t3tools/desktop#build: node scripts/build-preview-annotation-css.mjs FAILED (exit code: -1073741502)
8+
*
9+
* and an immediate re-run succeeded. `-1073741502` is `0xC0000142`, STATUS_DLL_INIT_FAILED — a
10+
* Windows *process startup* condition, not an error our code raised. Nothing in that line says so,
11+
* which sends whoever reads it looking for a bug in a script that is fine.
12+
*
13+
* THE FORM MATTERS. vp prints the signed 32-bit decimal, because that is what Node reports for a
14+
* child's status. The hex spelling appears nowhere in the output. The release workflow's original
15+
* classifier grepped for `0xC0000142|0xC0000005|STATUS_DLL_INIT_FAILED|Access violation` — none of
16+
* which the observed log contains — so the retry it was written to trigger could never have fired.
17+
* That is the bug this file exists to fix, and it is why every spelling is matched here rather
18+
* than whichever one seemed natural.
19+
*
20+
* Retryable and not-retryable are both worth naming. A status that means "the machine was under
21+
* pressure when this process started" deserves another attempt; one that means "this program
22+
* corrupted its own heap" does not, and quietly retrying it three times only makes the eventual
23+
* failure slower and more confusing.
24+
*
25+
* `.mjs` for the same reason as its siblings here: the workflow runs it on the runner image's
26+
* Node, with no `setup-node` step in front of it.
27+
*/
28+
29+
/**
30+
* @typedef {object} WindowsStatus
31+
* @property {string} name NTSTATUS symbol, as documented by Microsoft.
32+
* @property {number} unsigned The value as an unsigned 32-bit integer.
33+
* @property {number} signed The same value as Node reports it — signed 32-bit.
34+
* @property {boolean} transient Whether another attempt is reasonable.
35+
* @property {string} meaning What actually happened, in a sentence.
36+
* @property {string} advice What to do about it.
37+
*/
38+
39+
/** @type {readonly WindowsStatus[]} */
40+
export const WINDOWS_FATAL_STATUSES = [
41+
{
42+
name: "STATUS_DLL_INIT_FAILED",
43+
unsigned: 0xc0000142,
44+
signed: -1073741502,
45+
transient: true,
46+
meaning:
47+
"A DLL's initialisation routine failed while the process was starting, so the program never reached main().",
48+
advice:
49+
"Usually desktop-heap or session resource pressure from spawning several processes at once. Retry, and prefer serialising the build over widening the pool.",
50+
},
51+
{
52+
name: "STATUS_ACCESS_VIOLATION",
53+
unsigned: 0xc0000005,
54+
signed: -1073741819,
55+
transient: true,
56+
meaning: "The process read or wrote memory it does not own — a segfault, in Unix terms.",
57+
advice:
58+
"The weakest member of this table: it is a genuine crash, and it is only treated as retryable because it also shows up under the same concurrent-spawn pressure as STATUS_DLL_INIT_FAILED. If it recurs on retry, stop treating it as flaky.",
59+
},
60+
{
61+
name: "STATUS_DLL_NOT_FOUND",
62+
unsigned: 0xc0000135,
63+
signed: -1073741515,
64+
transient: false,
65+
meaning: "A DLL the program links against is not on the search path.",
66+
advice:
67+
"Deterministic, so retrying cannot help. Usually a missing Visual C++ redistributable or a toolchain step that did not run.",
68+
},
69+
{
70+
name: "STATUS_INVALID_IMAGE_FORMAT",
71+
unsigned: 0xc000007b,
72+
signed: -1073741701,
73+
transient: false,
74+
meaning:
75+
"An executable or DLL is the wrong architecture — a 32-bit image loaded into a 64-bit process, or the reverse.",
76+
advice: "Deterministic. Check which arch the failing dependency was built for.",
77+
},
78+
{
79+
name: "STATUS_HEAP_CORRUPTION",
80+
unsigned: 0xc0000374,
81+
signed: -1073740940,
82+
transient: false,
83+
meaning: "The process corrupted its own heap and Windows killed it.",
84+
advice:
85+
"A real bug, not resource pressure. Do not retry; capture the failing step and report it.",
86+
},
87+
{
88+
name: "STATUS_STACK_BUFFER_OVERRUN",
89+
unsigned: 0xc0000409,
90+
signed: -1073740791,
91+
transient: false,
92+
meaning:
93+
"A stack buffer was overrun and the security cookie check aborted the process. Also what Windows reports for a fail-fast abort.",
94+
advice: "A real bug. Do not retry.",
95+
},
96+
];
97+
98+
/**
99+
* Looks a status up by any of the spellings that appear in real logs: the signed decimal Node
100+
* reports, the unsigned decimal, the hex form, or the NTSTATUS name.
101+
*
102+
* @param {number | string} code
103+
* @returns {WindowsStatus | undefined}
104+
*/
105+
export function describeWindowsExitCode(code) {
106+
if (typeof code === "number") {
107+
return WINDOWS_FATAL_STATUSES.find((s) => s.signed === code || s.unsigned === code);
108+
}
109+
const token = String(code).trim();
110+
const numeric = /^-?\d+$/u.test(token)
111+
? Number(token)
112+
: /^0x[0-9a-f]+$/iu.test(token)
113+
? Number.parseInt(token, 16)
114+
: undefined;
115+
if (numeric !== undefined) return describeWindowsExitCode(numeric);
116+
return WINDOWS_FATAL_STATUSES.find((s) => s.name.toLowerCase() === token.toLowerCase());
117+
}
118+
119+
/**
120+
* Every status named anywhere in a build log, in table order.
121+
*
122+
* Matching is deliberately generous — signed decimal, unsigned decimal, hex, and the NTSTATUS
123+
* name — because which one appears depends on who printed it. vp prints signed decimal; Windows
124+
* dialogs print hex; a crash dump prints the name.
125+
*
126+
* @param {string} logText
127+
* @returns {readonly WindowsStatus[]}
128+
*/
129+
export function findWindowsFatalStatuses(logText) {
130+
const text = String(logText ?? "");
131+
return WINDOWS_FATAL_STATUSES.filter((status) => {
132+
const hex = status.unsigned.toString(16);
133+
const patterns = [
134+
// `\b` is wrong at both ends here. The leading `-` of the signed form is a non-word
135+
// character, so `\b-1073741502` never matches after `code: `; and a trailing `\b` would
136+
// still match inside `-10737415021`. Hence explicit "no digit or minus before, no digit
137+
// after", which is what stops a build id from being read as a crash.
138+
new RegExp(`(?<![\\d-])${status.signed}(?!\\d)`, "u"),
139+
new RegExp(`(?<![\\d-])${status.unsigned}(?!\\d)`, "u"),
140+
new RegExp(`0x0*${hex}\\b`, "iu"),
141+
new RegExp(`\\b${status.name}\\b`, "iu"),
142+
];
143+
return patterns.some((pattern) => pattern.test(text));
144+
});
145+
}
146+
147+
/**
148+
* True when every fatal status in the log is one worth retrying, and there is at least one.
149+
*
150+
* "Every", not "any": a log carrying both a heap corruption and a DLL-init failure is not a flaky
151+
* build, and retrying it would hide the corruption behind two more minutes of compute.
152+
*
153+
* @param {string} logText
154+
*/
155+
export function isTransientProcessStartFailure(logText) {
156+
const found = findWindowsFatalStatuses(logText);
157+
return found.length > 0 && found.every((status) => status.transient);
158+
}
159+
160+
/**
161+
* The human-readable report. Written for whoever is reading a failed CI run at the point where
162+
* the only other evidence is a negative integer.
163+
*
164+
* @param {string} logText
165+
*/
166+
export function explainWindowsFailure(logText) {
167+
const found = findWindowsFatalStatuses(logText);
168+
if (found.length === 0) return "";
169+
170+
const lines = [];
171+
for (const status of found) {
172+
lines.push(
173+
`${status.name} (0x${status.unsigned.toString(16).toUpperCase()}, reported by Node as ${status.signed})`,
174+
` What it means: ${status.meaning}`,
175+
` ${status.transient ? "Retryable" : "NOT retryable"}: ${status.advice}`,
176+
);
177+
}
178+
return lines.join("\n");
179+
}
180+
181+
// CLI: `node scripts/t3x/windows-exit-codes.mjs --classify <logfile>`
182+
//
183+
// Prints the explanation, then exits 0 when a retry is warranted and 1 when it is not — so the
184+
// caller can write `if node … --classify "$log"; then retry; fi` without parsing anything.
185+
if (
186+
process.argv[1] &&
187+
process.argv[1].endsWith("windows-exit-codes.mjs") &&
188+
process.argv.includes("--classify")
189+
) {
190+
const { readFileSync } = await import("node:fs");
191+
const file = process.argv[process.argv.indexOf("--classify") + 1];
192+
if (!file) {
193+
console.error("usage: windows-exit-codes.mjs --classify <logfile>");
194+
process.exit(2);
195+
}
196+
let text = "";
197+
try {
198+
text = readFileSync(file, "utf8");
199+
} catch (error) {
200+
console.error(`Could not read ${file}: ${error instanceof Error ? error.message : error}`);
201+
process.exit(2);
202+
}
203+
const explanation = explainWindowsFailure(text);
204+
if (explanation) console.error(explanation);
205+
else console.error("No known Windows fatal status appears in this log.");
206+
process.exit(isTransientProcessStartFailure(text) ? 0 : 1);
207+
}
Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
import { assert, describe, it } from "@effect/vitest";
2+
3+
import {
4+
describeWindowsExitCode,
5+
explainWindowsFailure,
6+
findWindowsFatalStatuses,
7+
isTransientProcessStartFailure,
8+
WINDOWS_FATAL_STATUSES,
9+
} from "./windows-exit-codes.mjs";
10+
11+
// Verbatim from issue #47. This is the only form the failure has ever actually been observed in,
12+
// so it is the case every classifier here has to handle — the hex spelling appears nowhere.
13+
const OBSERVED_LOG = `vp run: 0/3 cache hit (0%), 1 failed.
14+
15+
[1] @t3tools/web#build: vp build OK
16+
[2] t3#build: node scripts/cli.ts build OK
17+
[3] @t3tools/desktop#build: node scripts/build-preview-annotation-css.mjs FAILED (exit code: -1073741502)
18+
`;
19+
20+
describe("the status table", () => {
21+
it("keeps signed and unsigned forms of the same value in step", () => {
22+
for (const status of WINDOWS_FATAL_STATUSES) {
23+
assert.strictEqual(
24+
status.signed,
25+
status.unsigned - 2 ** 32,
26+
`${status.name}: signed and unsigned disagree`,
27+
);
28+
assert.isAbove(status.unsigned, 0x7fffffff, `${status.name} is not a fatal NTSTATUS`);
29+
}
30+
});
31+
32+
it("gives every status a meaning and advice, since a bare code is the problem", () => {
33+
for (const status of WINDOWS_FATAL_STATUSES) {
34+
assert.isAbove(status.meaning.length, 0, `${status.name} has no meaning`);
35+
assert.isAbove(status.advice.length, 0, `${status.name} has no advice`);
36+
}
37+
});
38+
});
39+
40+
describe("describeWindowsExitCode", () => {
41+
it("accepts every spelling the same status appears in", () => {
42+
const expected = "STATUS_DLL_INIT_FAILED";
43+
assert.strictEqual(describeWindowsExitCode(-1073741502)?.name, expected);
44+
assert.strictEqual(describeWindowsExitCode(0xc0000142)?.name, expected);
45+
assert.strictEqual(describeWindowsExitCode("-1073741502")?.name, expected);
46+
assert.strictEqual(describeWindowsExitCode("3221225794")?.name, expected);
47+
assert.strictEqual(describeWindowsExitCode("0xC0000142")?.name, expected);
48+
assert.strictEqual(describeWindowsExitCode("0xc0000142")?.name, expected);
49+
assert.strictEqual(describeWindowsExitCode("status_dll_init_failed")?.name, expected);
50+
});
51+
52+
it("returns nothing for an ordinary exit code", () => {
53+
assert.isUndefined(describeWindowsExitCode(1));
54+
assert.isUndefined(describeWindowsExitCode(0));
55+
assert.isUndefined(describeWindowsExitCode("not a code"));
56+
});
57+
});
58+
59+
describe("findWindowsFatalStatuses", () => {
60+
// The regression this file exists for. The release workflow's original classifier matched
61+
// `0xC0000142|0xC0000005|STATUS_DLL_INIT_FAILED|Access violation` — and the log below contains
62+
// none of them, so the retry it was written to trigger could never have fired.
63+
it("finds the status in the log exactly as vp printed it", () => {
64+
const found = findWindowsFatalStatuses(OBSERVED_LOG);
65+
assert.strictEqual(found.length, 1);
66+
assert.strictEqual(found[0]?.name, "STATUS_DLL_INIT_FAILED");
67+
});
68+
69+
it("does not match a longer number that merely contains a status", () => {
70+
assert.deepStrictEqual(findWindowsFatalStatuses("build id 31073741502991"), []);
71+
assert.deepStrictEqual(findWindowsFatalStatuses("code -10737415021"), []);
72+
});
73+
74+
it("finds nothing in an ordinary failure", () => {
75+
assert.deepStrictEqual(findWindowsFatalStatuses("FAILED (exit code: 1)"), []);
76+
assert.deepStrictEqual(findWindowsFatalStatuses(""), []);
77+
});
78+
79+
it("finds every distinct status when a log carries more than one", () => {
80+
const names = findWindowsFatalStatuses(
81+
"first: -1073741502\nsecond: STATUS_HEAP_CORRUPTION",
82+
).map((status) => status.name);
83+
assert.includeMembers(names, ["STATUS_DLL_INIT_FAILED", "STATUS_HEAP_CORRUPTION"]);
84+
});
85+
});
86+
87+
describe("isTransientProcessStartFailure", () => {
88+
it("says retry for the observed failure", () => {
89+
assert.isTrue(isTransientProcessStartFailure(OBSERVED_LOG));
90+
});
91+
92+
it("says do not retry for a crash that is a real bug", () => {
93+
assert.isFalse(isTransientProcessStartFailure("FAILED (exit code: -1073740940)"));
94+
assert.isFalse(isTransientProcessStartFailure("FAILED (exit code: -1073741701)"));
95+
});
96+
97+
// "Every", not "any". A log carrying a heap corruption as well is not a flaky build, and
98+
// retrying it hides the corruption behind two more minutes of compute.
99+
it("refuses to call a mixed log transient", () => {
100+
assert.isFalse(
101+
isTransientProcessStartFailure("[1] exit code: -1073741502\n[2] exit code: -1073740940"),
102+
);
103+
});
104+
105+
it("says nothing about a build that failed for an ordinary reason", () => {
106+
assert.isFalse(isTransientProcessStartFailure("FAILED (exit code: 1)"));
107+
});
108+
});
109+
110+
describe("explainWindowsFailure", () => {
111+
it("names the status in all three forms, so any of them can be searched for", () => {
112+
const text = explainWindowsFailure(OBSERVED_LOG);
113+
assert.include(text, "STATUS_DLL_INIT_FAILED");
114+
assert.include(text, "0xC0000142");
115+
assert.include(text, "-1073741502");
116+
});
117+
118+
it("says plainly whether a retry is warranted", () => {
119+
assert.include(explainWindowsFailure(OBSERVED_LOG), "Retryable");
120+
assert.include(explainWindowsFailure("exit code: -1073740940"), "NOT retryable");
121+
});
122+
123+
it("stays quiet when there is nothing to explain", () => {
124+
assert.strictEqual(explainWindowsFailure("FAILED (exit code: 1)"), "");
125+
});
126+
});

0 commit comments

Comments
 (0)