Skip to content

Commit a46f24e

Browse files
simclaude
andcommitted
fix(#5170): run-tests fails a chunk whose registered tests did not all reach the report (#4031)
--test-force-exit (the Windows post-test hang backstop, #1051/#869) can end the node --test parent while part of a test file's results is still unread on the child's pipe (nodejs/node#64833). The runner then printed a smaller count and exited 0, so every reported count was a lower bound. The count of registered tests now comes from the child over a channel that does not share that pipe: scripts/lib/test-registration-ledger.cjs is loaded into each test-file child with --require, counts the test()/it() registrations made through node:test and appends one line per file at exit. The ndjson reporter records the kind (suite or test) of every pass/fail, and analyzeChunkAccounting compares the two per file. A chunk that exited 0 with registered > reported now fails, naming the chunk and the counts, and the remaining chunks still run. The check is one-sided (run-time subtests are reported but not registered), so it can miss a loss but cannot invent one. --test-force-exit stays the default, so the Windows hang protection is intact. Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
1 parent e3a2933 commit a46f24e

4 files changed

Lines changed: 607 additions & 5 deletions

File tree

‎scripts/lib/ndjson-reporter.cjs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,13 +95,19 @@ module.exports = async function ndjsonEventReporter(source) {
9595
event.type === 'test:pass' ||
9696
event.type === 'test:fail'
9797
) {
98-
const { file, name, nesting, testNumber } = event.data || {};
98+
const { file, name, nesting, testNumber, details } = event.data || {};
99+
// #4031: `kind` is node:test's `details.type` ('suite' for a suite,
100+
// otherwise a test). run-tests.cjs's per-chunk accounting counts the
101+
// pass/fail events that are NOT suites and compares them with the
102+
// registration ledger (scripts/lib/test-registration-ledger.cjs).
103+
const kind = details && typeof details.type === 'string' ? details.type : undefined;
99104
const line = `${JSON.stringify({
100105
type: event.type,
101106
file,
102107
name,
103108
nesting,
104109
testNumber,
110+
kind,
105111
ts: Date.now(),
106112
})}\n`;
107113
try {
Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
'use strict';
2+
3+
// Registration ledger preload for scripts/run-tests.cjs (#4031).
4+
//
5+
// Why this exists. The runner passes `--test-force-exit` to every chunk (the
6+
// Windows post-test hang backstop, #1051/#869). Under process isolation each
7+
// test file's child streams its results to the `node --test` parent over a
8+
// pipe, and a force-exit can end the parent while bytes of that pipe are still
9+
// unread (nodejs/node#64833). The reporters never see the lost tail, so the
10+
// run printed a smaller pass count and exited 0: a test that executed and
11+
// passed (or failed) was indistinguishable from one that never existed.
12+
//
13+
// The parent cannot know what it did not receive, so the count of registered
14+
// tests has to come from the child, over a channel that does not share the
15+
// pipe: this module is loaded into each test-file child with `--require`
16+
// (`--test` forwards it to the file children), counts every `test()` / `it()`
17+
// registration made through `node:test`, and appends ONE line to the file named
18+
// by GSD_RUN_TESTS_LEDGER_FILE when the child exits. The runner compares that
19+
// per-file count with the leaf test results its ndjson reporter actually
20+
// received (analyzeChunkAccounting in run-tests.cjs) and fails the chunk when
21+
// registered > reported.
22+
//
23+
// The check is deliberately one-sided. Only module-level registrations are
24+
// counted: a subtest created at run time through `t.test()` is not, so a
25+
// reported total can exceed the registered one, never the reverse, absent a
26+
// loss. A count that errs low can hide a loss; it cannot invent one.
27+
//
28+
// Inert unless it is inside a test-file child (NODE_TEST_CONTEXT is set by
29+
// `node --test` for those, not for the runner parent) AND a ledger path was
30+
// supplied, so requiring it anywhere else is a no-op. Never throws into the
31+
// code it observes.
32+
33+
const fs = require('fs');
34+
const path = require('path');
35+
const Module = require('module');
36+
37+
const MODIFIERS = new Set(['skip', 'todo', 'only']);
38+
// `describe`/`suite` register suites, not tests; the `it`/`test` calls inside
39+
// their bodies go through the wrapped exports like any other.
40+
const REGISTERING = new Set(['test', 'it']);
41+
42+
function install(ledgerPath) {
43+
let registered = 0;
44+
45+
// Proxies are cached per target so `t.test === t.test` stays true for code
46+
// that compares or memoizes the functions it destructured.
47+
const cache = new WeakMap();
48+
const counted = (fn) => {
49+
let proxy = cache.get(fn);
50+
if (proxy === undefined) {
51+
proxy = new Proxy(fn, {
52+
apply(target, thisArg, args) {
53+
registered += 1;
54+
return Reflect.apply(target, thisArg, args);
55+
},
56+
get(target, prop) {
57+
const value = Reflect.get(target, prop);
58+
return typeof prop === 'string' && MODIFIERS.has(prop) && typeof value === 'function'
59+
? counted(value)
60+
: value;
61+
},
62+
});
63+
cache.set(fn, proxy);
64+
}
65+
return proxy;
66+
};
67+
68+
let wrappedExports = null;
69+
let wrappedOf = null;
70+
const wrapNodeTest = (real) => {
71+
if (wrappedOf === real) return wrappedExports;
72+
wrappedOf = real;
73+
// The module export is itself callable (`require('node:test')('name', fn)`).
74+
wrappedExports = new Proxy(real, {
75+
apply(target, thisArg, args) {
76+
registered += 1;
77+
return Reflect.apply(target, thisArg, args);
78+
},
79+
get(target, prop) {
80+
const value = Reflect.get(target, prop);
81+
if (typeof value !== 'function' || typeof prop !== 'string') return value;
82+
return REGISTERING.has(prop) || MODIFIERS.has(prop) ? counted(value) : value;
83+
},
84+
});
85+
return wrappedExports;
86+
};
87+
88+
const originalLoad = Module._load;
89+
Module._load = function loadWithRegistrationCount(request) {
90+
const loaded = Reflect.apply(originalLoad, this, arguments);
91+
return request === 'node:test' && typeof loaded === 'function' ? wrapNodeTest(loaded) : loaded;
92+
};
93+
94+
process.on('exit', () => {
95+
try {
96+
const file = process.argv[1] ? path.resolve(process.argv[1]) : null;
97+
fs.appendFileSync(ledgerPath, `${JSON.stringify({ type: 'registered', file, count: registered })}\n`);
98+
} catch {
99+
// Best-effort: the ledger is a diagnostic channel and must never change
100+
// the exit status of the test file it observes.
101+
}
102+
});
103+
}
104+
105+
const ledgerPath = process.env.GSD_RUN_TESTS_LEDGER_FILE;
106+
if (ledgerPath && process.env.NODE_TEST_CONTEXT) install(ledgerPath);
107+
108+
module.exports = { install };

‎scripts/run-tests.cjs‎

Lines changed: 164 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,8 @@
3535
// See docs/TESTING-SUITES.md for full grouping policy.
3636
'use strict';
3737

38-
const { appendFileSync, readdirSync, readFileSync, mkdtempSync, rmSync, unlinkSync, writeFileSync } = require('fs');
39-
const { join, basename } = require('path');
38+
const { appendFileSync, readdirSync, readFileSync, realpathSync, mkdtempSync, rmSync, unlinkSync, writeFileSync } = require('fs');
39+
const { join, basename, resolve } = require('path');
4040
const { tmpdir } = require('os');
4141
const { pathToFileURL } = require('url');
4242
const { execFileSync, spawn } = require('child_process');
@@ -1251,6 +1251,129 @@ function analyzeChunkEvents(eventsPath) {
12511251
};
12521252
}
12531253

1254+
// #4031: per-file test accounting for one chunk. `--test-force-exit` can end
1255+
// the `node --test` parent while part of a test file's results are still
1256+
// unread on the child's pipe, so the reporters (and this runner's own ndjson
1257+
// events file) never see them and the run reports a smaller count with exit 0.
1258+
// The count of REGISTERED tests therefore comes from the child, through a
1259+
// channel that does not share that pipe: scripts/lib/test-registration-ledger.cjs
1260+
// (loaded with `--require`) appends one `{type:'registered', file, count}`
1261+
// line per test-file child, and the ndjson reporter records every leaf
1262+
// `test:pass`/`test:fail` it received. A file whose registered count exceeds
1263+
// its reported count lost results.
1264+
//
1265+
// One-sided by construction: the ledger counts module-level `test()`/`it()`
1266+
// calls only, while a subtest created at run time (`t.test()`) is reported but
1267+
// not registered, so reported can exceed registered, never the reverse, absent
1268+
// a loss. A count that errs low can hide a loss; it cannot invent one.
1269+
//
1270+
// `available` is false when either file is missing/unreadable or the ledger
1271+
// holds no `registered` line: the chunk could not be accounted, which the
1272+
// caller reports instead of reading it as "nothing was lost". Truncated
1273+
// trailing lines are skipped, as in analyzeChunkEvents. Paths are compared
1274+
// after path.resolve (case-folded on win32).
1275+
function normalizeAccountedPath(p) {
1276+
let resolved = resolve(p);
1277+
try {
1278+
// The child's argv[1] and the runner's event `file` are the same file but
1279+
// not necessarily the same spelling (a symlinked temp root, 8.3 names).
1280+
resolved = realpathSync.native(resolved);
1281+
} catch {
1282+
// Not on disk (a synthetic path in a unit test): the resolved spelling stands.
1283+
}
1284+
return process.platform === 'win32' ? resolved.toLowerCase() : resolved;
1285+
}
1286+
1287+
function analyzeChunkAccounting(eventsPath, ledgerPath) {
1288+
const reported = new Map(); // normalized file -> leaf pass+fail events received
1289+
const registered = new Map(); // normalized file -> registrations counted in the child
1290+
const display = new Map(); // normalized file -> path as first seen, for messages
1291+
let eventsRead = true;
1292+
let ledgerRead = true;
1293+
let sawRegisteredLine = false;
1294+
let eventsRaw = '';
1295+
let ledgerRaw = '';
1296+
try {
1297+
eventsRaw = readFileSync(eventsPath, 'utf8');
1298+
} catch {
1299+
eventsRead = false;
1300+
}
1301+
try {
1302+
ledgerRaw = readFileSync(ledgerPath, 'utf8');
1303+
} catch {
1304+
ledgerRead = false;
1305+
}
1306+
const parse = (raw, onEvent) => {
1307+
for (const line of raw.split('\n')) {
1308+
if (line.trim() === '') continue;
1309+
let evt;
1310+
try {
1311+
evt = JSON.parse(line);
1312+
} catch {
1313+
continue; // truncated trailing line
1314+
}
1315+
onEvent(evt);
1316+
}
1317+
};
1318+
parse(eventsRaw, (evt) => {
1319+
if ((evt.type !== 'test:pass' && evt.type !== 'test:fail') || typeof evt.file !== 'string') return;
1320+
if (evt.kind === 'suite') return;
1321+
const key = normalizeAccountedPath(evt.file);
1322+
display.set(key, display.get(key) || evt.file);
1323+
reported.set(key, (reported.get(key) || 0) + 1);
1324+
});
1325+
parse(ledgerRaw, (evt) => {
1326+
if (evt.type !== 'registered' || typeof evt.file !== 'string' || !Number.isInteger(evt.count)) return;
1327+
sawRegisteredLine = true;
1328+
const key = normalizeAccountedPath(evt.file);
1329+
display.set(key, display.get(key) || evt.file);
1330+
registered.set(key, (registered.get(key) || 0) + evt.count);
1331+
});
1332+
// A registered file with no reported entry under its own spelling is matched
1333+
// to the single UNCLAIMED reported file that shares its basename, so a path
1334+
// spelling the two sides disagree on (case on a case-insensitive filesystem,
1335+
// a mount alias) never reads as "every test of this file was lost". Zero or
1336+
// several candidates leave it unmatched, which IS reported as a shortfall.
1337+
const reportedFor = (key) => {
1338+
if (reported.has(key)) return reported.get(key);
1339+
const candidates = [...reported.keys()].filter(
1340+
(k) => !registered.has(k) && basename(k) === basename(key),
1341+
);
1342+
return candidates.length === 1 ? reported.get(candidates[0]) : 0;
1343+
};
1344+
const shortfalls = [];
1345+
let registeredTotal = 0;
1346+
let reportedTotal = 0;
1347+
for (const [key, count] of registered) {
1348+
const got = reportedFor(key);
1349+
registeredTotal += count;
1350+
reportedTotal += Math.min(got, count);
1351+
if (count > got) shortfalls.push({ file: display.get(key), registered: count, reported: got });
1352+
}
1353+
return {
1354+
available: eventsRead && ledgerRead && sawRegisteredLine,
1355+
shortfalls,
1356+
registeredTotal,
1357+
reportedTotal,
1358+
};
1359+
}
1360+
1361+
// #4031: the loud failure for a chunk whose registered tests are not all
1362+
// accounted for. Names the chunk and the counts, per file and in total.
1363+
function formatAccountingFailure(chunkNumber, chunkCount, accounting) {
1364+
const lost = accounting.registeredTotal - accounting.reportedTotal;
1365+
const lines = accounting.shortfalls.map(
1366+
(s) => ` ${basename(s.file)}: ${s.registered} registered, ${s.reported} reported (${s.registered - s.reported} unaccounted)`,
1367+
);
1368+
return (
1369+
`run-tests: chunk ${chunkNumber}/${chunkCount} FAILED test accounting — ` +
1370+
`${accounting.registeredTotal} tests registered, ${accounting.reportedTotal} reported ` +
1371+
`(${lost} unaccounted). The runner exited 0, but results for tests that registered never ` +
1372+
`reached the report (--test-force-exit can end the runner before a test file's results ` +
1373+
`are read, nodejs/node#64833), so a green count here would be a lower bound.\n${lines.join('\n')}`
1374+
);
1375+
}
1376+
12541377
// #3889: ranks a killed chunk's files heaviest-first using the same weigher
12551378
// the packer used to build the chunk, and flags any file the timings table
12561379
// has no measurement for at all (as opposed to one that IS measured but
@@ -1964,7 +2087,15 @@ async function main() {
19642087
];
19652088
const reporterOverhead = reporterArgs.reduce((sum, a) => sum + a.length + 1, 0);
19662089

1967-
const FIXED_OVERHEAD = process.execPath.length + '--test'.length + concurrency.length + (forceExit ? '--test-force-exit'.length + 1 : 0) + reporterOverhead + 8;
2090+
// #4031: the registration-ledger preload (see analyzeChunkAccounting). Its
2091+
// per-chunk output path travels via env (GSD_RUN_TESTS_LEDGER_FILE), like the
2092+
// events path, so only this fixed `--require <path>` counts toward argv.
2093+
const ledgerPreloadPath = join(__dirname, 'lib', 'test-registration-ledger.cjs');
2094+
const ledgerArgs = ['--require', ledgerPreloadPath];
2095+
const ledgerOverhead = ledgerArgs.reduce((sum, a) => sum + a.length + 1, 0);
2096+
const ledgerPathFor = (i) => join(eventsDir, `chunk-${String(i).padStart(3, '0')}.ledger.ndjson`);
2097+
2098+
const FIXED_OVERHEAD = process.execPath.length + '--test'.length + concurrency.length + (forceExit ? '--test-force-exit'.length + 1 : 0) + reporterOverhead + ledgerOverhead + 8;
19682099

19692100
// Convert the ms-denominated isolation bar into the packer's weight units
19702101
// by dividing by the live table's own mean duration — see the comment
@@ -2047,6 +2178,7 @@ async function main() {
20472178
console.error(`run-tests: chunk ${i + 1}/${chunks.length} — ${chunks[i].length} files`);
20482179
}
20492180
const chunkEventsPath = eventsPathFor(i);
2181+
const chunkLedgerPath = ledgerPathFor(i);
20502182
const chunkStartedAt = process.hrtime.bigint();
20512183
// #4936: the timeout diagnostic is printed from runChunk's own timer the
20522184
// moment the bound is exceeded — not from the result, which only arrives
@@ -2128,10 +2260,15 @@ async function main() {
21282260
...(forceExit ? ['--test-force-exit'] : []),
21292261
concurrency,
21302262
...reporterArgs,
2263+
...ledgerArgs,
21312264
...chunks[i],
21322265
],
21332266
{
2134-
env: { ...process.env, GSD_RUN_TESTS_EVENTS_FILE: chunkEventsPath },
2267+
env: {
2268+
...process.env,
2269+
GSD_RUN_TESTS_EVENTS_FILE: chunkEventsPath,
2270+
GSD_RUN_TESTS_LEDGER_FILE: chunkLedgerPath,
2271+
},
21352272
timeoutMs: chunkTimeoutMs,
21362273
graceMs: chunkKillGraceMs,
21372274
onTimeout: reportChunkTimeout,
@@ -2152,7 +2289,26 @@ async function main() {
21522289
);
21532290
}
21542291
}
2292+
// #4031: a chunk that exited 0 must also ACCOUNT for every test its files
2293+
// registered. Run before the success path below, which deletes the events
2294+
// file this reads; a shortfall turns the chunk into an ordinary failure
2295+
// (non-timeout, so the remaining chunks still run and every failure is
2296+
// visible in one pass).
2297+
let accountingFailed = false;
21552298
if (!result.timedOut && result.code === 0 && !result.signal && !result.error) {
2299+
const accounting = analyzeChunkAccounting(chunkEventsPath, chunkLedgerPath);
2300+
if (accounting.shortfalls.length > 0) {
2301+
accountingFailed = true;
2302+
console.error(formatAccountingFailure(i + 1, chunks.length, accounting));
2303+
} else if (!accounting.available) {
2304+
console.error(
2305+
`run-tests: WARNING: chunk ${i + 1}/${chunks.length} could not be accounted — the ` +
2306+
'registration ledger or the reporter events file is missing or empty, so a ' +
2307+
'dropped result would not be detected for this chunk.',
2308+
);
2309+
}
2310+
}
2311+
if (!accountingFailed && !result.timedOut && result.code === 0 && !result.signal && !result.error) {
21562312
console.error(
21572313
`run-tests: chunk ${i + 1}/${chunks.length} completed in ${elapsedMs.toFixed(0)}ms`,
21582314
);
@@ -2294,6 +2450,10 @@ module.exports = {
22942450
ISOLATION_BUDGET_FRACTION,
22952451
partitionIsolatedFiles,
22962452
analyzeChunkEvents,
2453+
// #4031: per-file registered-vs-reported accounting, exported so the
2454+
// shortfall and accounted arms are unit-testable on synthetic ledgers.
2455+
analyzeChunkAccounting,
2456+
formatAccountingFailure,
22972457
// #4936: the chunk watchdog, exported so its arms are unit-testable with
22982458
// injected spawn/platform seams (a real wedged Windows child is not
22992459
// reproducible on demand).

0 commit comments

Comments
 (0)