Skip to content

Commit 65693d2

Browse files
committed
test(ci): mark runner-aborted timeouts inside the runner instead of inferring them
1 parent 35ac931 commit 65693d2

11 files changed

Lines changed: 174 additions & 102 deletions

‎.fallowrc.json‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@
1919
"scripts/patch-xcuitest-runner-icon.ts",
2020
"scripts/runner-request-count/run.ts",
2121
"scripts/lib/contention-retry-run.ts",
22+
"scripts/vitest-runner-timeout-setup.ts",
23+
"test/contention-retry-fixtures/vitest.fixture.config.ts",
24+
"test/contention-retry-fixtures/timeout-provenance.fixture.ts",
2225
"src/utils/update-check-entry.ts",
2326
"test/scripts/metro-prepare-packaged-smoke.mjs",
2427
"test/integration/*.test.ts",
@@ -79,6 +82,11 @@
7982
"file": "scripts/lib/{contention-retry.ts,contention-retry-reporter.ts}",
8083
"exports": ["default", "SUBPROCESS_STUB_TESTS"]
8184
},
85+
{
86+
"comment": "Contention retry gate fixtures (#1419): the config default is loaded by the child `vitest run --config` the gate test spawns.",
87+
"file": "test/contention-retry-fixtures/vitest.fixture.config.ts",
88+
"exports": ["default"]
89+
},
8290
{
8391
"comment": "Mutation-lane seam: readTestScope is consumed by vitest.mutation.config.ts, a tool config outside --production analysis.",
8492
"file": "scripts/mutation/test-scope.ts",

‎docs/agents/testing.md‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -306,14 +306,15 @@ the retry policy cannot drift apart.
306306
The CI Coverage job runs the suite through `pnpm test:coverage:ci`
307307
(`scripts/lib/contention-retry-run.ts`), which applies one rule:
308308

309-
- **Timeouts only, decided from runner metadata.** A rerun happens only when *every* failure in the
310-
run is a test the runner itself aborted, in a listed file. Eligibility keys on structured runner
311-
facts test code cannot author (`isRunnerTimeout`): the test must have consumed its whole configured
312-
budget — `TestCase.options.timeout` versus `diagnostic().duration` — with Vitest's timeout template
313-
and a diff-free plain `Error` required as corroboration. A hand-thrown `Error` carrying the exact
314-
timeout message returns long before the budget and is refused; a test that really does run to its
315-
budget is aborted by the runner, so the error is the runner's. One assertion failure — in a listed
316-
file or not — fails the job on the first run, so a real regression can never be papered over.
309+
- **Timeouts only, proven by the runner.** A rerun happens only when *every* failure in the run is a
310+
test the runner itself aborted, in a listed file. Eligibility comes from a mark written inside the
311+
runner (`scripts/vitest-runner-timeout-setup.ts`, a setup file on every project): the runner owns
312+
the controller behind `context.signal` and aborts it with the timeout error it raises, so a test
313+
that merely *throws* the exact timeout message — immediately, or after blocking the event loop past
314+
its budget — never carries the mark. Error text is never consulted for eligibility;
315+
`test/contention-retry-fixtures/` drives those forgeries through a real Vitest run in the gate. One
316+
assertion failure — in a listed file or not — fails the job on the first run, so a real regression
317+
can never be papered over.
317318
- **Anything a rerun cannot re-check blocks the retry.** Unhandled errors, module load/setup errors,
318319
a coverage-threshold miss, or a nonzero exit no failed test explains are recorded as blockers
319320
(`scripts/lib/contention-retry-blockers.ts`) and fail the job, so a green retry can never erase a

‎scripts/lib/contention-retry-policy.test.ts‎

Lines changed: 50 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,15 @@ import assert from 'node:assert/strict';
77
import fs from 'node:fs';
88
import path from 'node:path';
99
import { test } from 'node:test';
10+
import { runCmdSync } from '../../src/utils/exec.ts';
1011
import { runWithContentionRetry, type TestRun } from './contention-retry-lane.ts';
1112
import { processBlockers } from './contention-retry-blockers.ts';
1213
import contentionRetryReporter, {
1314
failedTestCase,
1415
runBlockers,
1516
writeFailureReport,
1617
} from './contention-retry-reporter.ts';
18+
import { RUNNER_TIMEOUT_META } from './runner-timeout-meta.ts';
1719
import {
1820
CONTENTION_RETRY_FILES,
1921
expiredRetryEntries,
@@ -26,6 +28,7 @@ import {
2628
} from './contention-retry.ts';
2729

2830
const repoRoot = path.resolve(import.meta.dirname, '../..');
31+
const FIXTURE_CONFIG = 'test/contention-retry-fixtures/vitest.fixture.config.ts';
2932

3033
const LISTED = 'src/daemon/__tests__/request-router-open.test.ts';
3134
// Vitest's own timeout error, verbatim (@vitest/runner `makeTimeoutError`).
@@ -51,23 +54,21 @@ function assertionFailure(overrides: Partial<TestFailure> = {}): TestFailure {
5154
return failure({ message: ASSERTION_MESSAGE, timeout: false, ...overrides });
5255
}
5356

54-
const TIMEOUT_MS = 5000;
57+
/** What the runner-timeout setup file writes on a test the runner aborted. */
58+
const RUNNER_ABORTED = { [RUNNER_TIMEOUT_META]: true };
5559

5660
function testCaseStub(
5761
errors: ReadonlyArray<Record<string, unknown>>,
58-
overrides: { state?: string; timeoutMs?: number; durationMs?: number } = {},
62+
overrides: { state?: string; meta?: Record<string, unknown> } = {},
5963
): unknown {
6064
return {
6165
fullName: 'opens a session',
6266
module: { moduleId: `${repoRoot}/${LISTED}` },
63-
options: { timeout: overrides.timeoutMs ?? TIMEOUT_MS },
64-
diagnostic: () => ({ duration: overrides.durationMs ?? TIMEOUT_MS + 4 }),
67+
meta: () => overrides.meta ?? RUNNER_ABORTED,
6568
result: () => ({ state: overrides.state ?? 'failed', errors }),
6669
};
6770
}
6871

69-
const RAN_TO_BUDGET = { timeoutMs: TIMEOUT_MS, durationMs: TIMEOUT_MS + 4 };
70-
7172
function lane(runs: { first: TestRun; retry?: TestRun; today?: Date }): {
7273
result: ReturnType<typeof runWithContentionRetry>;
7374
rerun: string[][];
@@ -150,57 +151,61 @@ test('the expiry gate fails the run before any test executes', async () => {
150151
assert.deepEqual(rerun, []);
151152
});
152153

153-
test('only a test the runner aborted at its own budget classifies as a timeout', () => {
154-
assert.ok(isRunnerTimeout([VITEST_TIMEOUT], RAN_TO_BUDGET));
155-
assert.ok(
156-
isRunnerTimeout(
157-
[
158-
{
159-
name: 'Error',
160-
message:
161-
'Hook timed out in 10000ms.\nIf this is a long-running hook, pass a timeout value as the last argument or configure it globally with "hookTimeout".',
162-
},
163-
],
164-
RAN_TO_BUDGET,
165-
),
166-
);
167-
// Runner metadata is the primary evidence: test code cannot make its own
168-
// failure consume the whole configured budget and still choose the error.
169-
assert.ok(!isRunnerTimeout([VITEST_TIMEOUT], { timeoutMs: TIMEOUT_MS, durationMs: 12 }));
170-
assert.ok(!isRunnerTimeout([VITEST_TIMEOUT], { timeoutMs: undefined, durationMs: 99_999 }));
171-
assert.ok(!isRunnerTimeout([VITEST_TIMEOUT], { timeoutMs: 0, durationMs: 0 }));
172-
assert.ok(!isRunnerTimeout([], RAN_TO_BUDGET));
173-
// An assertion must not be able to talk its way into a retry, whatever it says.
154+
test('only the runner-owned abort mark classifies a failure as a timeout', () => {
155+
assert.ok(isRunnerTimeout([VITEST_TIMEOUT], RUNNER_ABORTED));
156+
// Without the runner's mark, no message and no error shape can earn a retry.
174157
for (const impostor of [
158+
{ name: 'Error', message: VITEST_TIMEOUT.message },
175159
{ name: 'AssertionError', message: VITEST_TIMEOUT.message },
176-
{ name: 'AssertionError', message: 'expected ETIMEDOUT, got ECONNRESET' },
177-
{ name: 'Error', message: `saw: ${VITEST_TIMEOUT.message}` },
178160
{ name: 'Error', message: 'connect ETIMEDOUT 127.0.0.1:8080' },
179161
{ name: 'Error', message: 'Closing timeout while tearing down the daemon' },
180-
{ name: 'Error', message: VITEST_TIMEOUT.message, expected: 'OK', actual: 'DEVICE_IN_USE' },
181162
]) {
182-
assert.ok(!isRunnerTimeout([impostor], RAN_TO_BUDGET), `${impostor.name}: ${impostor.message}`);
163+
assert.ok(!isRunnerTimeout([impostor], {}), `${impostor.name}: ${impostor.message}`);
164+
assert.ok(!isRunnerTimeout([impostor], undefined));
165+
assert.ok(!isRunnerTimeout([impostor], { [RUNNER_TIMEOUT_META]: 'yes' }));
183166
}
167+
// An aborted test that also produced an assertion diff is a regression.
168+
assert.ok(
169+
!isRunnerTimeout(
170+
[VITEST_TIMEOUT, { name: 'AssertionError', message: 'x', expected: 1, actual: 2 }],
171+
RUNNER_ABORTED,
172+
),
173+
);
174+
assert.ok(!isRunnerTimeout([], RUNNER_ABORTED));
184175
});
185176

186-
test('an exact-template Error thrown by the test itself never retries', async () => {
187-
// The forgery the classifier must refuse: right name, right message, right
188-
// shape — but the test returned long before its budget was up.
189-
const forged = failedTestCase(
190-
testCaseStub([{ name: 'Error', message: VITEST_TIMEOUT.message }], { durationMs: 7 }) as never,
191-
);
192-
assert.ok(forged);
193-
assert.equal(forged.timeout, false, 'a hand-thrown timeout message is not a runner timeout');
194-
const { result, rerun } = lane({ first: { ok: false, failures: [{ ...forged, file: LISTED }] } });
195-
const resolved = await result;
196-
assert.equal(resolved.ok, false);
197-
assert.deepEqual(rerun, []);
198-
assert.equal(resolved.envelope.data.retryCount, 0);
177+
test('a real Vitest run marks only runner-aborted tests as retry-eligible', () => {
178+
// The classifier's inputs come from the runner, so this case drives a real
179+
// child Vitest run over test/contention-retry-fixtures/ rather than stubs.
180+
const report = path.join(repoRoot, '.tmp/contention-retry/fixture-gate.json');
181+
fs.rmSync(report, { force: true });
182+
runCmdSync('pnpm', ['exec', 'vitest', 'run', '--config', FIXTURE_CONFIG], {
183+
cwd: repoRoot,
184+
env: { ...process.env, CONTENTION_RETRY_FAILURES: report },
185+
allowFailure: true,
186+
});
187+
const { failures } = parseFailureReport(JSON.parse(fs.readFileSync(report, 'utf8')), repoRoot);
188+
assert.deepEqual(Object.fromEntries(failures.map((entry) => [entry.testName, entry.timeout])), {
189+
'genuine runner timeout': true,
190+
'timeout message thrown immediately': false,
191+
// The forgery this gate exists for: blocking the event loop past the
192+
// budget does not abort the signal, so the throw stays a plain failure.
193+
'timeout message thrown after blocking the event loop past the budget': false,
194+
'assertion failure after blocking the event loop past the budget': false,
195+
'plain assertion failure': false,
196+
});
197+
// Only the genuine timeout may reach a rerun.
198+
const listed = failures.map((entry) => ({ ...entry, file: LISTED }));
199+
const plan = planContentionRetry(listed);
200+
assert.equal(plan.retry, false);
201+
assert.equal(planContentionRetry(listed.filter((entry) => entry.timeout)).retry, true);
199202
});
200203

201204
test('an assertion message quoting a timeout still fails on the first run', async () => {
202205
const impostor = failedTestCase(
203-
testCaseStub([{ name: 'AssertionError', message: VITEST_TIMEOUT.message }]) as never,
206+
testCaseStub([{ name: 'AssertionError', message: VITEST_TIMEOUT.message }], {
207+
meta: {},
208+
}) as never,
204209
);
205210
assert.ok(impostor);
206211
assert.equal(impostor.timeout, false);

‎scripts/lib/contention-retry-reporter.ts‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,7 @@ export function failedTestCase(testCase: TestCase): TestFailure | null {
2222
file: (testCase.module as TestModule).moduleId,
2323
testName: testCase.fullName,
2424
message: errors.map((error) => `${error.name}: ${error.message}`).join('\n'),
25-
timeout: isRunnerTimeout(errors, {
26-
timeoutMs: testCase.options.timeout,
27-
durationMs: testCase.diagnostic()?.duration,
28-
}),
25+
timeout: isRunnerTimeout(errors, testCase.meta()),
2926
};
3027
}
3128

‎scripts/lib/contention-retry-run.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ function configHash(): string {
9494
'scripts/lib/contention-retry-reporter.ts',
9595
'scripts/lib/contention-retry-blockers.ts',
9696
'scripts/lib/run-blocker-bus.ts',
97+
'scripts/vitest-runner-timeout-setup.ts',
9798
]) {
9899
hash.update(fs.readFileSync(path.join(repoRoot, file)));
99100
}

‎scripts/lib/contention-retry.ts‎

Lines changed: 9 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,16 @@
88
//
99
// - the retry set is ENUMERATED here, never a glob: a glob would silently
1010
// enroll every future file under a directory,
11-
// - only timeout-shaped failures retry; an assertion failure in a listed file
12-
// fails the job on the first run,
11+
// - only tests the runner itself aborted at their timeout retry; an assertion
12+
// failure in a listed file fails the job on the first run,
1313
// - one retry, of the failed files only, and every retry is reported in the
1414
// job summary so a permanently flaky file cannot hide behind a green check,
1515
// - each entry is an owned waiver in the ADR 0011 sense: a tracking issue plus
1616
// a review date, and an expired entry fails the gate until it is renewed or
1717
// removed.
1818

19+
import { runnerTimedOut } from './runner-timeout-meta.ts';
20+
1921
/** One enumerated retry-eligible file, owned like an ADR 0011 waiver. */
2022
export type ContentionRetryEntry = {
2123
/** Repo-relative test file path. Exact path — never a glob. */
@@ -184,10 +186,6 @@ export function expiredRetryEntries(now: Date): readonly ContentionRetryEntry[]
184186
return CONTENTION_RETRY_FILES.filter((entry) => entry.reviewBy < today);
185187
}
186188

187-
/** Vitest's runner timeout error, verbatim (`makeTimeoutError`, @vitest/runner 4.1.8). */
188-
const RUNNER_TIMEOUT_MESSAGE =
189-
/^(Test|Hook) timed out in \d+ms\.\nIf this is a long-running (test|hook), pass a timeout value as the last argument/;
190-
191189
/** An error as the lane reporter receives it from Vitest. */
192190
export type ReportedError = {
193191
name?: unknown;
@@ -196,28 +194,14 @@ export type ReportedError = {
196194
actual?: unknown;
197195
};
198196

199-
/** `TestCase.options.timeout` and `TestCase.diagnostic().duration`. */
200-
export type TestBudget = {
201-
timeoutMs: number | undefined;
202-
durationMs: number | undefined;
203-
};
204-
205197
/** A test the runner aborted at its timeout — the only failure this lane retries. */
206-
export function isRunnerTimeout(errors: readonly ReportedError[], budget: TestBudget): boolean {
207-
return consumedWholeBudget(budget) && errors.length > 0 && errors.every(raisedByRunnerTimeout);
208-
}
209-
210-
/** The runner aborts a test only once it has run for its whole configured budget. */
211-
function consumedWholeBudget({ timeoutMs, durationMs }: TestBudget): boolean {
212-
if (typeof timeoutMs !== 'number' || timeoutMs <= 0) return false;
213-
return typeof durationMs === 'number' && durationMs >= timeoutMs;
198+
export function isRunnerTimeout(errors: readonly ReportedError[], meta: unknown): boolean {
199+
return runnerTimedOut(meta) && errors.length > 0 && errors.every(hasNoAssertionDiff);
214200
}
215201

216-
/** The runner raises a plain `Error` holding its template, never an assertion diff. */
217-
function raisedByRunnerTimeout(error: ReportedError): boolean {
218-
if (error.name !== 'Error') return false;
219-
if (error.expected !== undefined || error.actual !== undefined) return false;
220-
return typeof error.message === 'string' && RUNNER_TIMEOUT_MESSAGE.test(error.message);
202+
/** A timeout that also failed an assertion is a regression, not contention. */
203+
function hasNoAssertionDiff(error: ReportedError): boolean {
204+
return error.expected === undefined && error.actual === undefined;
221205
}
222206

223207
/** One failed test case as read from a runner report. */

‎scripts/lib/runner-timeout-meta.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
// The provenance mark the runner-timeout setup file writes on a test, and the
2+
// contention retry lane (#1419) reads back through `TestCase.meta()`.
3+
4+
export const RUNNER_TIMEOUT_META = 'agentDeviceRunnerTimeout';
5+
6+
/** True when the runner itself aborted the test with its own timeout error. */
7+
export function runnerTimedOut(meta: unknown): boolean {
8+
return (meta as Record<string, unknown> | undefined)?.[RUNNER_TIMEOUT_META] === true;
9+
}
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
// Marks, from inside the runner, every test the runner itself aborted at its
2+
// timeout — the only failures the contention retry lane (#1419) may rerun.
3+
//
4+
// `context.signal` is the runner's own AbortSignal: only the runner holds its
5+
// controller, and it aborts with the timeout error it raised. A test body can
6+
// throw whatever it likes, including the runner's exact timeout text, without
7+
// ever aborting the signal.
8+
9+
import { beforeEach, onTestFailed } from 'vitest';
10+
import { RUNNER_TIMEOUT_META } from './lib/runner-timeout-meta.ts';
11+
12+
/** Distinguishes the timeout abort from a run cancellation, on the runner's own reason. */
13+
const RUNNER_TIMEOUT_REASON = /^(Test|Hook) timed out in \d+ms\./;
14+
15+
beforeEach(({ signal, task }) => {
16+
onTestFailed(() => {
17+
if (
18+
signal.aborted &&
19+
signal.reason instanceof Error &&
20+
RUNNER_TIMEOUT_REASON.test(signal.reason.message)
21+
) {
22+
task.meta[RUNNER_TIMEOUT_META] = true;
23+
}
24+
});
25+
});
Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
// Failures the contention retry policy must classify, run by the gate test in a
2+
// child Vitest process (test/contention-retry-fixtures/vitest.fixture.config.ts)
3+
// and never by the normal suites. Each name is asserted there.
4+
5+
import { expect, test } from 'vitest';
6+
7+
const RUNNER_TIMEOUT_MESSAGE =
8+
'Test timed out in 50ms.\nIf this is a long-running test, pass a timeout value as the last argument or configure it globally with "testTimeout".';
9+
10+
function blockPastBudget(): void {
11+
const until = Date.now() + 200;
12+
while (Date.now() < until) {
13+
// Deny the runner's timeout timer a turn until the budget is long gone.
14+
}
15+
}
16+
17+
test('genuine runner timeout', async () => {
18+
await new Promise((resolve) => setTimeout(resolve, 5_000));
19+
}, 50);
20+
21+
test('timeout message thrown immediately', () => {
22+
throw new Error(RUNNER_TIMEOUT_MESSAGE);
23+
}, 50);
24+
25+
test('timeout message thrown after blocking the event loop past the budget', () => {
26+
blockPastBudget();
27+
throw new Error(`FORGED ${RUNNER_TIMEOUT_MESSAGE}`);
28+
}, 50);
29+
30+
test('assertion failure after blocking the event loop past the budget', () => {
31+
blockPastBudget();
32+
expect('DEVICE_IN_USE').toBe('OK');
33+
}, 50);
34+
35+
test('plain assertion failure', () => {
36+
expect('DEVICE_IN_USE').toBe('OK');
37+
});
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
// Child-process Vitest config for the contention retry gate's fixtures: the
2+
// same setup and reporters the real lanes use, pointed at the fixture file.
3+
4+
import { defineConfig } from 'vitest/config';
5+
import { reporters } from '../../vitest.config.ts';
6+
7+
export default defineConfig({
8+
test: {
9+
include: ['test/contention-retry-fixtures/*.fixture.ts'],
10+
reporters: reporters(),
11+
setupFiles: ['scripts/vitest-runner-timeout-setup.ts'],
12+
},
13+
});

0 commit comments

Comments
 (0)