Skip to content

Commit 9a0b8b2

Browse files
test(ci): retry the failed files in the first run's project and coverage modes
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
1 parent 8bbdcf3 commit 9a0b8b2

4 files changed

Lines changed: 83 additions & 8 deletions

File tree

‎docs/agents/testing.md‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -358,7 +358,10 @@ The CI Coverage job runs the suite through `pnpm test:coverage:ci`
358358
sink drains it and refuses the rerun. **Any new gate reporter must call `recordRunBlocker`**, and
359359
must be ordered before the sink in `reporters()`.
360360
- **One retry, of the failed files only.** Not the suite, and never twice. Two timed-out tests in one
361-
file are one retry, and count as one.
361+
file are one retry, and count as one. The rerun keeps the first run's execution modes — the same
362+
`--project` selection and the same V8 instrumentation (`scripts/lib/contention-retry-args.ts`), so
363+
a coverage-job failure is never accepted by a run that could not reproduce it. Its coverage lands
364+
in `coverage/contention-retry/`, leaving the first run's report as the changed-line gate's evidence.
362365
- **Retries stay visible.** Every retried file is named in the job summary with its tracking issue
363366
and review date, and the run writes the shared scheduled-lane envelope
364367
(`scripts/lib/lane-envelope.ts`, #1430) with the retry count, so a permanently flaky file shows up
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
// Vitest argv for the two runs of the contention retry lane (#1419). The retry
2+
// must execute the failed files the same way the first run did — same projects,
3+
// same V8 instrumentation — or a failure the Coverage job produced could be
4+
// accepted by a run that could not reproduce it.
5+
6+
export type RunModes = { projects: readonly string[]; coverage: boolean };
7+
8+
/** Where the retry's own coverage lands, so the first run's report stays the gate's evidence. */
9+
export const RETRY_COVERAGE_DIR = 'coverage/contention-retry';
10+
11+
function projectArgs(modes: RunModes): string[] {
12+
return modes.projects.flatMap((name) => ['--project', name]);
13+
}
14+
15+
export function firstRunArgs(modes: RunModes): string[] {
16+
return [...projectArgs(modes), ...(modes.coverage ? ['--coverage'] : [])];
17+
}
18+
19+
export function retryRunArgs(modes: RunModes, files: readonly string[]): string[] {
20+
if (!modes.coverage) return [...projectArgs(modes), ...files];
21+
// Global thresholds describe the whole suite; a handful of files can never meet
22+
// them. A first-run threshold failure is a blocker, so the retry never runs.
23+
return [
24+
...projectArgs(modes),
25+
'--coverage',
26+
`--coverage.reportsDirectory=${RETRY_COVERAGE_DIR}`,
27+
'--coverage.thresholds.statements=0',
28+
'--coverage.thresholds.lines=0',
29+
...files,
30+
];
31+
}

‎scripts/lib/contention-retry-policy.test.ts‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import contentionRetryReporter, {
1616
writeFailureReport,
1717
} from './contention-retry-reporter.ts';
1818
import { RUNNER_TIMEOUT_META, RUNNER_TIMEOUT_TOKEN_ENV } from './runner-timeout-meta.ts';
19+
import { firstRunArgs, RETRY_COVERAGE_DIR, retryRunArgs } from './contention-retry-args.ts';
1920
import {
2021
CONTENTION_RETRY_FILES,
2122
expiredRetryEntries,
@@ -290,6 +291,41 @@ test('the Coverage lane keeps the configured reporters, failure sink included',
290291
);
291292
});
292293

294+
test('the retry reruns the failed files in the first run modes', () => {
295+
const modes = { projects: ['unit-core', 'subprocess-stub'], coverage: true };
296+
assert.deepEqual(firstRunArgs(modes), [
297+
'--project',
298+
'unit-core',
299+
'--project',
300+
'subprocess-stub',
301+
'--coverage',
302+
]);
303+
const retry = retryRunArgs(modes, [LISTED]);
304+
assert.deepEqual(retry, [
305+
'--project',
306+
'unit-core',
307+
'--project',
308+
'subprocess-stub',
309+
'--coverage',
310+
`--coverage.reportsDirectory=${RETRY_COVERAGE_DIR}`,
311+
// A subset of files can never meet whole-suite thresholds, and a first-run
312+
// threshold failure is a blocker, so the retry never runs after one.
313+
'--coverage.thresholds.statements=0',
314+
'--coverage.thresholds.lines=0',
315+
LISTED,
316+
]);
317+
// The gate reads the first run's report; the retry must not overwrite it.
318+
assert.notEqual(RETRY_COVERAGE_DIR, 'coverage');
319+
assert.deepEqual(retryRunArgs({ projects: [], coverage: false }, [LISTED]), [LISTED]);
320+
321+
const runner = fs.readFileSync(
322+
path.join(repoRoot, 'scripts/lib/contention-retry-run.ts'),
323+
'utf8',
324+
);
325+
assert.match(runner, /runAll: \(\) => runVitest\(firstRunArgs\(modes\)/);
326+
assert.match(runner, /runFiles: \(files\) =>\s*runVitest\(retryRunArgs\(modes, files\)/);
327+
});
328+
293329
test('non-test failures block the retry instead of being rerun away', async () => {
294330
const covered = processBlockers({
295331
ok: false,

‎scripts/lib/contention-retry-run.ts‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import fs from 'node:fs';
1212
import path from 'node:path';
1313
import { runCmdStreaming, runCmdSync } from '../../src/utils/exec.ts';
1414
import { parseScriptArgs } from './cli-args.ts';
15+
import { firstRunArgs, retryRunArgs, type RunModes } from './contention-retry-args.ts';
1516
import { runWithContentionRetry, type TestRun } from './contention-retry-lane.ts';
1617
import { FAILURE_FILE_ENV } from './contention-retry-reporter.ts';
1718
import { RUNNER_TIMEOUT_TOKEN_ENV } from './runner-timeout-meta.ts';
@@ -95,6 +96,7 @@ function configHash(): string {
9596
for (const file of [
9697
'vitest.config.ts',
9798
'scripts/lib/contention-retry.ts',
99+
'scripts/lib/contention-retry-args.ts',
98100
'scripts/lib/contention-retry-reporter.ts',
99101
'scripts/lib/contention-retry-blockers.ts',
100102
'scripts/lib/run-blocker-bus.ts',
@@ -105,16 +107,19 @@ function configHash(): string {
105107
return `sha256:${hash.digest('hex').slice(0, 16)}`;
106108
}
107109

108-
const projects = (args.project ?? '')
109-
.split(',')
110-
.map((entry) => entry.trim())
111-
.filter(Boolean)
112-
.flatMap((name) => ['--project', name]);
110+
const modes: RunModes = {
111+
projects: (args.project ?? '')
112+
.split(',')
113+
.map((entry) => entry.trim())
114+
.filter(Boolean),
115+
coverage: args.coverage,
116+
};
113117

114118
const startedAtMs = Date.now();
115119
const result = await runWithContentionRetry({
116-
runAll: () => runVitest([...projects, ...(args.coverage ? ['--coverage'] : [])], reportPath),
117-
runFiles: (files) => runVitest([...files], reportPath.replace(/\.json$/, '.retry.json')),
120+
runAll: () => runVitest(firstRunArgs(modes), reportPath),
121+
runFiles: (files) =>
122+
runVitest(retryRunArgs(modes, files), reportPath.replace(/\.json$/, '.retry.json')),
118123
commit: runCmdSync('git', ['rev-parse', 'HEAD'], { cwd: repoRoot }).stdout.trim(),
119124
configHash: configHash(),
120125
vitestVersion: vitestVersion(),

0 commit comments

Comments
 (0)