Skip to content

Commit 97b61f2

Browse files
test(fuzz): replay the regression corpus through the worker watchdog (#1414)
A promoted hang case used to wedge the unit job until the CI timeout, because corpus replay called checkCase in-process. It now goes through the same worker-backed watchdog the nightly lane uses, so such a case fails against a 5s per-case budget; the file moves to the serialized subprocess-stub project with the rest of the worker-driven fuzz tests. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
1 parent bca12c8 commit 97b61f2

4 files changed

Lines changed: 38 additions & 33 deletions

File tree

‎docs/agents/testing.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -151,8 +151,9 @@ and replay path printed alongside the saved artifact.
151151
A nightly discovery reaches the unit lane by promotion, not hand-editing: the printed
152152
`promote:` command re-runs the downloaded artifact and appends it to
153153
`scripts/fuzz/corpus/regressions.json`, which `scripts/fuzz/corpus-replay.test.ts` replays on every
154-
PR. `scripts/fuzz/harness.test.ts` covers the harness itself — an untyped throw, an empty hint, and a
155-
wedged worker must each be reported, startup time is never charged against the per-case budget, and
154+
PR — through the same worker watchdog, so a promoted hang case fails against its per-case budget
155+
instead of wedging the unit job. `scripts/fuzz/harness.test.ts` covers the harness itself — an
156+
untyped throw, an empty hint, and a wedged worker must each be reported, startup time is never charged against the per-case budget, and
156157
every mode writes an envelope — using the broken-on-purpose targets in
157158
`scripts/fuzz/self-check-targets.ts` (also what `--self-check` runs in CI), so a regressed classifier
158159
or watchdog cannot pass silently. Adding a parser to the lane means adding a target to

‎scripts/fuzz/corpus-replay.test.ts‎

Lines changed: 28 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,28 @@
11
// Unit-lane replay of the parser fuzz regression corpus (#1414).
22
//
3-
// The nightly lane finds cases; this replays every case it ever found, in-process and
4-
// without a watchdog, so a regression fails in seconds on a PR instead of a night later.
5-
// Cases run synchronously here on purpose: a corpus case that hangs would hang the unit
6-
// suite, which is exactly the signal (the nightly lane is where hangs are diagnosed).
3+
// The nightly lane finds cases; this replays every case it ever found so a regression fails in
4+
// seconds on a PR instead of a night later. Cases go through the same worker-backed watchdog the
5+
// nightly lane uses: a promoted hang case must fail this test against its per-case budget, not
6+
// wedge the unit job until the CI timeout.
77

88
import fc from 'fast-check';
9-
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
9+
import { describe, expect, it } from 'vitest';
1010
import { arbitraryForTarget } from './arbitraries.ts';
1111
import { readCorpus } from './corpus.ts';
12-
import { checkCase } from './invariant.ts';
12+
import { runCases } from './execute.ts';
13+
import { describeFailure } from './invariant.ts';
1314
import { getFuzzTarget } from './registry.ts';
1415
import { FUZZ_TARGETS } from './targets.ts';
1516

17+
/**
18+
* Generous enough that a loaded CI runner never reports a healthy parser as hung, small enough
19+
* that a genuinely wedged case fails the file in seconds.
20+
*/
21+
const CASE_TIMEOUT_MS = 5_000;
22+
1623
describe('parser fuzz regression corpus', () => {
1724
const corpus = readCorpus();
1825

19-
// The batch-steps parser warns on deprecated step shapes; replaying those cases would
20-
// print one warning line per case into the unit-suite output.
21-
beforeEach(() => void vi.spyOn(process.stderr, 'write').mockReturnValue(true));
22-
afterEach(() => void vi.restoreAllMocks());
23-
2426
it('is non-empty and free of duplicates', () => {
2527
expect(corpus.length).toBeGreaterThan(0);
2628
const keys = corpus.map((entry) => `${entry.target}\u0000${entry.input}`);
@@ -35,17 +37,17 @@ describe('parser fuzz regression corpus', () => {
3537
}
3638
});
3739

38-
it.each(corpus.map((entry, index) => [index, entry] as const))(
39-
'case %i holds the typed-AppError invariant',
40-
(_index, entry) => {
41-
expect(checkCase(getFuzzTarget(entry.target), entry.input)).toBeNull();
42-
},
43-
);
44-
40+
// One worker per target rather than per case: startup is the only real cost here, and the
41+
// watchdog budget is per case either way.
4542
it.each(FUZZ_TARGETS.map((target) => [target.name, target] as const))(
46-
'%s seeds hold the invariant',
47-
(_name, target) => {
48-
for (const seed of target.seeds) expect(checkCase(target, seed)).toBeNull();
43+
'%s corpus cases and seeds hold the invariant, under the watchdog',
44+
async (name, target) => {
45+
const cases = [
46+
...target.seeds,
47+
...corpus.filter((entry) => entry.target === name).map((entry) => entry.input),
48+
];
49+
const failures = await runCases(target, cases, CASE_TIMEOUT_MS);
50+
expect(failures.map(describeFailure)).toEqual([]);
4951
},
5052
);
5153
});
@@ -59,10 +61,10 @@ describe('fuzz case generation', () => {
5961
expect(sample(7)).not.toEqual(sample(8));
6062
});
6163

62-
it('generates strings the target can be fed directly', () => {
63-
for (const input of fc.sample(arbitraryForTarget(target), { numRuns: 64, seed: 1 })) {
64-
expect(typeof input).toBe('string');
65-
expect(checkCase(target, input)).toBeNull();
66-
}
64+
it('generates strings the target can be fed directly', async () => {
65+
const inputs = fc.sample(arbitraryForTarget(target), { numRuns: 64, seed: 1 });
66+
expect(inputs.every((input) => typeof input === 'string')).toBe(true);
67+
const failures = await runCases(target, inputs, CASE_TIMEOUT_MS);
68+
expect(failures.map(describeFailure)).toEqual([]);
6769
});
6870
});

‎scripts/fuzz/execute.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,12 @@ type Session = { worker: Worker; ready: Promise<void> };
2929
/** A worker plus the promise that settles when it has finished importing the parsers. */
3030
function startSession(targetName: string): Session {
3131
const workerData: FuzzWorkerData = { targetName };
32-
// A restarted worker re-emits Node's type-stripping warning; one per hang would bury the report.
32+
// Type stripping is requested explicitly rather than inherited: under Vitest the parent's
33+
// execArgv carries no such flag, and the worker is a plain `.ts` file Node must strip itself.
34+
// The warning is silenced because a restarted worker re-emits it — one per hang buries the report.
3335
const worker = new Worker(WORKER_PATH, {
3436
workerData,
35-
execArgv: [...process.execArgv, '--disable-warning=ExperimentalWarning'],
37+
execArgv: ['--experimental-strip-types', '--disable-warning=ExperimentalWarning'],
3638
});
3739
const ready = new Promise<void>((resolve, reject) => {
3840
const timer = setTimeout(

‎vitest.config.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@ const SUBPROCESS_STUB_TESTS = [
1919
// Proves the parser fuzz harness still fails (#1414): every case spawns a node subprocess or a
2020
// worker thread and one target is a deliberate hang, so it waits real watchdog time.
2121
'scripts/fuzz/harness.test.ts',
22+
// Replays the fuzz regression corpus (#1414) through that same worker watchdog, so a promoted
23+
// hang case fails against its per-case budget instead of wedging the unit job.
24+
'scripts/fuzz/corpus-replay.test.ts',
2225
];
2326

2427
export default defineConfig({
@@ -41,9 +44,6 @@ export default defineConfig({
4144
include: [
4245
'src/**/*.test.ts',
4346
'scripts/__tests__/help-conformance-bench.test.ts',
44-
// Replays the parser fuzz regression corpus (#1414) in the unit lane; the
45-
// generating fuzz run itself is nightly (scripts/fuzz/run.ts).
46-
'scripts/fuzz/corpus-replay.test.ts',
4747
'scripts/__tests__/help-conformance-sample-outputs.test.ts',
4848
'scripts/__tests__/help-conformance-topic-coverage.test.ts',
4949
'test/skillgym/suites/local-cli-help-policy.test.ts',

0 commit comments

Comments
 (0)