Skip to content

Commit 5d91d7a

Browse files
test(fuzz): run envelope, artifact promotion, and harness self-check tests (#1414)
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
1 parent 8e80c3a commit 5d91d7a

19 files changed

Lines changed: 632 additions & 178 deletions

‎.github/workflows/replays-nightly.yml‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,11 @@ jobs:
3838
- name: Setup toolchain
3939
uses: ./.github/actions/setup-node-pnpm
4040

41+
# A lane whose classifier or watchdog regressed would pass forever. This runs the
42+
# broken-on-purpose targets first and fails unless each violation is caught.
43+
- name: Self-check the harness
44+
run: pnpm fuzz:parsers --self-check
45+
4146
- name: Fuzz parsers
4247
env:
4348
FUZZ_ITERATIONS: ${{ github.event.inputs.fuzz-iterations || '50000' }}
@@ -48,13 +53,26 @@ jobs:
4853
--seed "$FUZZ_SEED" \
4954
--artifact-dir .tmp/fuzz
5055
51-
- name: Upload failing cases
52-
if: failure()
56+
# Uploaded on pass as well as failure: .tmp/fuzz always holds run-envelope.json
57+
# (schemaVersion, commit/ref/run provenance, seed, config, per-target durations,
58+
# result), which is what freshness/health monitoring reads for a green run.
59+
- name: Upload run envelope and failing cases
60+
if: always()
5361
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
5462
with:
55-
name: parser-fuzz-failing-cases
63+
name: parser-fuzz-run-${{ github.run_id }}-${{ github.run_attempt }}
5664
path: .tmp/fuzz
57-
if-no-files-found: ignore
65+
if-no-files-found: warn
66+
67+
- name: Summarize
68+
if: always()
69+
run: |
70+
{
71+
echo '### Parser fuzz lane'
72+
echo '```json'
73+
cat .tmp/fuzz/run-envelope.json
74+
echo '```'
75+
} >> "$GITHUB_STEP_SUMMARY"
5876
5977
nightly-android:
6078
name: Android Replay Suite

‎docs/agents/testing.md‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -129,14 +129,25 @@ hangs (a worker-thread watchdog attributes a stall to the exact input).
129129
```sh
130130
pnpm fuzz:parsers # all targets, 2,000 cases each, seed 1
131131
pnpm fuzz:parsers --target selector --iterations 50000 --seed 7
132-
pnpm fuzz:parsers --input-file .tmp/fuzz/<case>.json # repro one saved failing case
132+
pnpm fuzz:parsers --input-file .tmp/fuzz/<case>.json # repro a saved case
133+
pnpm fuzz:parsers --input-file .tmp/fuzz/<case>.json --append-corpus # …and pin it
134+
pnpm fuzz:parsers --self-check # require the harness to still fail
133135
```
134136

135137
The generating run is nightly (`Parser Fuzz Lane` in `.github/workflows/replays-nightly.yml`, seeded
136-
by the run number); failing cases upload as artifacts and print that repro command. Every case the
137-
fuzzer catches is appended to `scripts/fuzz/corpus/regressions.json` (`--append-corpus`) and replayed
138-
in the unit lane by `scripts/fuzz/corpus-replay.test.ts`, so a fixed parser stays fixed on PRs.
139-
Adding a parser to the lane means adding a target to `scripts/fuzz/targets.ts` — nothing else.
138+
by the run number). Every run — green or red — writes `<artifact-dir>/run-envelope.json`
139+
(`schemaVersion`, commit/ref/run provenance, seed and config, per-target cases/failures/durations,
140+
result, repro commands) and uploads the artifact directory, so a passing scheduled run is auditable
141+
and not just silent; failing cases are uploaded alongside it.
142+
143+
A nightly discovery reaches the unit lane by promotion, not hand-editing: the printed
144+
`promote:` command re-runs the downloaded artifact and appends it to
145+
`scripts/fuzz/corpus/regressions.json`, which `scripts/fuzz/corpus-replay.test.ts` replays on every
146+
PR. `scripts/fuzz/harness.test.ts` covers the harness itself — an untyped throw, an empty hint, and a
147+
wedged worker must each be reported — using the broken-on-purpose targets in
148+
`scripts/fuzz/self-check-targets.ts` (also what `--self-check` runs in CI), so a regressed classifier
149+
or watchdog cannot pass silently. Adding a parser to the lane means adding a target to
150+
`scripts/fuzz/targets.ts` — nothing else.
140151

141152
## Live web smoke
142153

‎scripts/fuzz/corpus-replay.test.ts‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@
88
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
99
import { readCorpus } from './corpus.ts';
1010
import { checkCase } from './invariant.ts';
11-
import { FUZZ_TARGETS, getFuzzTarget } from './targets.ts';
11+
import { getFuzzTarget } from './registry.ts';
12+
import { FUZZ_TARGETS } from './targets.ts';
1213
import { generateCases } from './mutate.ts';
1314

1415
describe('parser fuzz regression corpus', () => {
@@ -25,9 +26,10 @@ describe('parser fuzz regression corpus', () => {
2526
expect(new Set(keys).size).toBe(keys.length);
2627
});
2728

28-
it('names only known targets and explains every entry', () => {
29+
it('names only real parser targets and explains every entry', () => {
30+
const parserTargets = new Set<string>(FUZZ_TARGETS.map((target) => target.name));
2931
for (const entry of corpus) {
30-
expect(() => getFuzzTarget(entry.target)).not.toThrow();
32+
expect(parserTargets).toContain(entry.target);
3133
expect(entry.note.trim()).not.toBe('');
3234
}
3335
});

‎scripts/fuzz/corpus.ts‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
import fs from 'node:fs';
99
import path from 'node:path';
1010
import { fileURLToPath } from 'node:url';
11-
import type { FuzzTargetName } from './targets.ts';
11+
import type { FuzzTargetName } from './target-types.ts';
1212

1313
export type CorpusEntry = {
1414
target: FuzzTargetName;
@@ -18,11 +18,11 @@ export type CorpusEntry = {
1818
note: string;
1919
};
2020

21-
const CORPUS_PATH = path.join(
22-
path.dirname(fileURLToPath(import.meta.url)),
23-
'corpus',
24-
'regressions.json',
25-
);
21+
// AGENT_DEVICE_FUZZ_CORPUS retargets the corpus file so the harness's own tests can exercise
22+
// the real promotion path against a scratch file instead of mutating the checked-in one.
23+
const CORPUS_PATH =
24+
process.env.AGENT_DEVICE_FUZZ_CORPUS ??
25+
path.join(path.dirname(fileURLToPath(import.meta.url)), 'corpus', 'regressions.json');
2626

2727
export function readCorpus(corpusPath = CORPUS_PATH): CorpusEntry[] {
2828
const raw = fs.readFileSync(corpusPath, 'utf8');

‎scripts/fuzz/envelope.ts‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
// Always-produced run envelope for the parser fuzz lane (#1414, scheduled-lane contract of #1412).
2+
//
3+
// A scheduled lane has to be auditable when it passes, not only when it fails: freshness and
4+
// health monitoring need a machine-readable record of *which* code, config, and seed produced
5+
// the verdict. So every run — green or red — writes one JSON envelope next to the artifacts.
6+
// #1430 is standardizing this shape across lanes; when it lands this module becomes the mapping
7+
// onto the shared writer rather than a second definition.
8+
9+
import fs from 'node:fs';
10+
import path from 'node:path';
11+
import type { FuzzFailure } from './invariant.ts';
12+
13+
const SCHEMA_VERSION = 1;
14+
const FILENAME = 'run-envelope.json';
15+
16+
export type FuzzTargetRun = {
17+
target: string;
18+
cases: number;
19+
failures: number;
20+
durationMs: number;
21+
};
22+
23+
export type FuzzRunEnvelope = {
24+
schemaVersion: number;
25+
lane: 'parser-fuzz';
26+
result: 'pass' | 'fail';
27+
startedAt: string;
28+
finishedAt: string;
29+
durationMs: number;
30+
provenance: {
31+
commitSha: string | null;
32+
ref: string | null;
33+
workflowRunId: string | null;
34+
workflowRunAttempt: string | null;
35+
nodeVersion: string;
36+
tool: string;
37+
corpusEntries: number;
38+
};
39+
config: {
40+
mode: 'generate' | 'replay-corpus' | 'self-check';
41+
seed: number;
42+
iterations: number;
43+
caseTimeoutMs: number;
44+
targets: string[];
45+
};
46+
targetRuns: FuzzTargetRun[];
47+
failures: (FuzzFailure & { artifact?: string })[];
48+
reproCommands: string[];
49+
};
50+
51+
function envOrNull(name: string): string | null {
52+
return process.env[name] ?? null;
53+
}
54+
55+
/** GitHub Actions exports these; a local run simply records `null`. */
56+
function provenanceFromEnv() {
57+
return {
58+
commitSha: envOrNull('GITHUB_SHA'),
59+
ref: envOrNull('GITHUB_REF'),
60+
workflowRunId: envOrNull('GITHUB_RUN_ID'),
61+
workflowRunAttempt: envOrNull('GITHUB_RUN_ATTEMPT'),
62+
nodeVersion: process.version,
63+
tool: 'scripts/fuzz/run.ts',
64+
};
65+
}
66+
67+
export function buildEnvelope(input: {
68+
startedAt: number;
69+
finishedAt: number;
70+
config: FuzzRunEnvelope['config'];
71+
corpusEntries: number;
72+
targetRuns: FuzzTargetRun[];
73+
failures: (FuzzFailure & { artifact?: string })[];
74+
reproCommands: string[];
75+
}): FuzzRunEnvelope {
76+
return {
77+
schemaVersion: SCHEMA_VERSION,
78+
lane: 'parser-fuzz',
79+
result: input.failures.length === 0 ? 'pass' : 'fail',
80+
startedAt: new Date(input.startedAt).toISOString(),
81+
finishedAt: new Date(input.finishedAt).toISOString(),
82+
durationMs: input.finishedAt - input.startedAt,
83+
provenance: { ...provenanceFromEnv(), corpusEntries: input.corpusEntries },
84+
config: input.config,
85+
targetRuns: input.targetRuns,
86+
failures: input.failures,
87+
reproCommands: input.reproCommands,
88+
};
89+
}
90+
91+
/** Writes the envelope into the artifact dir; returns its path. */
92+
export function writeEnvelope(artifactDir: string, envelope: FuzzRunEnvelope): string {
93+
fs.mkdirSync(artifactDir, { recursive: true });
94+
const file = path.join(artifactDir, FILENAME);
95+
fs.writeFileSync(file, `${JSON.stringify(envelope, null, 2)}\n`);
96+
return file;
97+
}

‎scripts/fuzz/execute.ts‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
// Case execution with hang detection for the parser fuzz lane (#1414).
2+
//
3+
// Cases run in a worker thread that publishes the index of the case it is about to run; this
4+
// module watchdogs that cursor, so a parser stuck in its own tick is reported as a `hang`
5+
// against the exact input instead of wedging the lane.
6+
7+
import path from 'node:path';
8+
import { fileURLToPath } from 'node:url';
9+
import { Worker } from 'node:worker_threads';
10+
import type { FuzzFailure } from './invariant.ts';
11+
import type { FuzzTarget } from './target-types.ts';
12+
import type { FuzzWorkerData, FuzzWorkerMessage } from './worker.ts';
13+
14+
const WORKER_PATH = path.join(path.dirname(fileURLToPath(import.meta.url)), 'worker.ts');
15+
16+
export async function runTarget(
17+
target: FuzzTarget,
18+
cases: string[],
19+
caseTimeoutMs: number,
20+
): Promise<FuzzFailure[]> {
21+
const progress = new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT);
22+
const cursor = new Int32Array(progress);
23+
const workerData: FuzzWorkerData = { targetName: target.name, cases, progress };
24+
const worker = new Worker(WORKER_PATH, { workerData });
25+
const failures: FuzzFailure[] = [];
26+
27+
return await new Promise<FuzzFailure[]>((resolve, reject) => {
28+
let settled = false;
29+
let lastIndex = -1;
30+
// Set again when the worker reports ready: startup is not case time.
31+
let lastAdvance = Date.now();
32+
33+
const finish = (result: FuzzFailure[]) => {
34+
if (settled) return;
35+
settled = true;
36+
clearInterval(watchdog);
37+
void worker.terminate();
38+
resolve(result);
39+
};
40+
41+
const watchdog = setInterval(() => {
42+
const index = Atomics.load(cursor, 0);
43+
if (index !== lastIndex) {
44+
lastIndex = index;
45+
lastAdvance = Date.now();
46+
return;
47+
}
48+
if (Date.now() - lastAdvance < caseTimeoutMs) return;
49+
failures.push({
50+
target: target.name,
51+
input: cases[index] ?? '',
52+
kind: 'hang',
53+
detail: `case ${index} did not finish within ${caseTimeoutMs}ms`,
54+
});
55+
finish(failures);
56+
}, 50);
57+
58+
worker.on('message', (message: FuzzWorkerMessage) => {
59+
if (message.kind === 'ready') lastAdvance = Date.now();
60+
else if (message.kind === 'failure') failures.push(message.failure);
61+
else finish(failures);
62+
});
63+
worker.on('error', (error) => {
64+
if (settled) return;
65+
settled = true;
66+
clearInterval(watchdog);
67+
reject(error);
68+
});
69+
worker.on('exit', () => finish(failures));
70+
});
71+
}

‎scripts/fuzz/harness.test.ts‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
// Tests of the fuzz harness itself (#1414).
2+
//
3+
// The corpus replay only proves clean inputs pass, so a regressed classifier or watchdog would
4+
// leave every test green. These run the harness against the broken-on-purpose targets through
5+
// the real CLI — same worker, watchdog, artifact, and promotion path a nightly failure takes —
6+
// and require each failure kind to be reported.
7+
8+
import { execFileSync } from 'node:child_process';
9+
import fs from 'node:fs';
10+
import os from 'node:os';
11+
import path from 'node:path';
12+
import { fileURLToPath } from 'node:url';
13+
import { describe, expect, it } from 'vitest';
14+
import { checkCase } from './invariant.ts';
15+
import { SELF_CHECK_TARGETS } from './self-check-targets.ts';
16+
17+
const FUZZ_DIR = path.dirname(fileURLToPath(import.meta.url));
18+
const RUN = path.join(FUZZ_DIR, 'run.ts');
19+
20+
function runHarness(
21+
args: readonly string[],
22+
env: Record<string, string> = {},
23+
): { status: number; stdout: string } {
24+
try {
25+
const stdout = execFileSync(process.execPath, ['--experimental-strip-types', RUN, ...args], {
26+
encoding: 'utf8',
27+
env: { ...process.env, ...env },
28+
});
29+
return { status: 0, stdout };
30+
} catch (error) {
31+
const failure = error as { status?: number; stdout?: string };
32+
return { status: failure.status ?? 1, stdout: failure.stdout ?? '' };
33+
}
34+
}
35+
36+
function targetNamed(name: string) {
37+
const target = SELF_CHECK_TARGETS.find((candidate) => candidate.name === name);
38+
if (!target) throw new Error(`missing self-check target ${name}`);
39+
return target;
40+
}
41+
42+
describe('fuzz invariant classifier', () => {
43+
it('reports a bare Error as untyped-throw', () => {
44+
const failure = checkCase(targetNamed('self-check-untyped-throw'), 'case');
45+
expect(failure?.kind).toBe('untyped-throw');
46+
expect(failure?.detail).toContain('Error: self-check untyped throw');
47+
});
48+
49+
it('reports an AppError with a blank hint as empty-hint', () => {
50+
const failure = checkCase(targetNamed('self-check-empty-hint'), 'case');
51+
expect(failure?.kind).toBe('empty-hint');
52+
});
53+
});
54+
55+
describe('fuzz harness self-check', () => {
56+
it('catches an untyped throw, an empty hint, and a wedged worker', () => {
57+
const { status, stdout } = runHarness(['--self-check', '--case-timeout-ms', '750']);
58+
expect(stdout).toContain('ok self-check-untyped-throw: expected untyped-throw');
59+
expect(stdout).toContain('ok self-check-empty-hint: expected empty-hint');
60+
expect(stdout).toContain('ok self-check-hang: expected hang, got hang');
61+
expect(status).toBe(0);
62+
});
63+
});
64+
65+
describe('artifact promotion', () => {
66+
it('promotes a saved failing case into the corpus, once', () => {
67+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'fuzz-promote-'));
68+
const corpus = path.join(dir, 'regressions.json');
69+
fs.writeFileSync(corpus, '[]\n');
70+
const artifact = path.join(dir, 'case.json');
71+
fs.writeFileSync(
72+
artifact,
73+
JSON.stringify({
74+
target: 'self-check-untyped-throw',
75+
input: 'promote me',
76+
kind: 'untyped-throw',
77+
detail: 'seeded',
78+
}),
79+
);
80+
const env = { AGENT_DEVICE_FUZZ_CORPUS: corpus };
81+
82+
const first = runHarness(['--input-file', artifact, '--append-corpus'], env);
83+
expect(first.stdout).toContain('Case still violates the invariant');
84+
expect(first.stdout).toContain('Appended 1 case(s)');
85+
expect(first.status).toBe(1);
86+
expect(JSON.parse(fs.readFileSync(corpus, 'utf8'))).toEqual([
87+
{
88+
target: 'self-check-untyped-throw',
89+
input: 'promote me',
90+
note: expect.stringContaining('untyped-throw'),
91+
},
92+
]);
93+
94+
const second = runHarness(['--input-file', artifact, '--append-corpus'], env);
95+
expect(second.stdout).toContain('already contains every failing case');
96+
expect(JSON.parse(fs.readFileSync(corpus, 'utf8'))).toHaveLength(1);
97+
fs.rmSync(dir, { recursive: true, force: true });
98+
});
99+
});

0 commit comments

Comments
 (0)