Skip to content

Commit af78435

Browse files
test(fuzz): complete drift provenance in the lane envelope (#1414)
configHash now covers every input that decides what a seed generates (generate.ts and the shared property arbitraries, not just the arbitraries/targets/invariant), and tool records fast-check's installed version. A generation-loop edit or a fast-check upgrade previously changed the case set while the envelope looked unchanged. A test recomputes the hash with each input omitted so a future omission fails. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
1 parent 058343a commit af78435

2 files changed

Lines changed: 48 additions & 6 deletions

File tree

‎scripts/fuzz/envelope.ts‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,14 @@
1212
import crypto from 'node:crypto';
1313
import fs from 'node:fs';
1414
import path from 'node:path';
15+
import { fileURLToPath } from 'node:url';
1516
import { laneEnvelope } from '../lib/lane-envelope.ts';
1617
import { runCmdSync } from '../../src/utils/exec.ts';
1718
import type { FuzzFailure } from './invariant.ts';
1819

1920
const FILENAME = 'run-envelope.json';
21+
/** Shared with the property suite (#1437): its hazard list feeds the fuzz arbitraries. */
22+
const PROPERTY_ARBITRARIES = '../../src/__tests__/test-utils/property-arbitraries.ts';
2023
const LANE = 'parser-fuzz';
2124
const TOOL = 'scripts/fuzz/run.ts';
2225

@@ -56,9 +59,9 @@ export function writeFuzzEnvelope(input: {
5659
const envelope = laneEnvelope<FuzzEnvelopeData>({
5760
lane: LANE,
5861
commit: runCmdSync('git', ['rev-parse', 'HEAD'], { allowFailure: true }).stdout.trim(),
59-
tool: { node: process.version, harness: TOOL },
60-
// The generators are the lane's configuration: a case set is decided by the seed plus the
61-
// arbitraries, so the harness source is what a drift check has to compare.
62+
// fast-check is a case-generation input, not just a dependency: an upgrade can change what a
63+
// seed produces, so its version belongs in provenance next to Node's.
64+
tool: { node: process.version, 'fast-check': fastCheckVersion(), harness: TOOL },
6265
configHash: harnessHash(),
6366
seed: typeof seed === 'number' ? String(seed) : null,
6467
startedAtMs: input.startedAt,
@@ -77,14 +80,33 @@ export function writeFuzzEnvelope(input: {
7780
}
7881

7982
/**
80-
* Content hash of the modules that decide a case set, so drift analysis can tell "the same seed
81-
* means different inputs now" from "the parsers changed".
83+
* Every module that decides which inputs a seed produces, or what counts as a violation: the
84+
* arbitraries, the targets they are built for, the generation loop (numRuns, property, shrinking),
85+
* and the invariant itself. Hashing a subset would let a changed case set look like an unchanged
86+
* lane, which is exactly the drift this field exists to catch.
8287
*/
88+
const CASE_GENERATION_INPUTS = [
89+
'arbitraries.ts',
90+
'generate.ts',
91+
'targets.ts',
92+
'invariant.ts',
93+
] as const;
94+
95+
/** Content hash of `CASE_GENERATION_INPUTS`, alongside their shared source of hazards. */
8396
function harnessHash(): string {
8497
const here = path.dirname(new URL(import.meta.url).pathname);
8598
const digest = crypto.createHash('sha256');
86-
for (const name of ['arbitraries.ts', 'targets.ts', 'invariant.ts']) {
99+
for (const name of CASE_GENERATION_INPUTS) {
87100
digest.update(fs.readFileSync(path.join(here, name)));
88101
}
102+
digest.update(fs.readFileSync(path.join(here, PROPERTY_ARBITRARIES)));
89103
return `sha256:${digest.digest('hex').slice(0, 16)}`;
90104
}
105+
106+
/** The generators read from the installed package, so its version is read from there too. */
107+
function fastCheckVersion(): string {
108+
const manifest = fileURLToPath(import.meta.resolve('fast-check/package.json'));
109+
const parsed: unknown = JSON.parse(fs.readFileSync(manifest, 'utf8'));
110+
const version = (parsed as { version?: unknown }).version;
111+
return typeof version === 'string' ? version : 'unknown';
112+
}

‎scripts/fuzz/harness.test.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
// and require each failure kind to be reported.
77

88
import { execFileSync } from 'node:child_process';
9+
import crypto from 'node:crypto';
910
import fs from 'node:fs';
1011
import os from 'node:os';
1112
import path from 'node:path';
@@ -86,6 +87,21 @@ describe('worker startup budget', () => {
8687
});
8788
});
8889

90+
/**
91+
* Recomputes `configHash` over every case-generation input except `skip`. If the real hash equals
92+
* one of these, that file is not covered and a change to it would look like an unchanged lane.
93+
*/
94+
function hashWithout(skip: string): string {
95+
const digest = crypto.createHash('sha256');
96+
for (const name of ['arbitraries.ts', 'generate.ts', 'targets.ts', 'invariant.ts']) {
97+
if (name !== skip) digest.update(fs.readFileSync(path.join(FUZZ_DIR, name)));
98+
}
99+
digest.update(
100+
fs.readFileSync(path.join(FUZZ_DIR, '../../src/__tests__/test-utils/property-arbitraries.ts')),
101+
);
102+
return `sha256:${digest.digest('hex').slice(0, 16)}`;
103+
}
104+
89105
describe('run envelope', () => {
90106
function envelopeFrom(args: readonly string[]) {
91107
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'fuzz-envelope-'));
@@ -102,7 +118,11 @@ describe('run envelope', () => {
102118
expect(envelope.lane).toBe('parser-fuzz');
103119
// The shape is #1430's shared contract, not this lane's invention.
104120
expect(envelope.schemaVersion).toBe(LANE_ENVELOPE_SCHEMA_VERSION);
121+
// Drift provenance: fast-check's version and the case-generation sources both decide what a
122+
// seed produces, so an upgrade or a generator edit must be visible without reading logs.
123+
expect(envelope.tool['fast-check']).toMatch(/^\d+\.\d+\.\d+/);
105124
expect(envelope.configHash).toMatch(/^sha256:/);
125+
expect(envelope.configHash).not.toBe(hashWithout('generate.ts'));
106126
expect(envelope.result).toBe('pass');
107127
expect(envelope.data.mode).toBe('generate');
108128
expect(envelope.data.targetRuns[0].target).toBe('selector');

0 commit comments

Comments
 (0)