|
| 1 | +// Tests of the fuzz harness itself (#1414). |
| 2 | +// |
| 3 | +// The corpus replay only proves clean inputs pass, so a regressed classifier or watchdog would |
| 4 | +// leave every test green. These run the harness against the broken-on-purpose targets through |
| 5 | +// the real CLI — same worker, watchdog, artifact, and promotion path a nightly failure takes — |
| 6 | +// and require each failure kind to be reported. |
| 7 | + |
| 8 | +import { execFileSync } from 'node:child_process'; |
| 9 | +import fs from 'node:fs'; |
| 10 | +import os from 'node:os'; |
| 11 | +import path from 'node:path'; |
| 12 | +import { fileURLToPath } from 'node:url'; |
| 13 | +import { describe, expect, it } from 'vitest'; |
| 14 | +import { checkCase } from './invariant.ts'; |
| 15 | +import { SELF_CHECK_TARGETS } from './self-check-targets.ts'; |
| 16 | + |
| 17 | +const FUZZ_DIR = path.dirname(fileURLToPath(import.meta.url)); |
| 18 | +const RUN = path.join(FUZZ_DIR, 'run.ts'); |
| 19 | + |
| 20 | +function runHarness( |
| 21 | + args: readonly string[], |
| 22 | + env: Record<string, string> = {}, |
| 23 | +): { status: number; stdout: string } { |
| 24 | + try { |
| 25 | + const stdout = execFileSync(process.execPath, ['--experimental-strip-types', RUN, ...args], { |
| 26 | + encoding: 'utf8', |
| 27 | + env: { ...process.env, ...env }, |
| 28 | + }); |
| 29 | + return { status: 0, stdout }; |
| 30 | + } catch (error) { |
| 31 | + const failure = error as { status?: number; stdout?: string }; |
| 32 | + return { status: failure.status ?? 1, stdout: failure.stdout ?? '' }; |
| 33 | + } |
| 34 | +} |
| 35 | + |
| 36 | +function targetNamed(name: string) { |
| 37 | + const target = SELF_CHECK_TARGETS.find((candidate) => candidate.name === name); |
| 38 | + if (!target) throw new Error(`missing self-check target ${name}`); |
| 39 | + return target; |
| 40 | +} |
| 41 | + |
| 42 | +describe('fuzz invariant classifier', () => { |
| 43 | + it('reports a bare Error as untyped-throw', () => { |
| 44 | + const failure = checkCase(targetNamed('self-check-untyped-throw'), 'case'); |
| 45 | + expect(failure?.kind).toBe('untyped-throw'); |
| 46 | + expect(failure?.detail).toContain('Error: self-check untyped throw'); |
| 47 | + }); |
| 48 | + |
| 49 | + it('reports an AppError with a blank hint as empty-hint', () => { |
| 50 | + const failure = checkCase(targetNamed('self-check-empty-hint'), 'case'); |
| 51 | + expect(failure?.kind).toBe('empty-hint'); |
| 52 | + }); |
| 53 | +}); |
| 54 | + |
| 55 | +describe('fuzz harness self-check', () => { |
| 56 | + it('catches an untyped throw, an empty hint, and a wedged worker', () => { |
| 57 | + const { status, stdout } = runHarness(['--self-check', '--case-timeout-ms', '750']); |
| 58 | + expect(stdout).toContain('ok self-check-untyped-throw: expected untyped-throw'); |
| 59 | + expect(stdout).toContain('ok self-check-empty-hint: expected empty-hint'); |
| 60 | + expect(stdout).toContain('ok self-check-hang: expected hang, got hang'); |
| 61 | + expect(status).toBe(0); |
| 62 | + }); |
| 63 | +}); |
| 64 | + |
| 65 | +describe('artifact promotion', () => { |
| 66 | + it('promotes a saved failing case into the corpus, once', () => { |
| 67 | + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'fuzz-promote-')); |
| 68 | + const corpus = path.join(dir, 'regressions.json'); |
| 69 | + fs.writeFileSync(corpus, '[]\n'); |
| 70 | + const artifact = path.join(dir, 'case.json'); |
| 71 | + fs.writeFileSync( |
| 72 | + artifact, |
| 73 | + JSON.stringify({ |
| 74 | + target: 'self-check-untyped-throw', |
| 75 | + input: 'promote me', |
| 76 | + kind: 'untyped-throw', |
| 77 | + detail: 'seeded', |
| 78 | + }), |
| 79 | + ); |
| 80 | + const env = { AGENT_DEVICE_FUZZ_CORPUS: corpus }; |
| 81 | + |
| 82 | + const first = runHarness(['--input-file', artifact, '--append-corpus'], env); |
| 83 | + expect(first.stdout).toContain('Case still violates the invariant'); |
| 84 | + expect(first.stdout).toContain('Appended 1 case(s)'); |
| 85 | + expect(first.status).toBe(1); |
| 86 | + expect(JSON.parse(fs.readFileSync(corpus, 'utf8'))).toEqual([ |
| 87 | + { |
| 88 | + target: 'self-check-untyped-throw', |
| 89 | + input: 'promote me', |
| 90 | + note: expect.stringContaining('untyped-throw'), |
| 91 | + }, |
| 92 | + ]); |
| 93 | + |
| 94 | + const second = runHarness(['--input-file', artifact, '--append-corpus'], env); |
| 95 | + expect(second.stdout).toContain('already contains every failing case'); |
| 96 | + expect(JSON.parse(fs.readFileSync(corpus, 'utf8'))).toHaveLength(1); |
| 97 | + fs.rmSync(dir, { recursive: true, force: true }); |
| 98 | + }); |
| 99 | +}); |
0 commit comments