1- // Case execution with hang detection for the parser fuzz lane (#1414).
1+ // Case execution with hang and crash detection for the parser fuzz lane (#1414).
22//
3- // Cases run one at a time in a worker thread: a synchronous parser that never returns cannot be
4- // timed out from inside its own tick, so the budget is enforced from *another* thread, which can
5- // terminate the wedged one and attribute the stall to the exact input. Cases go over the wire
6- // individually (rather than as one batch) because fast-check drives the loop — it decides the next
7- // input, including the shrink candidates it derives from a failing one.
3+ // Cases run one at a time in a child process, and the budget is enforced from the parent. Two
4+ // faults a case can produce cannot be observed from inside the context running it: a synchronous
5+ // parser that never returns cannot be timed out from its own tick, and a parser that faults the
6+ // process — a native crash, an out-of-memory abort — leaves no stack to catch. Both are
7+ // attributed here, to the exact input, by watching a process the parent can outlive.
8+ //
9+ // It is a process rather than a worker thread because a thread cannot contain the second fault:
10+ // a fault in a worker thread takes its whole process down, and that process is the caller's —
11+ // for the unit-lane replay, the Vitest worker running the test file, which then dies with no
12+ // test, file, or case named (#2053). Cases go over the wire individually (rather than as one
13+ // batch) because fast-check drives the loop — it decides the next input, including the shrink
14+ // candidates it derives from a failing one.
815
16+ import { fork , type ChildProcess } from 'node:child_process' ;
917import path from 'node:path' ;
1018import { fileURLToPath } from 'node:url' ;
11- import { Worker } from 'node:worker_threads' ;
1219import type { FuzzFailure } from './invariant.ts' ;
1320import type { FuzzTarget } from './target-types.ts' ;
14- import type { FuzzWorkerData , FuzzWorkerMessage } from './worker.ts' ;
21+ import type { FuzzWorkerMessage } from './worker.ts' ;
1522
1623const WORKER_PATH = path . join ( path . dirname ( fileURLToPath ( import . meta. url ) ) , 'worker.ts' ) ;
1724
1825/** Bound on worker startup only; a per-case budget must never be charged for it. */
1926const STARTUP_BUDGET_MS = 60_000 ;
2027
28+ /** How much of a dead worker's output a crash failure quotes. Enough for a V8 fatal banner. */
29+ const DEATH_OUTPUT_LIMIT = 2_000 ;
30+
2131export type CaseRunner = {
2232 /** The failure this input violates the invariant with, or `null` when it holds. */
2333 run : ( input : string ) => Promise < FuzzFailure | null > ;
2434 close : ( ) => Promise < void > ;
2535} ;
2636
27- type Session = { worker : Worker ; ready : Promise < void > } ;
37+ type Session = {
38+ child : ChildProcess ;
39+ /** Settles when the worker has finished importing the parsers. */
40+ ready : Promise < void > ;
41+ /**
42+ * How the process died, or `null` while it is alive — the death certificate a crash failure
43+ * quotes. A reader rather than a field because it is written by the session's own listeners.
44+ */
45+ death : ( ) => string | null ;
46+ } ;
47+
48+ function describeDeath ( code : number | null , signal : NodeJS . Signals | null , output : string ) : string {
49+ const how = signal === null ? `exit code ${ code } ` : `signal ${ signal } ` ;
50+ const tail = output . trim ( ) ;
51+ return tail === '' ? how : `${ how } : ${ tail } ` ;
52+ }
2853
29- /** A worker plus the promise that settles when it has finished importing the parsers. */
54+ /** A worker process plus the promise that settles when it has finished importing the parsers. */
3055function startSession ( targetName : string ) : Session {
31- const workerData : FuzzWorkerData = { targetName } ;
32- // Type stripping is requested explicitly rather than inherited: under Vitest the parent's
33- // execArgv carries no such flag, and the worker is a plain `.ts` file Node must strip itself.
34- // The warning is silenced because a restarted worker re-emits it — one per hang buries the report.
35- const worker = new Worker ( WORKER_PATH , {
36- workerData,
56+ const child = fork ( WORKER_PATH , [ targetName ] , {
57+ // Type stripping is requested explicitly rather than inherited: under Vitest the parent's
58+ // execArgv carries no such flag, and the worker is a plain `.ts` file Node must strip itself.
59+ // The warning is silenced because a restarted worker re-emits it — one per hang buries the report.
3760 execArgv : [ '--experimental-strip-types' , '--disable-warning=ExperimentalWarning' ] ,
61+ // stderr is piped rather than inherited so a fatal fault — which writes to the descriptor and
62+ // so escapes the worker's own silencing — is quoted in the failure instead of the lane's log.
63+ stdio : [ 'ignore' , 'inherit' , 'pipe' , 'ipc' ] ,
64+ } ) ;
65+ let death : string | null = null ;
66+ let output = '' ;
67+ child . stderr ?. setEncoding ( 'utf8' ) ;
68+ child . stderr ?. on ( 'data' , ( chunk : string ) => {
69+ output = ( output + chunk ) . slice ( - DEATH_OUTPUT_LIMIT ) ;
70+ } ) ;
71+ // `close` rather than `exit`: it fires once the piped stderr has drained too, so the death is
72+ // recorded with the output that explains it.
73+ child . once ( 'close' , ( code : number | null , signal : NodeJS . Signals | null ) => {
74+ death = describeDeath ( code , signal , output ) ;
3875 } ) ;
3976 const ready = new Promise < void > ( ( resolve , reject ) => {
4077 const timer = setTimeout (
4178 ( ) => reject ( new Error ( `fuzz worker did not start within ${ STARTUP_BUDGET_MS } ms` ) ) ,
4279 STARTUP_BUDGET_MS ,
4380 ) ;
44- worker . once ( 'message' , ( message : FuzzWorkerMessage ) => {
81+ child . once ( 'message' , ( message : FuzzWorkerMessage ) => {
4582 clearTimeout ( timer ) ;
4683 if ( message . kind === 'ready' ) resolve ( ) ;
4784 else reject ( new Error ( `unexpected first worker message: ${ message . kind } ` ) ) ;
4885 } ) ;
49- worker . once ( 'error' , ( error ) => {
86+ child . once ( 'error' , ( error ) => {
5087 clearTimeout ( timer ) ;
5188 reject ( error ) ;
5289 } ) ;
90+ child . once ( 'close' , ( ) => {
91+ clearTimeout ( timer ) ;
92+ reject ( new Error ( `fuzz worker died before it was ready (${ death } )` ) ) ;
93+ } ) ;
5394 } ) ;
54- return { worker, ready } ;
95+ return { child, ready, death : ( ) => death } ;
96+ }
97+
98+ /** Ends a worker process, and resolves once it is gone. Already-dead sessions are a no-op. */
99+ async function endSession ( session : Session ) : Promise < void > {
100+ if ( session . death ( ) !== null ) return ;
101+ const gone = new Promise < void > ( ( resolve ) => session . child . once ( 'close' , ( ) => resolve ( ) ) ) ;
102+ // SIGKILL, not SIGTERM: the session is ended when a case wedged the worker, and a process
103+ // spinning inside a synchronous parser is exactly the one a catchable signal cannot reach.
104+ session . child . kill ( 'SIGKILL' ) ;
105+ await gone ;
55106}
56107
57108/**
58109 * Runs one case on a started worker. Startup is already awaited by the caller, so the budget below
59110 * covers parser time only — the reason a slow import can never be misreported as a parser hang.
111+ *
112+ * `restart` reports that the worker is no longer usable: it is wedged in the case that just hung,
113+ * or it is dead.
60114 */
61115function runOnSession (
62116 session : Session ,
63117 target : FuzzTarget ,
64118 input : string ,
65119 caseTimeoutMs : number ,
66- ) : Promise < { failure : FuzzFailure | null ; hung : boolean } > {
120+ ) : Promise < { failure : FuzzFailure | null ; restart : boolean } > {
121+ const crashed = ( ) : { failure : FuzzFailure ; restart : boolean } => ( {
122+ failure : {
123+ target : target . name ,
124+ input,
125+ kind : 'crash' ,
126+ detail : `case killed the worker process (${ session . death ( ) } )` ,
127+ } ,
128+ restart : true ,
129+ } ) ;
130+ // A worker that died between cases is reported against the case that meets it, rather than
131+ // through the ERR_IPC_CHANNEL_CLOSED that sending to it would raise: the harness classifies
132+ // every death of a worker it was using, and an opaque transport error is not a classification.
133+ if ( session . death ( ) !== null ) return Promise . resolve ( crashed ( ) ) ;
67134 return new Promise ( ( resolve , reject ) => {
68- const settle = ( failure : FuzzFailure | null , hung : boolean ) => {
135+ const detach = ( ) => {
69136 clearTimeout ( timer ) ;
70- session . worker . off ( 'message' , onMessage ) ;
71- session . worker . off ( 'error' , onError ) ;
72- resolve ( { failure, hung } ) ;
137+ session . child . off ( 'message' , onMessage ) ;
138+ session . child . off ( 'error' , onError ) ;
139+ session . child . off ( 'close' , onClose ) ;
140+ } ;
141+ const settle = ( failure : FuzzFailure | null , restart : boolean ) => {
142+ detach ( ) ;
143+ resolve ( { failure, restart } ) ;
73144 } ;
74145 const onMessage = ( message : FuzzWorkerMessage ) => {
75146 if ( message . kind === 'result' ) settle ( message . failure , false ) ;
76147 } ;
77148 const onError = ( error : Error ) => {
78- clearTimeout ( timer ) ;
79- session . worker . off ( 'message' , onMessage ) ;
149+ detach ( ) ;
80150 reject ( error ) ;
81151 } ;
152+ const onClose = ( ) => {
153+ const { failure, restart } = crashed ( ) ;
154+ settle ( failure , restart ) ;
155+ } ;
82156 const timer = setTimeout ( ( ) => {
83157 settle (
84158 {
@@ -90,16 +164,17 @@ function runOnSession(
90164 true ,
91165 ) ;
92166 } , caseTimeoutMs ) ;
93- session . worker . on ( 'message' , onMessage ) ;
94- session . worker . once ( 'error' , onError ) ;
95- session . worker . postMessage ( { kind : 'case' , input } ) ;
167+ session . child . on ( 'message' , onMessage ) ;
168+ session . child . once ( 'error' , onError ) ;
169+ session . child . once ( 'close' , onClose ) ;
170+ session . child . send ( { kind : 'case' , input } ) ;
96171 } ) ;
97172}
98173
99174/**
100- * A worker-backed runner for one target. A hang leaves the thread wedged in its own loop, so the
101- * runner terminates it and starts a fresh one for the next case — otherwise a single hang would
102- * silently turn every later case into a hang too.
175+ * A worker-backed runner for one target. A hang leaves the worker wedged in its own loop and a
176+ * crash leaves no worker at all, so the runner ends the session and starts a fresh one for the
177+ * next case — otherwise a single such case would silently turn every later case into one too.
103178 */
104179export async function createCaseRunner (
105180 target : FuzzTarget ,
@@ -113,17 +188,17 @@ export async function createCaseRunner(
113188 run : async ( input ) => {
114189 if ( closed ) throw new Error ( 'fuzz case runner is closed' ) ;
115190 await session . ready ;
116- const { failure, hung } = await runOnSession ( session , target , input , caseTimeoutMs ) ;
117- if ( hung ) {
118- await session . worker . terminate ( ) ;
191+ const { failure, restart } = await runOnSession ( session , target , input , caseTimeoutMs ) ;
192+ if ( restart ) {
193+ await endSession ( session ) ;
119194 session = startSession ( target . name ) ;
120195 await session . ready ;
121196 }
122197 return failure ;
123198 } ,
124199 close : async ( ) => {
125200 closed = true ;
126- await session . worker . terminate ( ) ;
201+ await endSession ( session ) ;
127202 } ,
128203 } ;
129204}
0 commit comments