@@ -75,6 +75,17 @@ let proc: ReturnType<typeof Bun.spawn> | undefined
7575const campaignId = `release248-${ process . pid } -${ Date . now ( ) } `
7676const failures : unknown [ ] = [ ]
7777const cleanupChecks : unknown [ ] = [ ]
78+ const errors : unknown [ ] = [ ]
79+ const cleanups : { name : typeof ownedContainer ; child : typeof proc ; promise : Promise < void > } [ ] = [ ]
80+ let interruption : { signal : "SIGINT" | "SIGTERM" ; error : Error } | undefined
81+ function recordFailure ( error : unknown , field : "campaignError" | "cleanupError" = "campaignError" ) {
82+ if ( errors . includes ( error ) ) return
83+ errors . push ( error )
84+ failures . push ( { [ field ] : error instanceof Error ? error . message : String ( error ) } )
85+ }
86+ function checkInterrupted ( ) {
87+ if ( interruption ) throw interruption . error
88+ }
7889const arms : string [ ] = [ "current-default" , "previous-default" ]
7990const support : Record < string , boolean > = { }
8091const raw : Record < string , Record < string , unknown [ ] > > = { short : Object . fromEntries ( arms . map ( ( x ) => [ x , [ ] ] ) ) }
@@ -128,7 +139,9 @@ function checkpoint() {
128139 )
129140}
130141async function docker ( args : string [ ] ) {
142+ checkInterrupted ( )
131143 const child = Bun . spawn ( [ "docker" , ...args ] , { env : hostEnv , stdin : "ignore" , stdout : "pipe" , stderr : "pipe" } )
144+ proc = child
132145 let timeout = false
133146 const timer = setTimeout ( ( ) => {
134147 timeout = true
@@ -141,12 +154,16 @@ async function docker(args: string[]) {
141154 child . exited ,
142155 ] )
143156 if ( code || timeout ) {
157+ const error = new Error ( `Docker preflight failed (${ code } , timeout=${ timeout } ): ${ err } ` )
158+ recordFailure ( error )
144159 await cleanup ( )
145- throw new Error ( `Docker preflight failed ( ${ code } , timeout= ${ timeout } ): ${ err } ` )
160+ throw error
146161 }
162+ checkInterrupted ( )
147163 return out
148164 } finally {
149165 clearTimeout ( timer )
166+ if ( proc === child ) proc = undefined
150167 }
151168}
152169async function cleanupCommand ( args : string [ ] ) {
@@ -167,30 +184,70 @@ async function cleanupCommand(args: string[]) {
167184 clearTimeout ( timer )
168185 }
169186}
170- async function cleanup ( ) {
171- try {
172- if ( ownedContainer ) {
173- const name = ownedContainer
174- const removed = await cleanupCommand ( [ "rm" , "-f" , name ] )
175- const remaining = await cleanupCommand ( [ "ps" , "-aq" , "--filter" , `name=^/${ name } $` ] )
176- cleanupChecks . push ( { name, removed, remaining } )
187+ function cleanup ( ) {
188+ const name = ownedContainer
189+ const child = proc
190+ const running = cleanups . find ( ( item ) => item . name === name && item . child === child )
191+ if ( running ) return running . promise
192+ const promise = ( async ( ) => {
193+ const failed : unknown [ ] = [ ]
194+ try {
195+ if ( child && child . exitCode === null ) child . kill ( "SIGKILL" )
196+ } catch ( error ) {
197+ failed . push ( error )
198+ }
199+ if ( name ) {
200+ let removed : Awaited < ReturnType < typeof cleanupCommand > > | undefined
201+ let remaining : Awaited < ReturnType < typeof cleanupCommand > > | undefined
202+ try {
203+ removed = await cleanupCommand ( [ "rm" , "-f" , name ] )
204+ } catch ( error ) {
205+ failed . push ( error )
206+ }
207+ try {
208+ remaining = await cleanupCommand ( [ "ps" , "-aq" , "--filter" , `name=^/${ name } $` ] )
209+ } catch ( error ) {
210+ failed . push ( error )
211+ }
212+ cleanupChecks . push ( { name, removed, remaining, errors : failed . map ( String ) } )
177213 if (
214+ ! removed ||
215+ ! remaining ||
178216 removed . timedOut ||
179217 remaining . timedOut ||
180218 remaining . exit !== 0 ||
181219 remaining . stdout . trim ( ) ||
182220 ( removed . exit !== 0 && ! removed . stderr . includes ( "No such container" ) )
183221 )
184- throw new Error ( `Unverified container cleanup: ${ name } ` )
222+ failed . push ( new Error ( `Unverified container cleanup: ${ name } ` ) )
185223 }
186- } finally {
187- if ( proc && proc . exitCode === null ) proc . kill ( "SIGKILL" )
188- }
189- }
190- for ( const signal of [ "SIGINT" , "SIGTERM" ] as const )
191- process . on ( signal , ( ) => {
192- void cleanup ( ) . finally ( ( ) => process . exit ( 128 + ( signal === "SIGINT" ? 2 : 15 ) ) )
224+ try {
225+ if ( child ) await child . exited
226+ } catch ( error ) {
227+ failed . push ( error )
228+ }
229+ if ( failed . length ) {
230+ const error = new AggregateError ( failed , failed . map ( String ) . join ( "; " ) , { cause : failed [ 0 ] } )
231+ recordFailure ( error , "cleanupError" )
232+ throw error
233+ }
234+ } ) ( ) . finally ( ( ) => {
235+ const index = cleanups . findIndex ( ( item ) => item . promise === promise )
236+ cleanups . splice ( index , 1 )
193237 } )
238+ cleanups . push ( { name, child, promise } )
239+ return promise
240+ }
241+ const handlers = ( [ "SIGINT" , "SIGTERM" ] as const ) . map ( ( signal ) => {
242+ const handler = ( ) => {
243+ if ( interruption ) return
244+ interruption = { signal, error : new Error ( `Campaign interrupted by ${ signal } ` ) }
245+ recordFailure ( interruption . error )
246+ void cleanup ( ) . catch ( ( error ) => recordFailure ( error , "cleanupError" ) )
247+ }
248+ process . on ( signal , handler )
249+ return { signal, handler }
250+ } )
194251try {
195252 const image = JSON . parse ( await docker ( [ "image" , "inspect" , input . fixtureImage ] ) ) [ 0 ]
196253 if (
@@ -201,6 +258,7 @@ try {
201258 )
202259 throw new Error ( "Fixture image identity/platform mismatch" )
203260 for ( const name of [ "current" , "previous" ] as const ) {
261+ checkInterrupted ( )
204262 const binary = input [ name ]
205263 const script = `const fs=require('fs'),cp=require('child_process'),crypto=require('crypto'); const binary='/release/aictrl'; console.log(JSON.stringify({uid:process.getuid(),home:fs.readdirSync(process.env.HOME),binarySha256:crypto.createHash('sha256').update(fs.readFileSync(binary)).digest('hex'),version:cp.execFileSync(binary,['--version'],{encoding:'utf8'}).trim(),help:cp.execFileSync(binary,['run','--help'],{encoding:'utf8'}),fixtures:Object.fromEntries(['notes.txt','big.txt',...Array.from({length:3},(_,i)=>'notes-'+i+'.txt'),...Array.from({length:50},(_,i)=>'big-'+i+'.txt')].map(name=>{const b=fs.readFileSync('/workspace/'+name);return[name,{bytes:b.length,sha256:crypto.createHash('sha256').update(b).digest('hex')}]}))}));`
206264 ownedContainer = `${ campaignId } -preflight-${ name } `
@@ -227,6 +285,7 @@ try {
227285 ] ) ,
228286 )
229287 await cleanup ( )
288+ checkInterrupted ( )
230289 ownedContainer = undefined
231290 fs . writeFileSync ( path . join ( output , `${ name } -preflight.json` ) , JSON . stringify ( preflight , null , 2 ) + "\n" )
232291 if (
@@ -256,10 +315,12 @@ try {
256315 raw . short [ `${ name } -selected` ] = [ ]
257316 }
258317 }
318+ checkInterrupted ( )
259319 stub = new Stub ( 50 )
260320 stub . keepBodies = true
261321 for ( let rep = 0 ; rep < input . repeats ; rep ++ ) {
262322 for ( const arm of rep % 2 ? [ ...arms ] . reverse ( ) : arms ) {
323+ checkInterrupted ( )
263324 stub . reset ( "short" )
264325 const runId = `${ String ( rep + 1 ) . padStart ( 2 , "0" ) } -${ arm } -short`
265326 ownedContainer = `${ campaignId } -${ runId } `
@@ -318,12 +379,14 @@ try {
318379 )
319380 let timeout = false
320381 let cleanupFailure : string | undefined
382+ checkInterrupted ( )
321383 const child = Bun . spawn ( argv , { env : hostEnv , stdin : "ignore" , stdout : "pipe" , stderr : "pipe" } )
322384 proc = child
323385 const timer = setTimeout ( ( ) => {
324386 timeout = true
387+ recordFailure ( new Error ( `Capture ${ runId } timed out` ) )
325388 void cleanup ( ) . catch ( ( error ) => {
326- cleanupFailure = String ( error )
389+ cleanupFailure ?? = String ( error )
327390 } )
328391 } , TIMEOUT )
329392 let stdout = "" ,
@@ -340,7 +403,7 @@ try {
340403 try {
341404 await cleanup ( )
342405 } catch ( error ) {
343- cleanupFailure = String ( error )
406+ cleanupFailure ?? = String ( error )
344407 }
345408 if ( ! cleanupFailure ) ownedContainer = undefined
346409 proc = undefined
@@ -432,25 +495,44 @@ try {
432495 }
433496 }
434497} catch ( error ) {
435- failures . push ( { campaignError : error instanceof Error ? error . message : String ( error ) } )
436- checkpoint ( )
437- throw error
498+ recordFailure ( error )
438499} finally {
439500 try {
440501 await cleanup ( )
502+ } catch ( error ) {
503+ recordFailure ( error , "cleanupError" )
504+ }
505+ try {
441506 const remaining = await cleanupCommand ( [ "ps" , "-aq" , "--filter" , `label=aictrl.harness-benchmark=${ campaignId } ` ] )
442507 cleanupChecks . push ( { finalCampaignCheck : remaining } )
443508 if ( remaining . timedOut || remaining . exit !== 0 || remaining . stdout . trim ( ) )
444509 throw new Error ( "Campaign-owned containers remain or cleanup state could not be inspected" )
445- checkpoint ( )
446510 } catch ( error ) {
447- failures . push ( { cleanupError : String ( error ) } )
448- checkpoint ( )
449- throw error
450- } finally {
451- stub ?. server . stop ( true )
511+ recordFailure ( error , "cleanupError" )
512+ }
513+ try {
514+ await stub ?. server . stop ( true )
515+ } catch ( error ) {
516+ recordFailure ( error , "cleanupError" )
517+ }
518+ await Promise . allSettled ( cleanups . map ( ( item ) => item . promise ) )
519+ try {
452520 fs . rmSync ( hostHome , { recursive : true , force : true } )
521+ } catch ( error ) {
522+ recordFailure ( error , "cleanupError" )
523+ }
524+ try {
525+ checkpoint ( )
526+ } catch ( error ) {
527+ recordFailure ( error )
453528 }
529+ for ( const { signal, handler } of handlers ) process . off ( signal , handler )
530+ }
531+ if ( errors . length ) {
532+ const error = new AggregateError ( errors , errors . map ( String ) . join ( "; " ) , { cause : errors [ 0 ] } )
533+ if ( ! interruption ) throw error
534+ console . error ( error )
535+ process . exit ( 128 + ( interruption . signal === "SIGINT" ? 2 : 15 ) )
454536}
455537console . log (
456538 `Captured ${ input . repeats } sessions per supported arm; containers verified absent in ${ output } ; run release-count.py next` ,
0 commit comments