Skip to content

Commit e6762b3

Browse files
committed
fix: preserve release capture failure and cleanup evidence
1 parent 6808855 commit e6762b3

1 file changed

Lines changed: 109 additions & 27 deletions

File tree

‎script/token-budget/release.ts‎

Lines changed: 109 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,17 @@ let proc: ReturnType<typeof Bun.spawn> | undefined
7575
const campaignId = `release248-${process.pid}-${Date.now()}`
7676
const failures: unknown[] = []
7777
const cleanupChecks: unknown[] = []
78+
const errors: unknown[] = []
79+
const cleanups: { name: typeof ownedContainer; child: typeof proc; promise: Promise<void> }[] = []
80+
let interruption: { signal: "SIGINT" | "SIGTERM"; error: Error } | undefined
81+
function recordFailure(error: unknown, field: "campaignError" | "cleanupError" = "campaignError") {
82+
if (errors.includes(error)) return
83+
errors.push(error)
84+
failures.push({ [field]: error instanceof Error ? error.message : String(error) })
85+
}
86+
function checkInterrupted() {
87+
if (interruption) throw interruption.error
88+
}
7889
const arms: string[] = ["current-default", "previous-default"]
7990
const support: Record<string, boolean> = {}
8091
const raw: Record<string, Record<string, unknown[]>> = { short: Object.fromEntries(arms.map((x) => [x, []])) }
@@ -128,7 +139,9 @@ function checkpoint() {
128139
)
129140
}
130141
async function docker(args: string[]) {
142+
checkInterrupted()
131143
const child = Bun.spawn(["docker", ...args], { env: hostEnv, stdin: "ignore", stdout: "pipe", stderr: "pipe" })
144+
proc = child
132145
let timeout = false
133146
const timer = setTimeout(() => {
134147
timeout = true
@@ -141,12 +154,16 @@ async function docker(args: string[]) {
141154
child.exited,
142155
])
143156
if (code || timeout) {
157+
const error = new Error(`Docker preflight failed (${code}, timeout=${timeout}): ${err}`)
158+
recordFailure(error)
144159
await cleanup()
145-
throw new Error(`Docker preflight failed (${code}, timeout=${timeout}): ${err}`)
160+
throw error
146161
}
162+
checkInterrupted()
147163
return out
148164
} finally {
149165
clearTimeout(timer)
166+
if (proc === child) proc = undefined
150167
}
151168
}
152169
async function cleanupCommand(args: string[]) {
@@ -167,30 +184,70 @@ async function cleanupCommand(args: string[]) {
167184
clearTimeout(timer)
168185
}
169186
}
170-
async function cleanup() {
171-
try {
172-
if (ownedContainer) {
173-
const name = ownedContainer
174-
const removed = await cleanupCommand(["rm", "-f", name])
175-
const remaining = await cleanupCommand(["ps", "-aq", "--filter", `name=^/${name}$`])
176-
cleanupChecks.push({ name, removed, remaining })
187+
function cleanup() {
188+
const name = ownedContainer
189+
const child = proc
190+
const running = cleanups.find((item) => item.name === name && item.child === child)
191+
if (running) return running.promise
192+
const promise = (async () => {
193+
const failed: unknown[] = []
194+
try {
195+
if (child && child.exitCode === null) child.kill("SIGKILL")
196+
} catch (error) {
197+
failed.push(error)
198+
}
199+
if (name) {
200+
let removed: Awaited<ReturnType<typeof cleanupCommand>> | undefined
201+
let remaining: Awaited<ReturnType<typeof cleanupCommand>> | undefined
202+
try {
203+
removed = await cleanupCommand(["rm", "-f", name])
204+
} catch (error) {
205+
failed.push(error)
206+
}
207+
try {
208+
remaining = await cleanupCommand(["ps", "-aq", "--filter", `name=^/${name}$`])
209+
} catch (error) {
210+
failed.push(error)
211+
}
212+
cleanupChecks.push({ name, removed, remaining, errors: failed.map(String) })
177213
if (
214+
!removed ||
215+
!remaining ||
178216
removed.timedOut ||
179217
remaining.timedOut ||
180218
remaining.exit !== 0 ||
181219
remaining.stdout.trim() ||
182220
(removed.exit !== 0 && !removed.stderr.includes("No such container"))
183221
)
184-
throw new Error(`Unverified container cleanup: ${name}`)
222+
failed.push(new Error(`Unverified container cleanup: ${name}`))
185223
}
186-
} finally {
187-
if (proc && proc.exitCode === null) proc.kill("SIGKILL")
188-
}
189-
}
190-
for (const signal of ["SIGINT", "SIGTERM"] as const)
191-
process.on(signal, () => {
192-
void cleanup().finally(() => process.exit(128 + (signal === "SIGINT" ? 2 : 15)))
224+
try {
225+
if (child) await child.exited
226+
} catch (error) {
227+
failed.push(error)
228+
}
229+
if (failed.length) {
230+
const error = new AggregateError(failed, failed.map(String).join("; "), { cause: failed[0] })
231+
recordFailure(error, "cleanupError")
232+
throw error
233+
}
234+
})().finally(() => {
235+
const index = cleanups.findIndex((item) => item.promise === promise)
236+
cleanups.splice(index, 1)
193237
})
238+
cleanups.push({ name, child, promise })
239+
return promise
240+
}
241+
const handlers = (["SIGINT", "SIGTERM"] as const).map((signal) => {
242+
const handler = () => {
243+
if (interruption) return
244+
interruption = { signal, error: new Error(`Campaign interrupted by ${signal}`) }
245+
recordFailure(interruption.error)
246+
void cleanup().catch((error) => recordFailure(error, "cleanupError"))
247+
}
248+
process.on(signal, handler)
249+
return { signal, handler }
250+
})
194251
try {
195252
const image = JSON.parse(await docker(["image", "inspect", input.fixtureImage]))[0]
196253
if (
@@ -201,6 +258,7 @@ try {
201258
)
202259
throw new Error("Fixture image identity/platform mismatch")
203260
for (const name of ["current", "previous"] as const) {
261+
checkInterrupted()
204262
const binary = input[name]
205263
const script = `const fs=require('fs'),cp=require('child_process'),crypto=require('crypto'); const binary='/release/aictrl'; console.log(JSON.stringify({uid:process.getuid(),home:fs.readdirSync(process.env.HOME),binarySha256:crypto.createHash('sha256').update(fs.readFileSync(binary)).digest('hex'),version:cp.execFileSync(binary,['--version'],{encoding:'utf8'}).trim(),help:cp.execFileSync(binary,['run','--help'],{encoding:'utf8'}),fixtures:Object.fromEntries(['notes.txt','big.txt',...Array.from({length:3},(_,i)=>'notes-'+i+'.txt'),...Array.from({length:50},(_,i)=>'big-'+i+'.txt')].map(name=>{const b=fs.readFileSync('/workspace/'+name);return[name,{bytes:b.length,sha256:crypto.createHash('sha256').update(b).digest('hex')}]}))}));`
206264
ownedContainer = `${campaignId}-preflight-${name}`
@@ -227,6 +285,7 @@ try {
227285
]),
228286
)
229287
await cleanup()
288+
checkInterrupted()
230289
ownedContainer = undefined
231290
fs.writeFileSync(path.join(output, `${name}-preflight.json`), JSON.stringify(preflight, null, 2) + "\n")
232291
if (
@@ -256,10 +315,12 @@ try {
256315
raw.short[`${name}-selected`] = []
257316
}
258317
}
318+
checkInterrupted()
259319
stub = new Stub(50)
260320
stub.keepBodies = true
261321
for (let rep = 0; rep < input.repeats; rep++) {
262322
for (const arm of rep % 2 ? [...arms].reverse() : arms) {
323+
checkInterrupted()
263324
stub.reset("short")
264325
const runId = `${String(rep + 1).padStart(2, "0")}-${arm}-short`
265326
ownedContainer = `${campaignId}-${runId}`
@@ -318,12 +379,14 @@ try {
318379
)
319380
let timeout = false
320381
let cleanupFailure: string | undefined
382+
checkInterrupted()
321383
const child = Bun.spawn(argv, { env: hostEnv, stdin: "ignore", stdout: "pipe", stderr: "pipe" })
322384
proc = child
323385
const timer = setTimeout(() => {
324386
timeout = true
387+
recordFailure(new Error(`Capture ${runId} timed out`))
325388
void cleanup().catch((error) => {
326-
cleanupFailure = String(error)
389+
cleanupFailure ??= String(error)
327390
})
328391
}, TIMEOUT)
329392
let stdout = "",
@@ -340,7 +403,7 @@ try {
340403
try {
341404
await cleanup()
342405
} catch (error) {
343-
cleanupFailure = String(error)
406+
cleanupFailure ??= String(error)
344407
}
345408
if (!cleanupFailure) ownedContainer = undefined
346409
proc = undefined
@@ -432,25 +495,44 @@ try {
432495
}
433496
}
434497
} catch (error) {
435-
failures.push({ campaignError: error instanceof Error ? error.message : String(error) })
436-
checkpoint()
437-
throw error
498+
recordFailure(error)
438499
} finally {
439500
try {
440501
await cleanup()
502+
} catch (error) {
503+
recordFailure(error, "cleanupError")
504+
}
505+
try {
441506
const remaining = await cleanupCommand(["ps", "-aq", "--filter", `label=aictrl.harness-benchmark=${campaignId}`])
442507
cleanupChecks.push({ finalCampaignCheck: remaining })
443508
if (remaining.timedOut || remaining.exit !== 0 || remaining.stdout.trim())
444509
throw new Error("Campaign-owned containers remain or cleanup state could not be inspected")
445-
checkpoint()
446510
} catch (error) {
447-
failures.push({ cleanupError: String(error) })
448-
checkpoint()
449-
throw error
450-
} finally {
451-
stub?.server.stop(true)
511+
recordFailure(error, "cleanupError")
512+
}
513+
try {
514+
await stub?.server.stop(true)
515+
} catch (error) {
516+
recordFailure(error, "cleanupError")
517+
}
518+
await Promise.allSettled(cleanups.map((item) => item.promise))
519+
try {
452520
fs.rmSync(hostHome, { recursive: true, force: true })
521+
} catch (error) {
522+
recordFailure(error, "cleanupError")
523+
}
524+
try {
525+
checkpoint()
526+
} catch (error) {
527+
recordFailure(error)
453528
}
529+
for (const { signal, handler } of handlers) process.off(signal, handler)
530+
}
531+
if (errors.length) {
532+
const error = new AggregateError(errors, errors.map(String).join("; "), { cause: errors[0] })
533+
if (!interruption) throw error
534+
console.error(error)
535+
process.exit(128 + (interruption.signal === "SIGINT" ? 2 : 15))
454536
}
455537
console.log(
456538
`Captured ${input.repeats} sessions per supported arm; containers verified absent in ${output}; run release-count.py next`,

0 commit comments

Comments
 (0)