Skip to content

Commit f020ef7

Browse files
committed
test: cover runner failure paths and wait for signal-handler readiness
1 parent 1ccf512 commit f020ef7

2 files changed

Lines changed: 137 additions & 9 deletions

File tree

‎scripts/agent-evals/runner.test.ts‎

Lines changed: 96 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,8 @@ function harness(
7474
timeout?: () => void
7575
onPause?: (count: number) => void
7676
disconnectOnPause?: number
77+
onLaunch?: () => void
78+
onReady?: () => void
7779
} = {},
7880
) {
7981
let now = 0
@@ -87,6 +89,8 @@ function harness(
8789
let disconnected = false
8890
let unsubscribed = false
8991
let launched = false
92+
let connected = false
93+
let readyAnnounced = false
9094
let workspaceFiles: readonly { path: string; content: string }[] = []
9195
const commands: TaskCommand[] = []
9296
const signals: NodeJS.Signals[] = []
@@ -107,6 +111,7 @@ function harness(
107111
launched = true
108112
workspaceFiles = files
109113
if (scenario === "setup") throw new Error("private setup detail")
114+
options.onLaunch?.()
110115
return {
111116
get exited() {
112117
return exited
@@ -121,8 +126,15 @@ function harness(
121126
}
122127
},
123128
connect() {
129+
connected = true
124130
return {
125-
isReady: scenario !== "activation",
131+
get isReady() {
132+
if (!readyAnnounced && scenario !== "activation") {
133+
readyAnnounced = true
134+
options.onReady?.()
135+
}
136+
return scenario !== "activation"
137+
},
126138
onTaskEvent(listener) {
127139
onEvent = listener
128140
},
@@ -253,6 +265,9 @@ function harness(
253265
get launched() {
254266
return launched
255267
},
268+
get connected() {
269+
return connected
270+
},
256271
get workspaceFiles() {
257272
return workspaceFiles
258273
},
@@ -343,6 +358,86 @@ describe("agent fixture controller", () => {
343358
assert.equal(accepted.length, 3)
344359
})
345360

361+
it("reports a rejected task start without replay, and still tears down cleanly", async () => {
362+
registerTestFixture("controller-start-failure")
363+
const f = harness("start-failure")
364+
f.emit({ eventName: RooCodeEventName.TaskStarted, payload: ["root"] })
365+
const result = await runAgentFixture(f.runtime, {
366+
fixtureName: "controller-start-failure",
367+
credential: CREDENTIAL,
368+
})
369+
assert.equal(result.execution.code, "TASK_START_FAILED")
370+
assert.equal(result.execution.stopReason, "startNotAccepted")
371+
assert.equal(result.execution.activated, true)
372+
assert.equal(result.execution.taskAccepted, false)
373+
const trace = writtenTrace(f)
374+
assert.equal(trace.eventFlow.bufferedBeforeAck, 1)
375+
assert.equal(trace.eventFlow.replayedAfterAck, 0)
376+
assert.ok(trace.accounting.completeness.reasons.includes("taskNotAccepted"))
377+
assert.equal(result.teardown.status, "ok")
378+
assert.equal(result.teardown.hostExited, true)
379+
assert.equal(result.teardown.cleanup, "done")
380+
assert.ok(f.cleaned && f.disconnected && f.unsubscribed)
381+
})
382+
383+
it("never connects or starts a task when the interrupt lands during launch", async () => {
384+
registerTestFixture("controller-launch-interrupt")
385+
const f = harness("success", { onLaunch: () => f.interrupt() })
386+
const result = await runAgentFixture(f.runtime, {
387+
fixtureName: "controller-launch-interrupt",
388+
credential: CREDENTIAL,
389+
})
390+
assert.equal(f.launched, true)
391+
assert.equal(f.connected, false)
392+
assert.equal(f.commands.length, 0)
393+
assert.equal(result.execution.code, "SETUP_FAILED")
394+
assert.equal(result.execution.activated, false)
395+
assert.equal(result.execution.taskAccepted, false)
396+
assert.deepEqual(f.signals, ["SIGTERM"])
397+
assert.equal(result.teardown.status, "ok")
398+
assert.equal(result.teardown.hostExited, true)
399+
assert.equal(result.teardown.cleanup, "done")
400+
assert.ok(f.cleaned && f.unsubscribed)
401+
})
402+
403+
it("stops before the start command when the interrupt lands after activation", async () => {
404+
registerTestFixture("controller-activation-interrupt")
405+
const f = harness("success", { onReady: () => f.interrupt() })
406+
const result = await runAgentFixture(f.runtime, {
407+
fixtureName: "controller-activation-interrupt",
408+
credential: CREDENTIAL,
409+
})
410+
assert.equal(result.execution.activated, true)
411+
assert.equal(result.execution.taskAccepted, false)
412+
assert.equal(result.execution.code, "SETUP_FAILED")
413+
assert.ok(!f.commands.some((command) => command.commandName === TaskCommandName.StartNewTask))
414+
assert.deepEqual(f.signals, ["SIGTERM"])
415+
assert.equal(result.teardown.status, "ok")
416+
assert.equal(result.teardown.hostExited, true)
417+
assert.ok(f.cleaned && f.disconnected && f.unsubscribed)
418+
})
419+
420+
it("drops exactly one event past the pre-ack buffer limit and records the overflow caveat", async () => {
421+
registerTestFixture("controller-overflow")
422+
const f = harness("no-events")
423+
for (let index = 0; index < 257; index++) f.emit({ eventName: RooCodeEventName.TaskStarted, payload: ["root"] })
424+
const result = await runAgentFixture(f.runtime, {
425+
fixtureName: "controller-overflow",
426+
credential: CREDENTIAL,
427+
observers: { shouldStop: () => true },
428+
})
429+
assert.equal(result.execution.code, "STOPPED")
430+
assert.equal(result.execution.taskAccepted, true)
431+
const trace = writtenTrace(f)
432+
assert.equal(trace.eventFlow.observed, 258)
433+
assert.equal(trace.eventFlow.eventBufferLimit, 256)
434+
assert.equal(trace.eventFlow.bufferedBeforeAck, 256)
435+
assert.equal(trace.eventFlow.replayedAfterAck, 256)
436+
assert.equal(trace.eventFlow.bufferOverflow, true)
437+
assert.equal(trace.eventFlow.droppedByOverflow, 1)
438+
assert.ok(trace.accounting.completeness.reasons.includes("eventBufferOverflow"))
439+
})
440+
346441
it("bounds the run timeout, the activation wait, and the usage drain", async () => {
347442
registerTestFixture("controller-fast", {
348443
limits: {

‎scripts/solheim-smoke-workflow.test.mjs‎

Lines changed: 41 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,11 @@ function runRecorder(
147147
return { result, ...paths, cleanup: () => fs.rmSync(dir, { recursive: true, force: true }) }
148148
}
149149

150-
function runRecorderSignalled(args, { ffmpeg, ffprobeOk = true, delayMs, providerMarker, stopBudget } = {}) {
150+
// TERM waits until the stub proves its signal handler is armed: a bounded wait
151+
// for readyFile when one is given, or the fixed delayMs fallback otherwise. A
152+
// TERM that lands during Node boot would kill the stub before its handler
153+
// exists and turn the test into a boot-race coin flip.
154+
function runRecorderSignalled(args, { ffmpeg, ffprobeOk = true, delayMs, readyFile, providerMarker, stopBudget } = {}) {
151155
return new Promise((resolve) => {
152156
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "record-sh-"))
153157
const bin = stubRecorderBin(dir, { ffmpeg, ffprobeOk })
@@ -163,10 +167,34 @@ function runRecorderSignalled(args, { ffmpeg, ffprobeOk = true, delayMs, provide
163167
stdout += chunk
164168
})
165169
const watchdog = setTimeout(() => child.kill("SIGKILL"), 15_000)
166-
const timer = setTimeout(() => child.kill("SIGTERM"), delayMs)
170+
let timer
171+
let poll
172+
const sendTerm = () => {
173+
timer = setTimeout(() => child.kill("SIGTERM"), delayMs)
174+
}
175+
if (readyFile === undefined) {
176+
sendTerm()
177+
} else {
178+
const startedAt = Date.now()
179+
poll = setInterval(() => {
180+
if (fs.existsSync(readyFile)) {
181+
clearInterval(poll)
182+
poll = undefined
183+
sendTerm()
184+
return
185+
}
186+
// A stub that never becomes ready must not hang the test.
187+
if (Date.now() - startedAt > 10_000) {
188+
clearInterval(poll)
189+
poll = undefined
190+
child.kill("SIGKILL")
191+
}
192+
}, 25)
193+
}
167194
child.on("close", (code) => {
168195
clearTimeout(watchdog)
169-
clearTimeout(timer)
196+
if (timer !== undefined) clearTimeout(timer)
197+
if (poll !== undefined) clearInterval(poll)
170198
resolve({ code, stdout, ...paths, cleanup: () => fs.rmSync(dir, { recursive: true, force: true }) })
171199
})
172200
})
@@ -360,6 +388,7 @@ describe("record.sh provider stop behavior", () => {
360388
// The artifact lives beside the stub, not inside the recorder output
361389
// directory, so the assertion only observes the driver-style write.
362390
const artifact = path.join(dir, "verdict.json")
391+
const readyFile = path.join(dir, "driver-ready")
363392
const stub = path.join(dir, "driver-stub.mjs")
364393
fs.writeFileSync(
365394
stub,
@@ -374,10 +403,11 @@ describe("record.sh provider stop behavior", () => {
374403
` process.exit(143)`,
375404
` }, 1500)`,
376405
`})`,
406+
`fs.writeFileSync(process.argv[3], "ready")`,
377407
`setInterval(() => {}, 1000)`,
378408
].join("\n"),
379409
)
380-
const run = await runRecorderSignalled(["node", stub, artifact], { delayMs: 800, stopBudget: 10 })
410+
const run = await runRecorderSignalled(["node", stub, artifact, readyFile], { readyFile, stopBudget: 10 })
381411
try {
382412
assert.equal(run.code, 143)
383413
// The wrapper waited for the driver teardown instead of killing the
@@ -398,21 +428,24 @@ describe("record.sh provider stop behavior", () => {
398428
// never interpolated into the driver source.
399429
const hungPidFile = path.join(dir, "hung\\pid")
400430
const termSeenFile = path.join(dir, 'term"seen')
431+
const hungReadyFile = path.join(dir, "hung-ready")
401432
const hungStub = path.join(dir, "hung-driver.mjs")
402433
fs.writeFileSync(
403434
hungStub,
404435
[
405436
`import fs from "node:fs"`,
406437
`process.on("SIGTERM", () => fs.writeFileSync(process.argv[2], "seen"))`,
438+
`fs.writeFileSync(process.argv[4], "ready")`,
407439
`fs.writeFileSync(process.argv[3], String(process.pid))`,
408440
`setInterval(() => {}, 1000)`,
409441
].join("\n"),
410442
)
411443
const startedAt = Date.now()
412-
// The TERM lands well after the child booted, so the ignored TERM is a
413-
// handled signal, not a boot-time default kill.
414-
const run = await runRecorderSignalled(["node", hungStub, termSeenFile, hungPidFile], {
415-
delayMs: 900,
444+
// TERM lands only after the ready file proves the TERM handler is
445+
// armed, so the ignored TERM is a handled signal, never a boot-time
446+
// default kill.
447+
const run = await runRecorderSignalled(["node", hungStub, termSeenFile, hungPidFile, hungReadyFile], {
448+
readyFile: hungReadyFile,
416449
stopBudget: 2,
417450
})
418451
try {

0 commit comments

Comments
 (0)