Skip to content

Commit 036dfce

Browse files
committed
fix(smoke): persist sidebar focus diagnostics and cover teardown holds
1 parent 760b849 commit 036dfce

7 files changed

Lines changed: 163 additions & 54 deletions

File tree

‎.github/workflows/solheim-provider-smoke.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ jobs:
7474
path: |
7575
${{ runner.temp }}/solheim-provider-smoke/verdict.json
7676
${{ runner.temp }}/solheim-provider-smoke/trace.json
77+
${{ runner.temp }}/solheim-provider-smoke/sidebar-focus.json
7778
if-no-files-found: error
7879
retention-days: 7
7980

‎scripts/agent-evals/launcher.ts‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ export interface HostRuntimeOptions {
2020
// Optional --extensionTestsPath module. The module's run() must never
2121
// settle, because VS Code exits when it does. Off by default.
2222
testsPath?: string
23+
// Optional bounded status artifact written by the extension test module.
24+
testsDiagnosticsPath?: string
2325
}
2426

2527
const readCommit = async (root: string): Promise<MeasuredString> => {
@@ -94,7 +96,12 @@ export function createHostRuntime(options: HostRuntimeOptions): AgentRuntime {
9496
process.env.DISPLAY ? args : ["-a", executable, ...args],
9597
{
9698
detached: true,
97-
env: buildChildEnvironment(process.env, isolatedHome, socket),
99+
env: {
100+
...buildChildEnvironment(process.env, isolatedHome, socket),
101+
...(options.testsDiagnosticsPath
102+
? { AGENT_EVALS_TESTS_DIAGNOSTICS_PATH: options.testsDiagnosticsPath }
103+
: {}),
104+
},
98105
stdio: "ignore",
99106
},
100107
)

‎scripts/agent-evals/runner.test.ts‎

Lines changed: 61 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -252,6 +252,7 @@ function harness(
252252
signals,
253253
artifacts,
254254
emit,
255+
close,
255256
interrupt: () => interrupt(),
256257
get cleaned() {
257258
return cleaned
@@ -1051,57 +1052,68 @@ describe("agent fixture controller", () => {
10511052
assert.equal(result.teardown.status, "ok")
10521053
})
10531054

1054-
it("holds teardown after the completion message for the requested window", async () => {
1055-
registerTestFixture("controller-hold")
1056-
const f = harness("no-events")
1057-
f.emit({
1058-
eventName: RooCodeEventName.Message,
1059-
payload: [
1060-
{
1061-
taskId: "root",
1062-
action: "created",
1063-
message: { ts: 1, type: "say", say: "completion_result", text: "synthetic final answer" },
1064-
},
1065-
],
1066-
})
1067-
f.emit({
1068-
eventName: RooCodeEventName.TaskCompleted,
1069-
payload: [
1070-
"root",
1071-
{ totalTokensIn: 7, totalTokensOut: 3, totalCost: 0, contextTokens: 9 },
1072-
{},
1073-
{
1074-
isSubtask: false,
1075-
},
1076-
],
1077-
})
1078-
const holdPauses: number[] = []
1079-
let closeTaskSeenDuringHold = false
1080-
const pause = f.runtime.pause
1081-
f.runtime.pause = async (ms) => {
1082-
if (ms === 3_000) {
1083-
holdPauses.push(ms)
1084-
closeTaskSeenDuringHold = f.commands.some(
1085-
(command) => command.commandName === TaskCommandName.CloseTask,
1086-
)
1055+
for (const hostExited of [false, true]) {
1056+
it(`holds teardown only while the completed host is live (exited=${hostExited})`, async () => {
1057+
const fixtureName = `controller-hold-${hostExited}`
1058+
registerTestFixture(fixtureName)
1059+
const f = harness("no-events")
1060+
f.emit({
1061+
eventName: RooCodeEventName.Message,
1062+
payload: [
1063+
{
1064+
taskId: "root",
1065+
action: "created",
1066+
message: { ts: 1, type: "say", say: "completion_result", text: "synthetic final answer" },
1067+
},
1068+
],
1069+
})
1070+
f.emit({
1071+
eventName: RooCodeEventName.TaskCompleted,
1072+
payload: [
1073+
"root",
1074+
{ totalTokensIn: 7, totalTokensOut: 3, totalCost: 0, contextTokens: 9 },
1075+
{},
1076+
{
1077+
isSubtask: false,
1078+
},
1079+
],
1080+
})
1081+
const holdPauses: number[] = []
1082+
let closeTaskSeenDuringHold = false
1083+
const pause = f.runtime.pause
1084+
f.runtime.pause = async (ms) => {
1085+
if (ms === 3_000) {
1086+
holdPauses.push(ms)
1087+
closeTaskSeenDuringHold = f.commands.some(
1088+
(command) => command.commandName === TaskCommandName.CloseTask,
1089+
)
1090+
}
1091+
await pause(ms)
10871092
}
1088-
await pause(ms)
1089-
}
1090-
const result = await runAgentFixture(f.runtime, {
1091-
fixtureName: "controller-hold",
1092-
credential: CREDENTIAL,
1093-
observers: { shouldStop: () => true },
1094-
holdBeforeTeardownMs: 3_000,
1093+
const result = await runAgentFixture(f.runtime, {
1094+
fixtureName,
1095+
credential: CREDENTIAL,
1096+
observers: {
1097+
onAcceptedEvent: (event) => {
1098+
if (hostExited && event.eventName === RooCodeEventName.TaskCompleted) f.close()
1099+
},
1100+
shouldStop: () => true,
1101+
},
1102+
holdBeforeTeardownMs: 3_000,
1103+
})
1104+
assert.equal(result.execution.code, "COMPLETED")
1105+
assert.equal(result.execution.completionMessageSeen, true)
1106+
assert.deepEqual(holdPauses, hostExited ? [] : [3_000])
1107+
// The hold runs before teardown, so CloseTask is sent only after it.
1108+
assert.equal(closeTaskSeenDuringHold, false)
1109+
assert.equal(f.commands.at(-1)?.commandName, TaskCommandName.CloseTask)
1110+
assert.equal(result.teardown.status, "ok")
1111+
assert.equal(result.teardown.hostExited, true)
1112+
const trace = writtenTrace(f)
1113+
// CloseTask exits immediately, so none of the recording hold belongs to teardown.
1114+
assert.equal(trace.execution.timings.teardownMs, 0)
10951115
})
1096-
assert.equal(result.execution.code, "COMPLETED")
1097-
assert.equal(result.execution.completionMessageSeen, true)
1098-
assert.deepEqual(holdPauses, [3_000])
1099-
// The hold runs before teardown, so CloseTask is sent only after it.
1100-
assert.equal(closeTaskSeenDuringHold, false)
1101-
assert.equal(f.commands.at(-1)?.commandName, TaskCommandName.CloseTask)
1102-
assert.equal(result.teardown.status, "ok")
1103-
})
1104-
1116+
}
11051117
it("skips the teardown hold without a completion message", async () => {
11061118
registerTestFixture("controller-hold-skipped", {
11071119
limits: {

‎scripts/solheim-smoke-workflow.test.mjs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,10 @@ describe("Solheim provider smoke workflow policy", () => {
6767
.split("\n")
6868
.map((line) => line.trim()),
6969
)
70-
assert.deepEqual(paths, [[output + "/verdict.json", output + "/trace.json"], [output + "/smoke.mp4"]])
70+
assert.deepEqual(paths, [
71+
[output + "/verdict.json", output + "/trace.json", output + "/sidebar-focus.json"],
72+
[output + "/smoke.mp4"],
73+
])
7174
for (const step of uploads) {
7275
assert.equal(
7376
step.if,

‎scripts/solheim-smoke/driver.mts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,13 @@ const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..")
88
const out = process.env.SOLHEIM_SMOKE_OUT_DIR
99
if (!out) throw new Error("Set SOLHEIM_SMOKE_OUT_DIR to a scratch directory")
1010
await fs.mkdir(out, { recursive: true })
11+
const testsDiagnosticsPath = path.resolve(out, "sidebar-focus.json")
12+
// Replace stale evidence even if the host never reaches the sidebar module.
13+
await fs.writeFile(testsDiagnosticsPath, JSON.stringify({ status: "pending" }))
1114
const runtime = createHostRuntime({
1215
root,
1316
artifactsDir: out,
17+
testsDiagnosticsPath,
1418
// The runner module focuses the Zoo Code sidebar so the recording shows
1519
// the chat panel. Its run() never settles; the driver owns teardown.
1620
testsPath: path.join(root, "scripts", "solheim-smoke", "sidebar-runner.cjs"),

‎scripts/solheim-smoke/sidebar-runner.cjs‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
// The IPC start path keeps focusSidebar: false; this module owns focus instead.
88

99
const vscode = require("vscode")
10+
const fs = require("node:fs/promises")
1011

1112
const EXTENSION_ID = "ZooCodeOrganization.zoo-code"
1213
const FOCUS_COMMAND = "zoo-code.SidebarProvider.focus"
@@ -27,13 +28,21 @@ async function focusSidebarWhenReady() {
2728
}
2829
}
2930

31+
async function writeStatus(status) {
32+
const target = process.env.AGENT_EVALS_TESTS_DIAGNOSTICS_PATH
33+
if (!target) return
34+
// Fixed metadata only: extension errors can contain private host details.
35+
await fs.writeFile(target, JSON.stringify({ status }))
36+
}
37+
3038
exports.run = async () => {
3139
try {
3240
await focusSidebarWhenReady()
33-
} catch (error) {
41+
await writeStatus("ready")
42+
} catch {
3443
// Focus is best effort for the recording. A rejected run() would end
35-
// the test host and cut the smoke short, so log and stay pending.
36-
console.warn("[solheim-smoke] sidebar focus failed:", error && error.message ? error.message : error)
44+
// the test host and cut the smoke short. Persist bounded evidence instead.
45+
await writeStatus("failed").catch(() => {})
3746
}
3847
await new Promise(() => {})
3948
}
Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
import assert from "node:assert/strict"
2+
import fs from "node:fs/promises"
3+
import { describe, it } from "node:test"
4+
import vm from "node:vm"
5+
6+
const source = await fs.readFile(new URL("./sidebar-runner.cjs", import.meta.url), "utf8")
7+
8+
describe("sidebar recording diagnostics", () => {
9+
for (const scenario of ["ready", "focus-failed", "extension-missing", "write-failed"] as const) {
10+
it(`persists bounded evidence and keeps the host pending for ${scenario}`, async () => {
11+
const writes: { target: string; body: string }[] = []
12+
const commands: string[] = []
13+
const exports: { run?: () => Promise<void> } = {}
14+
const context = vm.createContext({
15+
exports,
16+
process: { env: { AGENT_EVALS_TESTS_DIAGNOSTICS_PATH: "/artifacts/sidebar-focus.json" } },
17+
require(name: string) {
18+
if (name === "node:fs/promises")
19+
return {
20+
async writeFile(target: string, body: string) {
21+
if (scenario === "write-failed") throw new Error("private filesystem detail")
22+
writes.push({ target, body })
23+
},
24+
}
25+
assert.equal(name, "vscode")
26+
return {
27+
extensions: {
28+
getExtension: () =>
29+
scenario === "extension-missing"
30+
? undefined
31+
: {
32+
isActive: true,
33+
exports: { isReady: () => true },
34+
},
35+
},
36+
commands: {
37+
async executeCommand(command: string) {
38+
commands.push(command)
39+
if (scenario === "focus-failed") throw new Error("private activation detail")
40+
},
41+
},
42+
}
43+
},
44+
})
45+
vm.runInContext(source, context)
46+
assert.ok(exports.run)
47+
let settled = false
48+
void exports.run().then(
49+
() => {
50+
settled = true
51+
},
52+
() => {
53+
settled = true
54+
},
55+
)
56+
// Flush async focus and file writes without waiting for the intentionally pending run().
57+
await new Promise<void>((resolve) => setImmediate(resolve))
58+
assert.equal(settled, false)
59+
assert.deepEqual(commands, scenario === "extension-missing" ? [] : ["zoo-code.SidebarProvider.focus"])
60+
assert.deepEqual(
61+
writes,
62+
scenario === "write-failed"
63+
? []
64+
: [
65+
{
66+
target: "/artifacts/sidebar-focus.json",
67+
body: JSON.stringify({ status: scenario === "ready" ? "ready" : "failed" }),
68+
},
69+
],
70+
)
71+
})
72+
}
73+
})

0 commit comments

Comments
 (0)