Skip to content

Commit a70f9f8

Browse files
committed
fix(smoke): address review on recorder, runner, and fixture policy
1 parent ff18e4f commit a70f9f8

6 files changed

Lines changed: 95 additions & 12 deletions

File tree

‎scripts/agent-evals/contract.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,22 @@ const serializeProjection = (projection: Record<string, unknown>): string | unde
126126
}
127127
}
128128

129+
// The built configuration is what the extension runs. It must carry exactly the
130+
// declared tool policy, so a drifted builder cannot widen tool access.
131+
export function configurationMatchesToolPolicy(
132+
config: Record<string, unknown>,
133+
policy: AgentFixture["toolPolicy"],
134+
): boolean {
135+
const sameList = (a: unknown, b: readonly string[]) =>
136+
Array.isArray(a) && a.length === b.length && b.every((item) => a.includes(item))
137+
if (config["mode"] !== policy.mode || !sameList(config["disabledTools"], policy.disabledTools)) return false
138+
const modes = config["customModes"]
139+
const custom = Array.isArray(modes) ? modes.find((m) => isPlainObject(m) && m["slug"] === policy.mode) : undefined
140+
if (!isPlainObject(custom) || custom["roleDefinition"] !== policy.roleDefinition) return false
141+
if (!sameList(custom["groups"], policy.groups)) return false
142+
return ALLOW_KEYS.every((key) => config[`alwaysAllow${key[0]!.toUpperCase()}${key.slice(1)}`] === policy.allow[key])
143+
}
144+
129145
export function validateFixture(fixture: AgentFixture): void {
130146
if (!isPlainObject(fixture)) fail("fixture must be an object")
131147
if (!isPlainObject(fixture.identity)) fail("fixture identity missing")

‎scripts/agent-evals/runner.test.ts‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,23 @@ function registerTestFixture(name: string, overrides: Partial<AgentFixture> = {}
4141
teardownStepMs: 5_000,
4242
},
4343
settingsProjection: { provider: "test", model: `model-${name}` },
44-
buildConfiguration: () => ({ apiProvider: "openai", openAiApiKey: CREDENTIAL }),
44+
buildConfiguration: () => ({
45+
apiProvider: "openai",
46+
openAiApiKey: CREDENTIAL,
47+
mode: "test",
48+
customModes: [{ slug: "test", name: "Test", roleDefinition: "Run one synthetic task.", groups: [] }],
49+
disabledTools: ["new_task"],
50+
alwaysAllowReadOnly: false,
51+
alwaysAllowReadOnlyOutsideWorkspace: false,
52+
alwaysAllowWrite: false,
53+
alwaysAllowWriteOutsideWorkspace: false,
54+
alwaysAllowWriteProtected: false,
55+
alwaysAllowExecute: false,
56+
alwaysAllowMcp: false,
57+
alwaysAllowModeSwitch: false,
58+
alwaysAllowSubtasks: false,
59+
alwaysAllowFollowupQuestions: false,
60+
}),
4561
...overrides,
4662
}
4763
registerFixture(fixture)
@@ -466,6 +482,18 @@ describe("agent fixture controller", () => {
466482
assert.deepEqual(trace.settings, { available: false, reason: "credentialMissing" })
467483
})
468484

485+
it("rejects a configuration that widens the declared tool policy, and never launches", async () => {
486+
const base = registerTestFixture("controller-policy-base")
487+
registerTestFixture("controller-policy", {
488+
buildConfiguration: (credential) => ({ ...base.buildConfiguration(credential), alwaysAllowExecute: true }),
489+
})
490+
const f = harness("success")
491+
const result = await runAgentFixture(f.runtime, { fixtureName: "controller-policy", credential: CREDENTIAL })
492+
assert.equal(result.execution.stopReason, "fixtureRejected")
493+
assert.equal(f.launched, false)
494+
assert.deepEqual(writtenTrace(f).settings, { available: false, reason: "configurationViolatesToolPolicy" })
495+
})
496+
469497
it("rejects a fixture whose projection carries the credential, and never launches", async () => {
470498
registerTestFixture("controller-leak", { settingsProjection: { apiKey: CREDENTIAL } })
471499
const f = harness("success")

‎scripts/agent-evals/runner.ts‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { randomUUID } from "node:crypto"
22
import { RooCodeEventName, TaskCommandName, taskEventSchema, type TaskCommand, type TaskEvent } from "@roo-code/types"
3-
import type { AgentFixtureWorkspaceFile } from "./contract.ts"
3+
import { configurationMatchesToolPolicy, type AgentFixtureWorkspaceFile } from "./contract.ts"
44
import { getFixture } from "./registry.ts"
55
import { createTaskStartWaiter, type TaskStartWaiter } from "./task-start.ts"
66
import {
@@ -82,7 +82,11 @@ const traceSettings = (fixtureProjection: Record<string, unknown>, credential: s
8282
if (projectionJson === undefined) return { available: false, reason: "projectionNotSerializable" }
8383
if (credential.length > 0 && projectionJson.includes(credential))
8484
return { available: false, reason: "credentialInProjection" }
85-
return { available: true, projection: fixtureProjection, digest: digestProjection(fixtureProjection) }
85+
return {
86+
available: true,
87+
projection: JSON.parse(projectionJson) as Record<string, unknown>,
88+
digest: digestProjection(fixtureProjection),
89+
}
8690
} catch {
8791
return { available: false, reason: "projectionNotSerializable" }
8892
}
@@ -224,11 +228,16 @@ export async function runAgentFixture(runtime: AgentRuntime, options: AgentRunOp
224228
settings = { available: false, reason: "credentialMissing" }
225229
} else {
226230
settings = traceSettings(fixture.settingsProjection, credential)
231+
const configuration = fixture.buildConfiguration(credential)
232+
if (settings.available && !configurationMatchesToolPolicy(configuration, fixture.toolPolicy))
233+
settings = { available: false, reason: "configurationViolatesToolPolicy" }
227234
if (!settings.available) {
228235
stopReason = "fixtureRejected"
229236
} else {
230237
host = await runtime.launch(fixture.workspace.files)
231238
host.onExit(markLost)
239+
// An interrupt during launch must not connect or send a task.
240+
if (sessionLost) throw new Error("startup interrupted")
232241
client = runtime.connect()
233242
client.onDisconnect(() => {
234243
sessionLost = true
@@ -244,6 +253,7 @@ export async function runAgentFixture(runtime: AgentRuntime, options: AgentRunOp
244253
code = "ACTIVATION_FAILED"
245254
stopReason = "activationFailed"
246255
} else {
256+
if (sessionLost) throw new Error("startup interrupted")
247257
const requestId = randomUUID()
248258
waiter = createTaskStartWaiter({ requestId, timeoutMs: fixture.limits.taskStartTimeoutMs })
249259
// A synchronous send failure must not leave a rejected timer promise.
@@ -255,7 +265,7 @@ export async function runAgentFixture(runtime: AgentRuntime, options: AgentRunOp
255265
requestId,
256266
text: fixture.prompt,
257267
newTab: false,
258-
configuration: fixture.buildConfiguration(credential),
268+
configuration,
259269
},
260270
})
261271
try {

‎scripts/solheim-smoke-workflow.test.mjs‎

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import assert from "node:assert/strict"
22
import fs from "node:fs"
3-
import { URL } from "node:url"
3+
import { URL, fileURLToPath } from "node:url"
44
import { describe, it } from "node:test"
55
import { spawn, spawnSync } from "node:child_process"
66
import os from "node:os"
@@ -118,19 +118,20 @@ function recorderEnv(bin, outDir, providerMarker, extraEnv) {
118118

119119
const stopBudgetEnv = (stopBudget) => (stopBudget ? { SOLHEIM_SMOKE_STOP_BUDGET_S: String(stopBudget) } : {})
120120

121-
const recorderScript = new URL("./solheim-smoke/record.sh", import.meta.url).pathname
121+
const recorderScript = fileURLToPath(new URL("./solheim-smoke/record.sh", import.meta.url))
122122

123123
function runRecorder(
124124
args,
125-
{ ffmpeg, ffmpegWrites = true, ffprobeOk = true, seedFinal, providerMarker, stopBudget } = {},
125+
{ ffmpeg, ffmpegWrites = true, ffprobeOk = true, seedFinal, seedMetadata, providerMarker, stopBudget, extraEnv } = {},
126126
) {
127127
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "record-sh-"))
128128
const bin = stubRecorderBin(dir, { ffmpeg, ffmpegWrites, ffprobeOk })
129129
const paths = recorderPaths(dir)
130130
fs.mkdirSync(paths.outDir, { recursive: true })
131131
if (seedFinal !== undefined) fs.writeFileSync(paths.finalRecording, seedFinal)
132+
for (const name of seedMetadata ?? []) fs.writeFileSync(path.join(paths.outDir, name), "stale")
132133
const result = spawnSync("bash", [recorderScript, ...args], {
133-
env: recorderEnv(bin, paths.outDir, providerMarker, stopBudgetEnv(stopBudget)),
134+
env: recorderEnv(bin, paths.outDir, providerMarker, { ...stopBudgetEnv(stopBudget), ...extraEnv }),
134135
encoding: "utf8",
135136
timeout: 20_000,
136137
})
@@ -193,6 +194,28 @@ describe("record.sh exit and validation behavior", () => {
193194
run.cleanup()
194195
}
195196
})
197+
it("removes stale verdict and trace metadata at start", () => {
198+
const run = runRecorder(["true"], {
199+
seedMetadata: ["verdict.json", "trace.json", "verdict.json.1.tmp"],
200+
})
201+
try {
202+
for (const name of ["verdict.json", "trace.json", "verdict.json.1.tmp"])
203+
assert.equal(fs.existsSync(path.join(run.outDir, name)), false, name)
204+
} finally {
205+
run.cleanup()
206+
}
207+
})
208+
209+
it("refuses a supplied command while the provider credential is set", () => {
210+
const run = runRecorder(["true"], { extraEnv: { SOLHEIM_API_KEY: "sk-test" } })
211+
try {
212+
assert.equal(run.result.status, 2)
213+
assert.match(run.result.stderr, /Refusing/)
214+
} finally {
215+
run.cleanup()
216+
}
217+
})
218+
196219
it("fails when the recording is missing", () => {
197220
const run = runRecorder(["true"], { ffmpegWrites: false })
198221
try {

‎scripts/solheim-smoke/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ about Solheim, the prompt, or the expected completion.
1313

1414
The active workflow is `.github/workflows/solheim-provider-smoke.yml`. It is manual-only, restricted to
1515
`refs/heads/main`, and checks out the exact triggering SHA. It builds trusted extension code before the
16-
single credential-bearing step. There is one job, no PR input or approval gate, no review/posting job,
16+
single credential-bearing step. There is one job, no PR input, no review/posting job,
1717
and `contents: read` is the only GitHub permission. This deliberately validates the provider integration
1818
on main, not arbitrary PR artifacts. It is not a required PR review check.
1919

‎scripts/solheim-smoke/record.sh‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,10 @@ mkdir -p "$SOLHEIM_SMOKE_OUT_DIR"
77
final_recording="$SOLHEIM_SMOKE_OUT_DIR/smoke.mp4"
88
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
99

10-
# A stale final from an earlier attempt must never survive into this attempt's
11-
# upload. Remove only this exact path.
12-
rm -f "$final_recording"
10+
# Stale output from an earlier attempt must never survive into this attempt's
11+
# upload. Remove only these exact paths and pending artifact writes.
12+
rm -f "$final_recording" "$SOLHEIM_SMOKE_OUT_DIR/verdict.json" "$SOLHEIM_SMOKE_OUT_DIR/trace.json" \
13+
"$SOLHEIM_SMOKE_OUT_DIR"/verdict.json.*.tmp "$SOLHEIM_SMOKE_OUT_DIR"/trace.json.*.tmp
1314

1415
# The recorder needs display access, not provider/GitHub credentials. Fragmented
1516
# MP4 keeps completed fragments readable if the action is forcibly interrupted.
@@ -103,8 +104,13 @@ sleep 0.5
103104
kill -0 "$recorder_pid" 2>/dev/null || exit 1
104105

105106
# Arguments are used for credential-free recorder tests; CI supplies none.
107+
# An injected command would inherit the provider credential, so reject it.
106108
# Backgrounding the provider keeps the signal traps responsive during the run.
107109
if [[ "$#" -gt 0 ]]; then
110+
if [[ -n "${SOLHEIM_API_KEY:-}" ]]; then
111+
echo "Refusing to run a supplied command while SOLHEIM_API_KEY is set" >&2
112+
exit 2
113+
fi
108114
"$@" &
109115
else
110116
# Invoke the Node driver directly: a package-manager wrapper would sit

0 commit comments

Comments
 (0)