Skip to content

Commit 18de7a0

Browse files
onlydoleclaude
andcommitted
test(evals): guard case fails on main without a model
Models recover from both the old success and the new error, so the background wait guard case passes either way. Check in run.test.ts that its seeded wait on an unknown ID gets T3's error, and that the control's matched wait doesn't. The check fails on main's server code and passes with this fix. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 44f0a6e commit 18de7a0

1 file changed

Lines changed: 28 additions & 0 deletions

File tree

‎evals/mcp-tool-use/run.test.ts‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import {
2828
t3Default,
2929
t3SystemAppend,
3030
t3SubagentContext,
31+
withBackgroundWait,
3132
withDevices,
3233
} from "./run.ts";
3334
import { briefRoute, gradeDevice, shellInvocations } from "./device-grading.ts";
@@ -569,6 +570,33 @@ it.effect("builds device cases from product-backed worlds and synthetic prior tu
569570
}).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
570571
);
571572

573+
// Models recover from either result, so this keeps the guard case failing on the old success.
574+
it.effect("seeds T3's error for a prior background wait whose IDs match nothing", () =>
575+
Effect.gen(function* () {
576+
const cases = NodeFS.readFileSync(
577+
new URL("./cases/orchestration.ndjson", import.meta.url),
578+
"utf8",
579+
)
580+
.trim()
581+
.split("\n")
582+
.map((line) => decodeDeviceCase(line));
583+
const base = buildBase(template, { model: "claude-sonnet-5-5", keepUserInstructions: false });
584+
for (const [id, isError] of [
585+
["background-wait-unmatched-id", true],
586+
["background-wait-matched-id", false],
587+
] as const) {
588+
const testCase = cases.find((c) => c.id === id)!;
589+
const [wait] = (testCase.setup.priorTurns ?? [])
590+
.flatMap((turn) => turn.toolCalls ?? [])
591+
.filter((call) => call.tool === "t3_background_wait");
592+
assert.isDefined(wait, id);
593+
const sample = makeSample(testCase, base);
594+
yield* withBackgroundWait(sample);
595+
assert.equal(sample.priorErrors.has(wait), isError, id);
596+
}
597+
}).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
598+
);
599+
572600
it.each([
573601
["Do not use SSH for UI but use device_open and its returned command.", "managed"],
574602
["Use device_open instead of SSH for screenshots.", "managed"],

0 commit comments

Comments
 (0)