Skip to content

Commit 9cf617f

Browse files
t3dotggclaude
andcommitted
fix(evals): unloaded deferred calls are not graded; templates drop attachments
- Claude Code rejects a deferred tool that was not loaded, so that call never ran. The grader no longer records it. Mocked tool errors still count. - trim-capture keeps only text blocks of the first user turn, so user images and documents never reach a template. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 5cc594a commit 9cf617f

3 files changed

Lines changed: 28 additions & 1 deletion

File tree

‎evals/mcp-tool-use/run.test.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import {
77
executeCodex,
88
gradeSample,
99
makeSample,
10+
ranCalls,
1011
type McpToolUseCase,
1112
mockOutput,
1213
normalizeToolInput,
@@ -287,6 +288,20 @@ it("replays earlier capability results with the model under test as the caller",
287288
assert.include(String(replayed?.content), '"inheritedModel":"claude-opus-5-5"');
288289
});
289290

291+
it("does not record a deferred tool called before it was loaded", () => {
292+
const base = buildBase(template, { model: "claude-sonnet-5-5", keepUserInstructions: false });
293+
const sample = makeSample(childCase, base);
294+
const use = (name: string) => ({ type: "tool_use" as const, id: name, name, input: {} });
295+
const ran = ranCalls(sample, 0, [
296+
use("mcp__t3-code__link_pull_request"),
297+
use("mcp__host__term_Bash"),
298+
]);
299+
assert.deepEqual(
300+
ran.map((call) => call.tool),
301+
["Bash"],
302+
);
303+
});
304+
290305
it("answers simple mocked commands itself and gives the simulator the sample's history", () => {
291306
const sample = makeSample(
292307
childCase,

‎evals/mcp-tool-use/run.ts‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1404,6 +1404,16 @@ const recordedCall = (step: number, use: typeof ToolUse.Type): Call => {
14041404
return { step, tool, input: claudeAgent ? { run_in_background: true, ...input } : input };
14051405
};
14061406

1407+
/** Calls that ran. Claude Code rejects a deferred tool that was not loaded, so it never runs. */
1408+
export const ranCalls = (
1409+
{ base, tools }: Sample,
1410+
step: number,
1411+
uses: ReadonlyArray<typeof ToolUse.Type>,
1412+
) =>
1413+
uses
1414+
.filter((use) => base.codex !== undefined || hasTool(tools, use.name))
1415+
.map((use) => recordedCall(step, use));
1416+
14071417
/**
14081418
* Runs one sample until the model ends its turn, uses up its steps, or fails a check that no
14091419
* later step can undo.
@@ -1438,7 +1448,7 @@ const runSample = Effect.fn("evaluateMcpToolUse.runSample")(function* (
14381448
messages.push({ role: "assistant", content: response.content });
14391449
said.push(...response.content.filter(isText).map((block) => block.text));
14401450
const uses = response.content.filter(isToolUse);
1441-
calls.push(...uses.map((use) => recordedCall(step, use)));
1451+
calls.push(...ranCalls(sample, step, uses));
14421452
if (response.stop_reason !== "tool_use") {
14431453
stopReason = response.stop_reason ?? "unknown";
14441454
break;

‎evals/mcp-tool-use/trim-capture.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,8 @@ const trimClaude = (native: Native) => {
8787
if (!first || !Array.isArray(first.content))
8888
throw new Error("The capture has no first user turn.");
8989
const blocks = (first.content as ReadonlyArray<Block>).flatMap((block) => {
90+
// run.ts reads only text here. Images and documents are the user's attachments.
91+
if (block.type !== "text") return [];
9092
const text = block.text ?? "";
9193
// The user's CLAUDE.md files.
9294
if (text.includes("Codebase and user instructions are")) return [];

0 commit comments

Comments
 (0)