Skip to content

Commit 24e8ee1

Browse files
t3dotggclaude
andcommitted
test(server): MCP eval cases follow the native-first subagent policy
Same-provider subagents with no extra settings that the parent waits for now pass on native tools. Cross-provider work, explicit settings, and work that outlives the turn still need delegate_task. A Claude Agent call with no run_in_background counts as background, which is Claude Code's default. The secret check no longer matches "paste the build log here", and "not on PATH" counts as an unavailable provider. Sol's code-mode cells take more model calls, so unset --max-steps now means 6 for Claude and 10 for Sol. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 9684d6e commit 24e8ee1

3 files changed

Lines changed: 22 additions & 19 deletions

File tree

‎apps/server/scripts/evaluate-mcp-tool-use.ts‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1282,6 +1282,14 @@ const callModel = (apiKey: string, body: unknown, proxy?: Config["proxy"]) =>
12821282
Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(ApiResponse))),
12831283
);
12841284

1285+
/** A tool call as graded. Claude Code runs an `Agent` in the background unless told otherwise. */
1286+
const recordedCall = (step: number, use: typeof ToolUse.Type): Call => {
1287+
const tool = normalizeToolName(use.name);
1288+
const input = normalizeToolInput(use.name, use.input);
1289+
const claudeAgent = tool === "Agent" && !use.name.endsWith("spawn_agent") && isRecord(input);
1290+
return { step, tool, input: claudeAgent ? { run_in_background: true, ...input } : input };
1291+
};
1292+
12851293
/**
12861294
* Runs one sample until the model ends its turn, uses up its steps, or fails a check that no
12871295
* later step can undo.
@@ -1313,13 +1321,7 @@ const runSample = Effect.fn("evaluateMcpToolUse.runSample")(function* (
13131321
messages.push({ role: "assistant", content: response.content });
13141322
said.push(...response.content.filter(isText).map((block) => block.text));
13151323
const uses = response.content.filter(isToolUse);
1316-
calls.push(
1317-
...uses.map((use) => ({
1318-
step,
1319-
tool: normalizeToolName(use.name),
1320-
input: normalizeToolInput(use.name, use.input),
1321-
})),
1322-
);
1324+
calls.push(...uses.map((use) => recordedCall(step, use)));
13231325
if (response.stop_reason !== "tool_use") {
13241326
stopReason = response.stop_reason ?? "unknown";
13251327
break;
@@ -1481,7 +1483,7 @@ const Flags = Schema.Struct({
14811483
case: Schema.optional(Schema.Array(Schema.String)),
14821484
samples: Schema.optional(PositiveInt),
14831485
concurrency: PositiveInt,
1484-
"max-steps": PositiveInt,
1486+
"max-steps": Schema.optional(PositiveInt),
14851487
effort: Schema.optional(Schema.Literals(EFFORTS)),
14861488
"shell-model": Schema.Literals(Object.keys(PRICES) as Array<PricedModel>),
14871489
"user-instructions": Schema.Literals(["keep", "drop"]),
@@ -1511,7 +1513,8 @@ const main = Effect.gen(function* () {
15111513
// Unset: 2 samples per case, and a 3rd when those 2 disagree.
15121514
samples: string,
15131515
concurrency: { ...string, default: "4" },
1514-
"max-steps": { ...string, default: "6" },
1516+
// Unset: 6 for Claude, 10 for Sol, whose code-mode cells take more steps.
1517+
"max-steps": string,
15151518
effort: string,
15161519
"shell-model": { ...string, default: "claude-haiku-5-5" },
15171520
// T3's text must work for every user, so the captured CLAUDE.md is dropped by default.
@@ -1591,7 +1594,7 @@ const main = Effect.gen(function* () {
15911594
model: flags.model,
15921595
...(flags.model === SOL ? { proxy: proxyConfig() } : {}),
15931596
effort,
1594-
maxSteps: flags["max-steps"],
1597+
maxSteps: flags["max-steps"] ?? (flags.model === SOL ? 10 : 6),
15951598
shellModel: flags["shell-model"],
15961599
};
15971600
const previous =

0 commit comments

Comments
 (0)