Skip to content

Commit d492068

Browse files
author
PR Batch Tester
committed
fix(evals): resolve omitted WebMCP tab targets
1 parent 29e2204 commit d492068

2 files changed

Lines changed: 36 additions & 23 deletions

File tree

‎evals/mcp-tool-use/run.test.ts‎

Lines changed: 32 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -181,8 +181,14 @@ it("grades the WebMCP tools an expression executes, not the text it contains", (
181181
if (!testCase) throw new Error(`Missing case ${id}`);
182182
return testCase.expect;
183183
};
184-
const run = (expression: unknown, tabId = "tab_1"): SampleRun => ({
185-
calls: [{ step: 0, tool: "preview_evaluate", input: { tabId, expression } }],
184+
const run = (expression: unknown, tabId: string | null = "tab_1"): SampleRun => ({
185+
calls: [
186+
{
187+
step: 0,
188+
tool: "preview_evaluate",
189+
input: { ...(tabId === null ? {} : { tabId }), expression },
190+
},
191+
],
186192
text: "",
187193
stopReason: "end_turn",
188194
});
@@ -216,14 +222,21 @@ it("grades the WebMCP tools an expression executes, not the text it contains", (
216222
null,
217223
])
218224
assert.lengthOf(failed(write, run(expression)), 1, String(expression));
225+
assert.deepEqual(failed(write, run(create, null)), []);
219226
assert.lengthOf(failed(write, run(create, "wrong_tab")), 1);
220227
const saveTools = "document.modelContext.getTools().then(tools => { window.saved = tools })";
221228
const useSaved = `document.modelContext.executeTool(window.saved.find(t => t.name === 'create_note'), {title: 'Groceries', body: 'milk, eggs, bread'})`;
222-
const across = (saveTab: string): SampleRun => ({
229+
const across = (saveTab: string | null, useTab: string | null = "tab_1"): SampleRun => ({
223230
...run(saveTools, saveTab),
224-
calls: [...run(saveTools, saveTab).calls, ...run(useSaved).calls],
231+
calls: [...run(saveTools, saveTab).calls, ...run(useSaved, useTab).calls],
225232
});
226-
assert.deepEqual(failed(write, across("tab_1")), []);
233+
for (const [saveTab, useTab] of [
234+
["tab_1", "tab_1"],
235+
[null, "tab_1"],
236+
["tab_1", null],
237+
[null, null],
238+
] as const)
239+
assert.deepEqual(failed(write, across(saveTab, useTab)), []);
227240
assert.lengthOf(failed(write, across("tab_2")), 1);
228241
assert.deepEqual(
229242
failed(write, {
@@ -336,22 +349,20 @@ it("rejects partial WebMCP results from an unexpectedly terminated replay", () =
336349
});
337350

338351
it("contains page rejections no expression handles", () => {
339-
const grader = new URL("./run.ts", import.meta.url).href;
340-
const script = `
341-
import { gradeSample } from ${JSON.stringify(grader)};
342-
const webmcp = { tabId: "tab_1", origin: "https://acme.dev", tools: [{ name: "mock", description: "Mock." }], calls: [{ tool: "mock" }] };
343-
const grade = (expression) => gradeSample({ webmcp }, { calls: [{ step: 0, tool: "preview_evaluate", input: { tabId: "tab_1", expression } }], text: "", stopReason: "end_turn" }).every((check) => check.pass);
344-
const first = grade("document.modelContext.getTools().then(() => { throw new Error('page') }); (async () => { throw new Error('page') })(); 1");
345-
await new Promise((resolve) => setImmediate(resolve));
346-
const later = grade("document.modelContext.getTools().then(([tool]) => document.modelContext.executeTool(tool, {}))");
347-
process.stdout.write(JSON.stringify([first, later]));
348-
`;
349-
const child = NodeChildProcess.spawnSync(
350-
process.execPath,
351-
["--input-type=module", "--no-warnings", "-e", script],
352-
{ encoding: "utf8" },
353-
);
354-
assert.deepEqual([child.status, child.stdout], [0, "[false,true]"], child.stderr);
352+
const tools = [{ name: "mock", description: "Mock." }];
353+
assert.deepEqual(
354+
executeWebMcp(
355+
[
356+
[
357+
"document.modelContext.getTools().then(() => { throw new Error('page') }); (async () => { throw new Error('page') })(); 1",
358+
"document.modelContext.getTools().then(([tool]) => document.modelContext.executeTool(tool, {}))",
359+
],
360+
],
361+
"https://acme.dev",
362+
tools,
363+
),
364+
[[{ tool: "mock", input: {} }]],
365+
);
355366
});
356367

357368
it("bounds numbers and reads target.options in either schema form", () => {

‎evals/mcp-tool-use/run.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1466,8 +1466,10 @@ const gradeWebMcp = (expect: McpToolUseCase["expect"]["webmcp"], run: SampleRun)
14661466
// Expressions on one tab share its page, as in the browser; other tabs never see it.
14671467
const pages = new Map<unknown, Array<string>>();
14681468
for (const { tool, input } of run.calls)
1469-
if (tool === "preview_evaluate" && isRecord(input) && typeof input.expression === "string")
1470-
pages.set(input.tabId, [...(pages.get(input.tabId) ?? []), input.expression]);
1469+
if (tool === "preview_evaluate" && isRecord(input) && typeof input.expression === "string") {
1470+
const tabId = input.tabId ?? expect.tabId;
1471+
pages.set(tabId, [...(pages.get(tabId) ?? []), input.expression]);
1472+
}
14711473
const tabs = [...pages.keys()];
14721474
const executed = executeWebMcp([...pages.values()], expect.origin, expect.tools).flatMap(
14731475
(page, at) => page.map((execution) => ({ ...execution, tabId: tabs[at] })),

0 commit comments

Comments
 (0)