Skip to content

Commit 568b3d4

Browse files
author
PR Batch Tester
committed
fix(evals): reject incomplete WebMCP replays
1 parent d492068 commit 568b3d4

2 files changed

Lines changed: 23 additions & 12 deletions

File tree

‎evals/mcp-tool-use/run.test.ts‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -302,18 +302,26 @@ it("runs WebMCP expressions without host objects", () => {
302302
assert.deepEqual(page(expression), [], expression);
303303
assert.deepEqual(
304304
page(
305-
"document.modelContext.getTools().then(([tool]) => { document.modelContext.executeTool(tool); while (true) {} })",
305+
"document.modelContext.getTools().then(([tool]) => { document.modelContext.executeTool(tool); throw new Error('page') })",
306306
),
307307
[{ tool: "mock", input: {} }],
308308
);
309+
for (const expression of [
310+
"document.modelContext.getTools().then(([tool]) => { document.modelContext.executeTool(tool); while (true) {} })",
311+
"document.modelContext.getTools().then(([tool]) => { for (let n = 0; n < 129; n++) document.modelContext.executeTool(tool) })",
312+
"document.modelContext.getTools().then(async ([tool]) => { for (let n = 0; n < 129; n++) await document.modelContext.executeTool(tool).catch(() => {}) })",
313+
])
314+
assert.throws(() => page(expression), /WebMCP replay terminated/, expression);
309315
});
310316

311317
it("rejects partial WebMCP results from an unexpectedly terminated replay", () => {
312318
const partial = '[0,"list_notes",{}]\n';
313319
using child = vi.spyOn(NodeChildProcess, "spawnSync");
314-
for (const [status, signal] of [
315-
[1, null],
316-
[null, "SIGKILL"],
320+
for (const [status, signal, code] of [
321+
[1, null, undefined],
322+
[null, "SIGKILL", undefined],
323+
[null, "SIGTERM", "ETIMEDOUT"],
324+
[null, "SIGTERM", "ENOBUFS"],
317325
] as const) {
318326
child.mockReturnValue({
319327
pid: 1,
@@ -322,6 +330,7 @@ it("rejects partial WebMCP results from an unexpectedly terminated replay", () =
322330
stderr: "",
323331
status,
324332
signal,
333+
...(code === undefined ? {} : { error: Object.assign(new Error(code), { code }) }),
325334
});
326335
assert.throws(
327336
() =>
@@ -343,7 +352,7 @@ it("rejects partial WebMCP results from an unexpectedly terminated replay", () =
343352
stopReason: "end_turn",
344353
},
345354
),
346-
/WebMCP replay terminated/,
355+
/WebMCP replay terminated|ETIMEDOUT|ENOBUFS/,
347356
);
348357
}
349358
});

‎evals/mcp-tool-use/run.ts‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1385,7 +1385,7 @@ const setup = (tools, origin) => {
13851385
delete globalThis.__tools; delete globalThis.__origin;
13861386
const page = globalThis, stringify = JSON.stringify, parse = JSON.parse, names = tools.map(t => t.name);
13871387
let executed = [], count = 0;
1388-
Object.defineProperty(page, "__take", { value: () => { const taken = stringify(executed); executed = []; return taken; } });
1388+
Object.defineProperty(page, "__take", { value: () => { const taken = stringify({ executed, complete: count <= 128 }); executed = []; return taken; } });
13891389
page.window = page.self = page;
13901390
page.location = { origin, href: origin + "/" };
13911391
page.console = { log() {}, info() {}, warn() {}, error() {}, debug() {} };
@@ -1413,10 +1413,13 @@ for (const [page, expressions] of pages.entries()) {
14131413
for (const expression of expressions) {
14141414
try {
14151415
run(expression);
1416-
} catch {
1416+
} catch (error) {
1417+
if (error?.code === "ERR_SCRIPT_EXECUTION_TIMEOUT") throw error;
14171418
// A page error still keeps the executions that ran before it.
14181419
}
1419-
for (const [tool, input] of JSON.parse(run("__take()")))
1420+
const { executed, complete } = JSON.parse(run("__take()"));
1421+
if (!complete) throw new Error("WebMCP replay exceeded 128 tool executions.");
1422+
for (const [tool, input] of executed)
14201423
process.stdout.write(JSON.stringify([page, tool, input]) + "\n");
14211424
}
14221425
}`;
@@ -1443,10 +1446,9 @@ export const executeWebMcp = (
14431446
maxBuffer: 16 * 1024 * 1024,
14441447
},
14451448
);
1446-
// A hung or oversized page keeps what it printed; a child that never started is a harness fault.
1447-
const code = (child.error as NodeJS.ErrnoException | undefined)?.code;
1448-
if (child.error !== undefined && code !== "ETIMEDOUT" && code !== "ENOBUFS") throw child.error;
1449-
if (child.error === undefined && (child.status !== 0 || child.signal !== null)) {
1449+
// An incomplete replay cannot establish that a forbidden tool was never called.
1450+
if (child.error !== undefined) throw child.error;
1451+
if (child.status !== 0 || child.signal !== null) {
14501452
throw new Error(
14511453
`WebMCP replay terminated with status ${child.status} and signal ${child.signal}.`,
14521454
);

0 commit comments

Comments
 (0)