Skip to content

Commit 29e2204

Browse files
author
PR Batch Tester
committed
test(evals): reject abnormal WebMCP replay termination
1 parent bf43b59 commit 29e2204

2 files changed

Lines changed: 48 additions & 0 deletions

File tree

‎evals/mcp-tool-use/run.test.ts‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices";
55
import * as Effect from "effect/Effect";
66
import * as Schema from "effect/Schema";
77
import { assert, expect, it } from "@effect/vitest";
8+
import { vi } from "vite-plus/test";
89
import { ComposerContextId, EnvironmentId, ThreadId } from "@t3tools/contracts";
910
import { projectComposerContextForProvider } from "@t3tools/shared/composerContextReferences";
1011

@@ -32,6 +33,8 @@ import {
3233
import { briefRoute, gradeDevice, shellInvocations } from "./device-grading.ts";
3334
import { taskPrompt } from "../../apps/server/src/mcp/delegatedTaskModes.ts";
3435

36+
vi.mock("node:child_process", { spy: true });
37+
3538
it("grades Codex shell and subagent calls with the shared rules", () => {
3639
const calls = [
3740
{ name: "exec_command", input: { cmd: "sleep 60" } },
@@ -292,6 +295,46 @@ it("runs WebMCP expressions without host objects", () => {
292295
);
293296
});
294297

298+
it("rejects partial WebMCP results from an unexpectedly terminated replay", () => {
299+
const partial = '[0,"list_notes",{}]\n';
300+
using child = vi.spyOn(NodeChildProcess, "spawnSync");
301+
for (const [status, signal] of [
302+
[1, null],
303+
[null, "SIGKILL"],
304+
] as const) {
305+
child.mockReturnValue({
306+
pid: 1,
307+
output: [null, partial, ""],
308+
stdout: partial,
309+
stderr: "",
310+
status,
311+
signal,
312+
});
313+
assert.throws(
314+
() =>
315+
gradeSample(
316+
{
317+
webmcp: {
318+
tabId: "tab_1",
319+
origin: "https://acme.dev",
320+
tools: [{ name: "list_notes", description: "List notes." }],
321+
calls: [{ tool: "list_notes" }],
322+
notCalls: ["delete_note"],
323+
},
324+
},
325+
{
326+
calls: [
327+
{ step: 0, tool: "preview_evaluate", input: { tabId: "tab_1", expression: "1" } },
328+
],
329+
text: "",
330+
stopReason: "end_turn",
331+
},
332+
),
333+
/WebMCP replay terminated/,
334+
);
335+
}
336+
});
337+
295338
it("contains page rejections no expression handles", () => {
296339
const grader = new URL("./run.ts", import.meta.url).href;
297340
const script = `

‎evals/mcp-tool-use/run.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1446,6 +1446,11 @@ export const executeWebMcp = (
14461446
// A hung or oversized page keeps what it printed; a child that never started is a harness fault.
14471447
const code = (child.error as NodeJS.ErrnoException | undefined)?.code;
14481448
if (child.error !== undefined && code !== "ETIMEDOUT" && code !== "ENOBUFS") throw child.error;
1449+
if (child.error === undefined && (child.status !== 0 || child.signal !== null)) {
1450+
throw new Error(
1451+
`WebMCP replay terminated with status ${child.status} and signal ${child.signal}.`,
1452+
);
1453+
}
14491454
for (const line of (child.stdout ?? "").split("\n")) {
14501455
const decoded = decodeWebMcpExecuted(line);
14511456
if (Option.isSome(decoded)) {

0 commit comments

Comments
 (0)