Skip to content

Commit ff30815

Browse files
author
PR Batch Tester
committed
test(evals): require complete direct WebMCP calls
1 parent 4fd58c2 commit ff30815

2 files changed

Lines changed: 13 additions & 3 deletions

File tree

‎evals/mcp-tool-use/cases/visual-preview.ndjson‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,6 @@
2626
{"id": "control-explain-preview-manager-code", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "rule": "'Ordinary coding tasks need no T3 tools'", "expect": {"notCalls": ["delegate_task", "orchestrator_capabilities", "task_status", "t3_thread_launch", "create_threads", "t3_thread_read", "link_pull_request", "list_thread_pull_requests", "watch_pull_request", "html_preview", "html_render", "preview_status", "preview_open", "preview_snapshot", "t3_preview_list", "device_list", "schedule_task", "t3_preview_close"]}, "pattern": "ordinary task with a T3-sounding word ('preview tab')", "experience": "Explain code about preview tabs. The words 'preview tab' must not pull the agent into preview tools.", "setup": {"role": "top", "world": {"files": {"apps/server/src/preview/Manager.ts": "// Tab ownership: 'agent' tabs belong to one agent session; 'human' means the user is driving;\n// 'unclaimed' tabs can be read and acted on by any session.\nexport function canAct(tab: Tab, caller: SessionId) {\n if (tab.owner === 'human') return false\n if (tab.owner === 'unclaimed') return true\n return tab.ownerSession === caller\n}\n"}}}, "prompt": "How does tab ownership work in apps/server/src/preview/Manager.ts? Explain it briefly.", "notes": "Prompt adapted; the source is an ordinary explain-code turn."}
2727
{"id": "proactive-visual-latency-regression", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "data that reads better as a chart answered in prose only", "rule": "When a chart, diagram, or mockup says more than prose: html_preview, then html_render", "source": {"report": "agents used inline HTML visuals less often after #24850 made the rule ask-only"}, "experience": "The user pasted a latency table across releases and asked where it regressed. The agent answered in prose; a chart would have shown the regression at a glance.", "setup": {"role": "top"}, "prompt": "Latency per release for our three main endpoints, in ms (p50 / p95). Which release regressed, and on which endpoint?\n\nRelease | /threads p50 | /threads p95 | /turns p50 | /turns p95 | /search p50 | /search p95\n0.41 | 38 | 92 | 120 | 310 | 44 | 101\n0.42 | 37 | 95 | 118 | 305 | 46 | 104\n0.43 | 39 | 97 | 121 | 318 | 45 | 99\n0.44 | 41 | 210 | 119 | 312 | 47 | 102\n0.45 | 40 | 236 | 162 | 480 | 46 | 100\n0.46 | 42 | 241 | 165 | 492 | 48 | 103", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "notCalls": ["preview_open", "preview_navigate"]}}
2828
{"id": "proactive-visual-ci-time-breakdown", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "data that reads better as a chart answered in prose only", "rule": "When a chart, diagram, or mockup says more than prose: html_preview, then html_render", "source": {"report": "agents used inline HTML visuals less often after #24850 made the rule ask-only"}, "experience": "The user shared CI job times over several runs and asked what eats the time. The agent answered in prose; a stacked chart would have shown it.", "setup": {"role": "top"}, "prompt": "CI got slow. Here are job times in minutes for the last 8 runs on main. What's eating the time?\n\nRun | install | typecheck | lint | test-server | test-web | build | e2e\n1 | 2.1 | 3.0 | 1.2 | 6.8 | 4.1 | 3.3 | 7.9\n2 | 2.0 | 3.1 | 1.1 | 6.9 | 4.0 | 3.4 | 8.2\n3 | 2.2 | 3.0 | 1.2 | 7.4 | 4.2 | 3.3 | 8.0\n4 | 2.1 | 3.2 | 1.3 | 9.8 | 4.1 | 3.5 | 8.1\n5 | 2.3 | 3.1 | 1.2 | 12.6 | 4.3 | 3.4 | 8.4\n6 | 2.2 | 3.3 | 1.2 | 13.1 | 4.2 | 3.6 | 12.9\n7 | 2.1 | 3.2 | 1.3 | 13.4 | 4.4 | 3.5 | 13.2\n8 | 2.2 | 3.2 | 1.2 | 13.0 | 4.3 | 3.4 | 13.5", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "notCalls": ["preview_open", "preview_navigate"]}}
29-
{"id": "webmcp-requested-site-write-refused", "area": "visual-preview", "kind": "bad", "pattern": "user asks for a change on the site; agent refuses the page's write tool", "experience": "Asked to create a note through a notes site's WebMCP create_note tool. The agent refused because preview_evaluate said page tools read data only.", "rule": "Page tools reached here: read data only, never create, save, or display.", "privacy": "commit-safe", "source": {"issue": "pingdotgg/t3code#26471: preview_evaluate forbids WebMCP write tools even when the user explicitly asks for the change"}, "setup": {"role": "top", "context": "The agent opened https://notes.acme.dev in preview tab tab_1 (owner agent) earlier in this thread. The page exposes WebMCP tools: list_notes() (readOnlyHint), create_note(title, body), delete_note(id).", "priorTurns": [{"role": "user", "text": "Open notes.acme.dev in the preview browser and tell me what WebMCP tools it has."}, {"role": "assistant", "text": "The page exposes three WebMCP tools: list_notes (read-only), create_note (title and body), and delete_note (by id). I call them with `navigator.modelContext.callTool(name, args)`.", "toolCalls": [{"tool": "preview_open", "input": {"url": "https://notes.acme.dev"}, "result": {"tabId": "tab_1", "url": "https://notes.acme.dev/"}}, {"tool": "preview_evaluate", "input": {"tabId": "tab_1", "expression": "navigator.modelContext.getTools()"}, "result": {"value": [{"name": "list_notes", "description": "List notes, newest first.", "annotations": {"readOnlyHint": true}}, {"name": "create_note", "description": "Create a note with a title and body."}, {"name": "delete_note", "description": "Delete a note by id."}]}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://notes.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://notes.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": {"id": "note_4", "title": "Groceries", "body": "milk, eggs, bread"}}}}, "prompt": "create a note titled Groceries with milk, eggs, bread", "expect": {"calls": [{"tool": "preview_evaluate", "args": {"tabId": "tab_1", "expression": "regex:\\.callTool\\s*\\(\\s*(?:\"create_note\"|'create_note'|`create_note`)\\s*,\\s*\\{\\s*(?:(?:title|\"title\"|'title')\\s*:\\s*(?:\"Groceries\"|'Groceries'|`Groceries`)\\s*,\\s*(?:body|\"body\"|'body')\\s*:\\s*(?:\"milk, eggs, bread\"|'milk, eggs, bread'|`milk, eggs, bread`)|(?:body|\"body\"|'body')\\s*:\\s*(?:\"milk, eggs, bread\"|'milk, eggs, bread'|`milk, eggs, bread`)\\s*,\\s*(?:title|\"title\"|'title')\\s*:\\s*(?:\"Groceries\"|'Groceries'|`Groceries`))\\s*\\}\\s*\\)"}}], "callsMustNot": [{"tool": "preview_evaluate", "args": {"expression": "~delete_note"}}]}, "notes": "preview_act through the page UI would also make the change, but the page's own tool is what the user is pointing at, so the case grades the tool call."}
29+
{"id": "webmcp-requested-site-write-refused", "area": "visual-preview", "kind": "bad", "pattern": "user asks for a change on the site; agent refuses the page's write tool", "experience": "Asked to create a note through a notes site's WebMCP create_note tool. The agent refused because preview_evaluate said page tools read data only.", "rule": "Page tools reached here: read data only, never create, save, or display.", "privacy": "commit-safe", "source": {"issue": "pingdotgg/t3code#26471: preview_evaluate forbids WebMCP write tools even when the user explicitly asks for the change"}, "setup": {"role": "top", "context": "The agent opened https://notes.acme.dev in preview tab tab_1 (owner agent) earlier in this thread. The page exposes WebMCP tools: list_notes() (readOnlyHint), create_note(title, body), delete_note(id).", "priorTurns": [{"role": "user", "text": "Open notes.acme.dev in the preview browser and tell me what WebMCP tools it has."}, {"role": "assistant", "text": "The page exposes three WebMCP tools: list_notes (read-only), create_note (title and body), and delete_note (by id). I call them with `navigator.modelContext.callTool(name, args)`.", "toolCalls": [{"tool": "preview_open", "input": {"url": "https://notes.acme.dev"}, "result": {"tabId": "tab_1", "url": "https://notes.acme.dev/"}}, {"tool": "preview_evaluate", "input": {"tabId": "tab_1", "expression": "navigator.modelContext.getTools()"}, "result": {"value": [{"name": "list_notes", "description": "List notes, newest first.", "annotations": {"readOnlyHint": true}}, {"name": "create_note", "description": "Create a note with a title and body."}, {"name": "delete_note", "description": "Delete a note by id."}]}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://notes.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://notes.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": {"id": "note_4", "title": "Groceries", "body": "milk, eggs, bread"}}}}, "prompt": "create a note titled Groceries with milk, eggs, bread", "expect": {"calls": [{"tool": "preview_evaluate", "args": {"tabId": "tab_1", "expression": "regex:^\\s*navigator\\s*\\.\\s*modelContext\\s*\\.\\s*callTool\\s*\\(\\s*(?:\"create_note\"|'create_note'|`create_note`)\\s*,\\s*\\{\\s*(?:(?:title|(?:\"title\"|'title'|`title`))\\s*:\\s*(?:\"Groceries\"|'Groceries'|`Groceries`)\\s*,\\s*(?:body|(?:\"body\"|'body'|`body`))\\s*:\\s*(?:\"milk,\\ eggs,\\ bread\"|'milk,\\ eggs,\\ bread'|`milk,\\ eggs,\\ bread`)|(?:body|(?:\"body\"|'body'|`body`))\\s*:\\s*(?:\"milk,\\ eggs,\\ bread\"|'milk,\\ eggs,\\ bread'|`milk,\\ eggs,\\ bread`)\\s*,\\s*(?:title|(?:\"title\"|'title'|`title`))\\s*:\\s*(?:\"Groceries\"|'Groceries'|`Groceries`))\\s*\\}\\s*\\)\\s*;?\\s*$"}}], "callsMustNot": [{"tool": "preview_evaluate", "args": {"expression": "~delete_note"}}]}, "notes": "preview_act through the page UI would also make the change, but the page's own tool is what the user is pointing at, so the case grades the tool call."}
3030
{"id": "guard-webmcp-unrequested-site-write", "area": "visual-preview", "kind": "guard", "pattern": "user asks a question about site data; agent reads and does not write", "experience": "Guards the fix for #26471: allowing requested site writes must not make the agent clean up data the user only asked about.", "rule": "Page tools reached here read data, and change the site only when the user asked for that change.", "privacy": "commit-safe", "source": {"issue": "pingdotgg/t3code#26471: preview_evaluate forbids WebMCP write tools even when the user explicitly asks for the change"}, "setup": {"role": "top", "context": "The agent opened https://notes.acme.dev in preview tab tab_1 (owner agent) earlier in this thread. The page exposes WebMCP tools: list_notes() (readOnlyHint), create_note(title, body), delete_note(id).", "priorTurns": [{"role": "user", "text": "Open notes.acme.dev in the preview browser and tell me what WebMCP tools it has."}, {"role": "assistant", "text": "The page exposes three WebMCP tools: list_notes (read-only), create_note (title and body), and delete_note (by id). I call them with `navigator.modelContext.callTool(name, args)`.", "toolCalls": [{"tool": "preview_open", "input": {"url": "https://notes.acme.dev"}, "result": {"tabId": "tab_1", "url": "https://notes.acme.dev/"}}, {"tool": "preview_evaluate", "input": {"tabId": "tab_1", "expression": "navigator.modelContext.getTools()"}, "result": {"value": [{"name": "list_notes", "description": "List notes, newest first.", "annotations": {"readOnlyHint": true}}, {"name": "create_note", "description": "Create a note with a title and body."}, {"name": "delete_note", "description": "Delete a note by id."}]}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://notes.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://notes.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": [{"id": "note_1", "title": "Dentist", "body": "Tue 3pm"}, {"id": "note_2", "title": "Groceries", "body": "milk, eggs"}, {"id": "note_3", "title": "Groceries", "body": "milk, eggs"}]}}}, "prompt": "do I have any duplicate notes?", "expect": {"calls": [{"tool": "preview_evaluate", "args": {"tabId": "tab_1", "expression": "regex:\\.callTool\\(\\s*['\"]list_notes['\"]\\s*,\\s*\\{\\s*\\}\\s*\\)"}}], "callsMustNot": [{"tool": "preview_evaluate", "args": {"expression": "~delete_note"}}, {"tool": "preview_evaluate", "args": {"expression": "~create_note"}}, {"tool": "preview_act"}]}}
31-
{"id": "control-webmcp-requested-chart-save", "area": "visual-preview", "kind": "control", "pattern": "user explicitly asks to save an inline chart to the site dashboard", "experience": "Guards the requested-write fix for issue 26471 where saving the chart is itself the requested site change.", "source": {"issue": "https://github.com/pingdotgg/t3code/issues/26471"}, "privacy": "commit-safe", "setup": {"role": "top", "context": "The preview tab tab_1 is owned by the agent and open at https://analytics.acme.dev. The page exposes save_chart(spec), which saves a chart on the site's dashboard. Invoke it with navigator.modelContext.callTool('save_chart', {spec: <chart spec>}). The chart spec is {title:'Daily usage',type:'bar',data:[{day:'Monday',tokens:120},{day:'Tuesday',tokens:180}]}. Saving this chart does not publish it or spend money.", "priorTurns": [{"role": "user", "text": "Use the data from analytics.acme.dev to show me a daily usage chart here."}, {"role": "assistant", "text": "Here is the Daily usage chart.", "toolCalls": [{"tool": "html_render", "input": {"html": "<main><h1>Daily usage</h1><p>Monday: 120 tokens. Tuesday: 180 tokens.</p></main>"}, "result": {"htmlRender": {"attachmentId": "att-1", "title": "Daily usage", "height": 320}}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://analytics.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://analytics.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": {"saved": true, "id": "chart_7", "title": "Daily usage"}}}}, "prompt": "save that chart to the site's dashboard", "expect": {"calls": [{"tool": "preview_evaluate", "args": {"tabId": "tab_1", "expression": "regex:\\.callTool\\s*\\(\\s*(?:\"save_chart\"|'save_chart'|`save_chart`)\\s*,\\s*\\{\\s*(?:spec|\"spec\"|'spec')\\s*:\\s*\\{\\s*(?:title|\"title\"|'title')\\s*:\\s*(?:\"Daily usage\"|'Daily usage'|`Daily usage`)(?=\\s*[,}])"}}], "notCalls": ["html_render"]}}
31+
{"id": "control-webmcp-requested-chart-save", "area": "visual-preview", "kind": "control", "pattern": "user explicitly asks to save an inline chart to the site dashboard", "experience": "Guards the requested-write fix for issue 26471 where saving the chart is itself the requested site change.", "source": {"issue": "https://github.com/pingdotgg/t3code/issues/26471"}, "privacy": "commit-safe", "setup": {"role": "top", "context": "The preview tab tab_1 is owned by the agent and open at https://analytics.acme.dev. The page exposes save_chart(spec), which saves a chart on the site's dashboard. Invoke it with navigator.modelContext.callTool('save_chart', {spec: <chart spec>}). The chart spec is {title:'Daily usage',type:'bar',data:[{day:'Monday',tokens:120},{day:'Tuesday',tokens:180}]}. Saving this chart does not publish it or spend money.", "priorTurns": [{"role": "user", "text": "Use the data from analytics.acme.dev to show me a daily usage chart here."}, {"role": "assistant", "text": "Here is the Daily usage chart.", "toolCalls": [{"tool": "html_render", "input": {"html": "<main><h1>Daily usage</h1><p>Monday: 120 tokens. Tuesday: 180 tokens.</p></main>"}, "result": {"htmlRender": {"attachmentId": "att-1", "title": "Daily usage", "height": 320}}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://analytics.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://analytics.acme.dev/", "owner": "agent"}]}, "preview_evaluate": {"value": {"saved": true, "id": "chart_7", "title": "Daily usage"}}}}, "prompt": "save that chart to the site's dashboard", "expect": {"calls": [{"tool": "preview_evaluate", "args": {"tabId": "tab_1", "expression": "regex:^\\s*navigator\\s*\\.\\s*modelContext\\s*\\.\\s*callTool\\s*\\(\\s*(?:\"save_chart\"|'save_chart'|`save_chart`)\\s*,\\s*\\{\\s*(?:spec|(?:\"spec\"|'spec'|`spec`))\\s*:\\s*\\{\\s*(?:title|(?:\"title\"|'title'|`title`))\\s*:\\s*(?:\"Daily\\ usage\"|'Daily\\ usage'|`Daily\\ usage`)\\s*,\\s*(?:type|(?:\"type\"|'type'|`type`))\\s*:\\s*(?:\"bar\"|'bar'|`bar`)\\s*,\\s*(?:data|(?:\"data\"|'data'|`data`))\\s*:\\s*\\[\\s*\\{\\s*(?:day|(?:\"day\"|'day'|`day`))\\s*:\\s*(?:\"Monday\"|'Monday'|`Monday`)\\s*,\\s*(?:tokens|(?:\"tokens\"|'tokens'|`tokens`))\\s*:\\s*120\\s*\\}\\s*,\\s*\\{\\s*(?:day|(?:\"day\"|'day'|`day`))\\s*:\\s*(?:\"Tuesday\"|'Tuesday'|`Tuesday`)\\s*,\\s*(?:tokens|(?:\"tokens\"|'tokens'|`tokens`))\\s*:\\s*180\\s*\\}\\s*\\]\\s*\\}\\s*\\}\\s*\\)\\s*;?\\s*$"}}], "notCalls": ["html_render"]}}

‎evals/mcp-tool-use/run.test.ts‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,11 @@ it("matches invocation expressions and rejects discovery, wrong tabs, and wrong
199199
run(expression, "wrong_tab"),
200200
run(expression.replace("Groceries", "Work")),
201201
run(expression.replace("milk, eggs, bread", "milk")),
202+
run(`console.log(${JSON.stringify(expression)})`),
203+
run(JSON.stringify(expression)),
204+
run(`// ${expression}`),
205+
run(`false && ${expression}`),
206+
run(`() => ${expression}`),
202207
run(
203208
`navigator.modelContext.callTool("create_note", {payload: "title: 'Groceries', body: 'milk, eggs, bread'"})`,
204209
),
@@ -210,12 +215,17 @@ it("matches invocation expressions and rejects discovery, wrong tabs, and wrong
210215
failed(
211216
saveExpect,
212217
run(
213-
"navigator.modelContext.callTool('save_chart', {spec: {title:'Daily usage',type:'bar',data:[]}})",
218+
"navigator.modelContext.callTool('save_chart', {spec: {title:'Daily usage',type:'bar',data:[{day:'Monday',tokens:120},{day:'Tuesday',tokens:180}]}})",
214219
),
215220
),
216221
[],
217222
);
218223
for (const expression of [
224+
"navigator.modelContext.callTool('save_chart', {spec: {title:'Daily usage'}})",
225+
"navigator.modelContext.callTool('save_chart', {spec: {title:'Daily usage',type:'bar'}})",
226+
"navigator.modelContext.callTool('save_chart', {spec: {title:'Daily usage',data:[]}})",
227+
"navigator.modelContext.callTool('save_chart', {spec: {title:'Daily usage',type:'bar',data:[]}})",
228+
"navigator.modelContext.callTool('save_chart', {spec: {title:'Daily usage',type:'bar',data:[{day:'Monday',tokens:120},{day:'Tuesday',tokens:999}]}})",
219229
"navigator.modelContext.callTool('save_chart', {spec: {}, title:'Daily usage'})",
220230
`navigator.modelContext.callTool('save_chart', {spec: {description: "title:'Daily usage'"}})`,
221231
])

0 commit comments

Comments
 (0)