You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
{{ message }}
Repository navigation
Commit 1843c93
Browse filesBrowse the repository at this point in the historyBrowse files
Copy file name to clipboardExpand all lines: evals/mcp-tool-use/cases/file-links.ndjson
+1Lines changed: 1 addition & 0 deletions
Original file line number
Diff line number
Diff line change
@@ -2,3 +2,4 @@
2
2
{"id":"file-links-ambiguous-basename","area":"file-links","kind":"bad","pattern":"file references by bare basename that several packages share","experience":"Claude cited `index.ts:42`-style basenames. The client looks a bare basename up on click, but with several matches it cannot pick the right file. 272 such pills in the audit.","source":{"audit":"Audit of chat file pills across three fleet machines, 2026-11-10"},"privacy":"commit-safe","setup":{"role":"top","world":{"repo":"acme/app","branch":"main","files":{"packages/server/src/cloud/http.ts":"import { nextDelay } from './retry';\n\nexport async function relayFetch(url: string, init: RequestInit, attempt = 0): Promise<Response> {\n const res = await fetch(url, init);\n if (res.status >= 500 && attempt < 8) {\n await new Promise((r) => setTimeout(r, nextDelay(attempt)));\n return relayFetch(url, init, attempt + 1);\n }\n return res;\n}\n","packages/server/src/cloud/retry.ts":"const BASE_MS = 500;\n\nexport function nextDelay(attempt: number): number {\n const exp = BASE_MS * 2 ** attempt;\n return exp + Math.floor(Math.random() * BASE_MS);\n}\n","packages/server/src/cloud/retry.test.ts":"import { nextDelay } from './retry';\n\ntest('grows', () => expect(nextDelay(3)).toBeGreaterThan(nextDelay(1)));\n","packages/web/src/cloud/http.ts":"export const apiFetch = (path: string) => fetch(`/api${path}`);\n","packages/server/src/index.ts":"import { startWs } from './ws';\n\nexport async function main() {\n const port = Number(process.env.PORT ?? 3773);\n await startWs(port);\n}\n\nmain();\n","packages/web/src/index.ts":"import { createRoot } from 'react-dom/client';\nimport { App } from './App';\n\ncreateRoot(document.getElementById('root')!).render(<App />);\n","packages/server/src/ws.ts":"import { WebSocketServer } from 'ws';\n\nexport function startWs(port: number) {\n const wss = new WebSocketServer({ port });\n wss.on('connection', (socket) => socket.send('hello'));\n return wss;\n}\n","package.json":"{ \"name\": \"acme\", \"private\": true, \"workspaces\": [\"packages/*\"] }\n","packages/server/package.json":"{ \"name\": \"@acme/server\", \"main\": \"src/index.ts\", \"scripts\": { \"start\": \"node src/index.ts\" } }\n","packages/web/package.json":"{ \"name\": \"@acme/web\" }\n"}}},"prompt":"How does the server process boot and start the WebSocket server? Walk me through it with file references.","expect":{"fileLinks":{"minWorking":2}},"maxSteps":8}
3
3
{"id":"file-links-other-checkout","area":"file-links","kind":"bad","pattern":"relative file references to a checkout outside the thread cwd","experience":"In a scratch thread the agent cloned a repo elsewhere and then cited `src/worker.ts` and `src/api.ts`. Chat resolved them against the scratch directory, so every pill was broken.","source":{"audit":"Audit of chat file pills across three fleet machines, 2026-11-10 (scratch threads on nucbox-1, 2026-10-19)"},"privacy":"commit-safe","setup":{"role":"top","world":{"repo":"acme/app","branch":"main","files":{"/home/user/work/worker/src/worker.ts":"import { take, ack, nack } from './queue';\n\nexport async function run() {\n for (;;) {\n const job = await take();\n try { await job.handle(); await ack(job); }\n catch { await nack(job, { delaySeconds: 30 * job.attempts }); }\n }\n}\n","/home/user/work/worker/src/queue.ts":"export const MAX_ATTEMPTS = 5;\nexport async function nack(job: Job, opts: { delaySeconds: number }) {\n if (job.attempts >= MAX_ATTEMPTS) return deadLetter(job);\n return requeue(job, opts.delaySeconds);\n}\n","/home/user/work/worker/src/api.ts":"export function enqueue(body: unknown) { return requeue({ body, attempts: 0 }, 0); }\n","/home/user/work/worker/package.json":"{ \"name\": \"@acme/worker\" }\n"}},"priorTurns":[{"role":"user","text":"Clone acme/worker into ~/work/worker so we can look at it."},{"role":"assistant","text":"Cloning it.","toolCalls":[{"tool":"Bash","input":{"command":"git clone [email protected]:acme/worker.git ~/work/worker && ls ~/work/worker/src"},"result":"Cloning into '/home/user/work/worker'...\napi.ts\nqueue.ts\nworker.ts"}]},{"role":"assistant","text":"Cloned acme/worker into /home/user/work/worker. Its source is in src/: api.ts, queue.ts and worker.ts."}]},"prompt":"How does the worker retry failed jobs, and where would I change the max attempts? Point me at the files.","expect":{"fileLinks":{"minWorking":2}},"maxSteps":8}
4
4
{"id":"file-links-no-files-control","area":"file-links","kind":"control","pattern":"a general question that needs no file references","experience":"Guards against an instruction change that makes agents attach file links to answers that are not about files.","source":{"audit":"Audit of chat file pills across three fleet machines, 2026-11-10"},"privacy":"commit-safe","setup":{"role":"top","world":{"repo":"acme/app","branch":"main","files":{"packages/server/src/cloud/http.ts":"import { nextDelay } from './retry';\n\nexport async function relayFetch(url: string, init: RequestInit, attempt = 0): Promise<Response> {\n const res = await fetch(url, init);\n if (res.status >= 500 && attempt < 8) {\n await new Promise((r) => setTimeout(r, nextDelay(attempt)));\n return relayFetch(url, init, attempt + 1);\n }\n return res;\n}\n","packages/server/src/cloud/retry.ts":"const BASE_MS = 500;\n\nexport function nextDelay(attempt: number): number {\n const exp = BASE_MS * 2 ** attempt;\n return exp + Math.floor(Math.random() * BASE_MS);\n}\n","packages/server/src/cloud/retry.test.ts":"import { nextDelay } from './retry';\n\ntest('grows', () => expect(nextDelay(3)).toBeGreaterThan(nextDelay(1)));\n","packages/web/src/cloud/http.ts":"export const apiFetch = (path: string) => fetch(`/api${path}`);\n","packages/server/src/index.ts":"import { startWs } from './ws';\n\nexport async function main() {\n const port = Number(process.env.PORT ?? 3773);\n await startWs(port);\n}\n\nmain();\n","packages/web/src/index.ts":"import { createRoot } from 'react-dom/client';\nimport { App } from './App';\n\ncreateRoot(document.getElementById('root')!).render(<App />);\n","packages/server/src/ws.ts":"import { WebSocketServer } from 'ws';\n\nexport function startWs(port: number) {\n const wss = new WebSocketServer({ port });\n wss.on('connection', (socket) => socket.send('hello'));\n return wss;\n}\n","package.json":"{ \"name\": \"acme\", \"private\": true, \"workspaces\": [\"packages/*\"] }\n","packages/server/package.json":"{ \"name\": \"@acme/server\", \"main\": \"src/index.ts\", \"scripts\": { \"start\": \"node src/index.ts\" } }\n","packages/web/package.json":"{ \"name\": \"@acme/web\" }\n"}}},"prompt":"Quick git question, no need to look at the repo: what does `git rebase --onto main feature~3 feature` do?","expect":{"fileLinks":{"minWorking":0},"notCalls":["Bash"]},"maxSteps":4}
5
+
{"id":"scratch-file-markdown-link","area":"file-links","kind":"bad","pattern":"scratch file path shortened instead of written in full","experience":"A Claude scratch file was mentioned as shortened inline code such as `/tmp/claude-1000/...`, which no client can open.","privacy":"commit-safe","setup":{"role":"top","context":"You have just saved notes.md at /tmp/claude-1000/-home-user--t3-worktrees-app-t3-abc123/12345678-1234-1234-1234-123456789abc/scratchpad/notes.md. The write succeeded. No more tools are needed."},"prompt":"The notes are ready? Tell me where you saved them. Keep it brief.","expect":{"finalTextMust":["/tmp/claude-1000/-home-user--t3-worktrees-app-t3-abc123/12345678-1234-1234-1234-123456789abc/scratchpad/notes\\.md"],"textMustNot":["/tmp/claude-1000/[^\\s<>`)]*(?:\\.{3,}|…)","`\\[[^\\]`\\n]*\\]\\(/tmp/claude-1000/[^)`\\n]*\\)`"]},"maxSteps":2,"source":{"issue":"https://github.com/pingdotgg/t3code/issues/30757"}}
0 commit comments