Skip to content

Commit a99935c

Browse files
fix(evals): preview media must exist at its literal path
Mobile opens PDFs, images, video and audio in a preview with no workspace lookup, so the grader no longer rescues them by suffix. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 1606486 commit a99935c

2 files changed

Lines changed: 18 additions & 0 deletions

File tree

‎evals/mcp-tool-use/file-link-grading.test.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ const files = [
99
"packages/server/src/index.ts",
1010
"packages/web/src/index.ts",
1111
"packages/server/src/ws.ts",
12+
"packages/server/docs/report.pdf",
1213
"/home/user/work/worker/src/worker.ts",
1314
];
1415

@@ -23,6 +24,7 @@ it("grades pills the way chat resolves them against the thread cwd", () => {
2324
"`/usr/bin/node`, `git rebase --onto`",
2425
"```md\n[example](docs/missing.md) and `src/missing.ts`\n```",
2526
"Write links as `[example](docs/missing.md)`.",
27+
"[report](docs/report.pdf)",
2628
].join("\n");
2729
const pills = Object.fromEntries(
2830
fileLinkPills(text, cwd, files).map((pill) => [pill.source, pill.works]),
@@ -37,6 +39,7 @@ it("grades pills the way chat resolves them against the thread cwd", () => {
3739
"./ws.ts": false,
3840
"./packages/server/src/ws.ts": true,
3941
"WS.ts:3": true,
42+
"docs/report.pdf": false,
4043
"index.ts:5": false,
4144
"src/index.ts": false,
4245
"/home/user/app/packages/server/src/ws.ts": true,

‎evals/mcp-tool-use/file-link-grading.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
import * as Schema from "effect/Schema";
22
import { splitFilePathPosition } from "@t3tools/shared/fileLinks";
3+
import {
4+
audioMimeTypeFromExtension,
5+
mediaMimeTypeFromExtension,
6+
} from "@t3tools/shared/mediaPreview";
37
import {
48
extractInlineCodeSpans,
59
extractMarkdownLinkHrefs,
@@ -29,6 +33,16 @@ const normalizeAbsolutePath = (path: string) => {
2933
return `/${segments.join("/")}`;
3034
};
3135

36+
/** Mobile opens these in a preview at the literal path, with no workspace lookup. */
37+
const opensAsPreview = (path: string) => {
38+
const extension = path.slice(path.lastIndexOf(".")).toLowerCase();
39+
return (
40+
extension === ".pdf" ||
41+
mediaMimeTypeFromExtension(extension) !== null ||
42+
audioMimeTypeFromExtension(extension) !== null
43+
);
44+
};
45+
3246
/** The client opens an exact basename match, or a case-insensitive one when only one exists. */
3347
const uniqueBasename = (name: string, entries: ReadonlyArray<{ readonly path: string }>) => {
3448
const basenames = entries.map((entry) => entry.path.split("/").at(-1) ?? "");
@@ -83,6 +97,7 @@ export const fileLinkPills = (
8397
const works =
8498
files.some((file) => file === path || file.startsWith(`${path}/`)) ||
8599
(!relative.startsWith("/") &&
100+
!opensAsPreview(path) &&
86101
// The client opens the first of several same-named files, which may be the wrong one.
87102
(needsWorkspaceBasenameLookup(relative)
88103
? uniqueBasename(relative, entries)

0 commit comments

Comments
 (0)