Skip to content

Commit c7cc38e

Browse files
fix(evals): grade dot segments and basename case as the client does
The resolver keeps ./ and ../ segments, which the file system resolves, and the client opens a unique case-insensitive basename match. The grader counted both as broken. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 8a89941 commit c7cc38e

2 files changed

Lines changed: 18 additions & 3 deletions

File tree

‎evals/mcp-tool-use/file-link-grading.test.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ it("grades pills the way chat resolves them against the thread cwd", () => {
1818
"`packages/server/src/cloud/http.ts:6` and `cloud/http.ts:6`",
1919
"`ws.ts:3`, `index.ts:5`, `src/index.ts`, `src/worker.ts`",
2020
"[worker.ts](/home/user/work/worker/src/worker.ts)",
21-
"[http](cloud/http.ts#L4), [ws](./ws.ts)",
21+
"[http](cloud/http.ts#L4), [ws](./ws.ts), `./packages/server/src/ws.ts`, `WS.ts:3`",
2222
"[`src/ws.ts`](/home/user/app/packages/server/src/ws.ts), [`src/index.ts` boots it](/home/user/app/packages/server/src/ws.ts)",
2323
"`/usr/bin/node`, `git rebase --onto`",
2424
].join("\n");
@@ -33,6 +33,8 @@ it("grades pills the way chat resolves them against the thread cwd", () => {
3333
"ws.ts:3": true,
3434
"cloud/http.ts#L4": true,
3535
"./ws.ts": false,
36+
"./packages/server/src/ws.ts": true,
37+
"WS.ts:3": true,
3638
"index.ts:5": false,
3739
"src/index.ts": false,
3840
"/home/user/app/packages/server/src/ws.ts": true,

‎evals/mcp-tool-use/file-link-grading.ts‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
// @effect-diagnostics nodeBuiltinImport:off
2+
import * as NodePath from "node:path";
13
import * as Schema from "effect/Schema";
24
import { splitFilePathPosition } from "@t3tools/shared/fileLinks";
35
import {
@@ -19,6 +21,17 @@ export const FileLinksExpect = Schema.Struct({
1921
minWorking: Schema.Number,
2022
});
2123

24+
/** The client opens an exact basename match, or a case-insensitive one when only one exists. */
25+
const uniqueBasename = (name: string, entries: ReadonlyArray<{ readonly path: string }>) => {
26+
const basenames = entries.map((entry) => entry.path.split("/").at(-1) ?? "");
27+
const exact = basenames.filter((basename) => basename === name).length;
28+
return (
29+
exact === 1 ||
30+
(exact === 0 &&
31+
basenames.filter((basename) => basename.toLowerCase() === name.toLowerCase()).length === 1)
32+
);
33+
};
34+
2235
/**
2336
* Chat turns every file-shaped markdown link and inline code span into a pill resolved against
2437
* the thread's cwd. A pill is broken when it lands on no world file or directory and the client's
@@ -47,7 +60,7 @@ export const fileLinkPills = (
4760
return [...new Set(sources)].flatMap((source) => {
4861
const target = resolveMarkdownFileLinkTarget(source, cwd);
4962
if (target === null) return [];
50-
const path = splitFilePathPosition(target).path.replace(/\/+$/, "");
63+
const path = NodePath.posix.normalize(splitFilePathPosition(target).path).replace(/\/+$/, "");
5164
// System paths such as /usr/bin/node are not what these cases test.
5265
if (!/^\/(?:home|Users)\//.test(path)) return [];
5366
const relative = parseMarkdownFileLink(source)?.path ?? path;
@@ -61,7 +74,7 @@ export const fileLinkPills = (
6174
(!relative.startsWith("/") &&
6275
// The client opens the first of several same-named files, which may be the wrong one.
6376
(needsWorkspaceBasenameLookup(relative)
64-
? entries.filter((entry) => entry.path.split("/").at(-1) === relative).length === 1
77+
? uniqueBasename(relative, entries)
6578
: pickWorkspaceLinkMatch(relative, cwd, entries) !== null));
6679
return [{ source, path, works }];
6780
});

0 commit comments

Comments
 (0)