Skip to content

Commit a367d12

Browse files
committed
test(evals): match briefs from the creation tool field
1 parent 48e26dc commit a367d12

2 files changed

Lines changed: 18 additions & 13 deletions

File tree

‎evals/mcp-tool-use/run.test.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,12 @@ it("counts the new threads an agent asks for, one brief per thread, over both cr
101101
assert.deepEqual(failed(expect, run(batch("Fix reconnect, csv and admin"))), [label]);
102102
assert.deepEqual(failed(expect, run(batch("reconnect, csv, admin", "csv", "csv"))), [label]);
103103
assert.deepEqual(failed(expect, run(batch("reconnect", "csv", "admin", "docs"))), [label]);
104+
for (const call of [
105+
{ tool: "t3_thread_launch", input: { prompt: "csv" } },
106+
{ tool: "create_threads", input: { threads: [{ message: "csv" }] } },
107+
]) {
108+
assert.deepEqual(failed(expect, run(launch("reconnect"), launch("admin"), call)), [label]);
109+
}
104110
assert.deepEqual(
105111
failed(
106112
expect,

‎evals/mcp-tool-use/run.ts‎

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1386,30 +1386,29 @@ const matches = (expected: unknown, actual: unknown): boolean => {
13861386

13871387
/** Each new top-level thread request in a run; a failed call created nothing. */
13881388
const newThreadRequests = (calls: ReadonlyArray<Call>) =>
1389-
calls.flatMap((call): ReadonlyArray<unknown> => {
1390-
if (call.failed === true) return [];
1391-
if (call.tool === "t3_thread_launch") return [call.input];
1392-
if (call.tool !== "create_threads" || !isRecord(call.input)) return [];
1393-
return Array.isArray(call.input.threads) ? call.input.threads : [];
1389+
calls.flatMap((call): ReadonlyArray<{ request: unknown; brief: unknown }> => {
1390+
if (call.failed === true || !isRecord(call.input)) return [];
1391+
if (call.tool === "t3_thread_launch")
1392+
return [{ request: call.input, brief: call.input.message }];
1393+
if (call.tool !== "create_threads" || !Array.isArray(call.input.threads)) return [];
1394+
return call.input.threads.map((request: unknown) => ({
1395+
request,
1396+
brief: isRecord(request) ? request.prompt : undefined,
1397+
}));
13941398
});
13951399

13961400
/** Whether every expected value can take a different request it matches. */
13971401
const matchesDistinct = (
13981402
expected: ReadonlyArray<unknown>,
1399-
requests: ReadonlyArray<unknown>,
1403+
requests: ReturnType<typeof newThreadRequests>,
14001404
taken: ReadonlySet<number> = new Set(),
14011405
): boolean => {
14021406
const [first, ...rest] = expected;
14031407
if (expected.length === 0) return true;
14041408
return requests.some(
1405-
(request, i) =>
1409+
({ request, brief }, i) =>
14061410
!taken.has(i) &&
1407-
matches(
1408-
first,
1409-
typeof first === "string" && isRecord(request)
1410-
? (request.message ?? request.prompt)
1411-
: request,
1412-
) &&
1411+
matches(first, typeof first === "string" ? brief : request) &&
14131412
matchesDistinct(rest, requests, new Set([...taken, i])),
14141413
);
14151414
};

0 commit comments

Comments
 (0)