Skip to content

Commit c341be3

Browse files
fix(evals): ci_logs in the eval runner reads five logs like the service
Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent cc9b074 commit c341be3

3 files changed

Lines changed: 63 additions & 13 deletions

File tree

‎apps/server/src/pullRequest/PullRequestService.test.ts‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1293,7 +1293,12 @@ it.effect(
12931293
getCheckLogs: (input) =>
12941294
Effect.sync(() => {
12951295
asked.push(input.checks.map((entry) => entry.name));
1296-
return [];
1296+
return input.checks.map((entry) => ({
1297+
...entry,
1298+
log: "boom",
1299+
truncated: false,
1300+
detail: null,
1301+
}));
12971302
}),
12981303
}),
12991304
],
@@ -1313,10 +1318,19 @@ it.effect(
13131318
// The host is asked for five; the other three come back unread, by name.
13141319
assert.lengthOf(asked[0] ?? [], 5);
13151320
assert.deepStrictEqual(
1316-
many.map((log) => log.name),
1317-
["Shard 3", "Shard 4", "Shard 5"],
1321+
many.map((log) => [log.name, log.log]),
1322+
[
1323+
["Test", "boom"],
1324+
["E2E", "boom"],
1325+
["Shard 0", "boom"],
1326+
["Shard 1", "boom"],
1327+
["Shard 2", "boom"],
1328+
["Shard 3", null],
1329+
["Shard 4", null],
1330+
["Shard 5", null],
1331+
],
13181332
);
1319-
assert.include(many[0]?.detail ?? "", "Ask for this check by name");
1333+
assert.include(many.at(-1)?.detail ?? "", "Ask for this check by name");
13201334
}),
13211335
);
13221336

‎evals/mcp-tool-use/run.test.ts‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -716,6 +716,31 @@ it("lists and reads world PRs in their own state, with the diff when asked", ()
716716
}
717717
).checks.map((check) => check.name);
718718
assert.deepEqual(ciNames({ check: " test " }), ["Test"]);
719+
const shards = makeSample(
720+
{
721+
...childCase,
722+
setup: {
723+
...childCase.setup,
724+
world: {
725+
...childCase.setup.world,
726+
prs: [
727+
{
728+
number: 7,
729+
checks: Array.from({ length: 6 }, (_, index) => ({
730+
name: `Shard ${index}`,
731+
state: "fail",
732+
})),
733+
},
734+
],
735+
},
736+
},
737+
},
738+
base,
739+
);
740+
const logs = (
741+
t3Default(shards, "ci_logs", { number: 7 }) as { checks: Array<{ log: string | null }> }
742+
).checks.map((check) => check.log !== null);
743+
assert.deepEqual(logs, [true, true, true, true, true, false]);
719744
assert.deepEqual(ciNames({}), ["Lint"]);
720745
const read: Readonly<Record<string, unknown>> = t3Default(sample, "pull_request_read", {
721746
number: 5,

‎evals/mcp-tool-use/run.ts‎

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1151,6 +1151,9 @@ const branchPullRequest = (sample: Sample) => {
11511151
return { number, created: true, pr: undefined };
11521152
};
11531153

1154+
/** PullRequestService reads this many logs when no check is named. */
1155+
const DEFAULT_CHECK_LOGS = 5;
1156+
11541157
/** The checks `ci_logs` and `ci_rerun` act on, selected as PullRequestService does. */
11551158
const selectedWorldChecks = (sample: Sample, input: Readonly<Record<string, unknown>>) => {
11561159
const number = typeof input.number === "number" ? input.number : branchPullRequest(sample).number;
@@ -1382,19 +1385,27 @@ export const t3Default = (
13821385
};
13831386
case "ci_logs":
13841387
case "ci_rerun": {
1385-
const { number, checks } = selectedWorldChecks(sample, input);
1388+
const { number, checks, named } = selectedWorldChecks(sample, input);
13861389
return {
13871390
...pullRequest(sample.repo, { number }),
1388-
checks: checks.map((check) =>
1391+
checks: checks.map((check, index) =>
13891392
name === "ci_rerun"
13901393
? { name: valueText(check.name), rerun: true, detail: null }
1391-
: {
1392-
name: valueText(check.name),
1393-
url: null,
1394-
log: valueText(check.log ?? `Error: ${valueText(check.name)} failed`),
1395-
truncated: false,
1396-
detail: null,
1397-
},
1394+
: named === null && index >= DEFAULT_CHECK_LOGS
1395+
? {
1396+
name: valueText(check.name),
1397+
url: null,
1398+
log: null,
1399+
truncated: false,
1400+
detail: "Not read, to keep the reply short. Ask for this check by name.",
1401+
}
1402+
: {
1403+
name: valueText(check.name),
1404+
url: null,
1405+
log: valueText(check.log ?? `Error: ${valueText(check.name)} failed`),
1406+
truncated: false,
1407+
detail: null,
1408+
},
13981409
),
13991410
};
14001411
}

0 commit comments

Comments
 (0)