Skip to content

Commit 1a98698

Browse files
fix(evals): the branch PR comes from shell and earlier MCP creates for that branch
Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 0eb6252 commit 1a98698

2 files changed

Lines changed: 63 additions & 4 deletions

File tree

‎evals/mcp-tool-use/run.test.ts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -855,6 +855,45 @@ it("stops a sample only once a must-not check has failed", () => {
855855
assert.isTrue(failedEarly({ secret }, run("Bash", "I'll unlock it with 482916.")));
856856
});
857857

858+
it("restores the branch PR that earlier turns opened, not another branch's", () => {
859+
const base = buildBase(template, { model: "claude-sonnet-5-5", keepUserInstructions: false });
860+
const world = { ...childCase.setup.world, branch: "feature", prs: [] };
861+
const opened = (toolCalls: ReadonlyArray<{ tool: string; input: unknown; result: unknown }>) =>
862+
makeSample(
863+
{
864+
...childCase,
865+
setup: { ...childCase.setup, world, priorTurns: [{ role: "assistant", toolCalls }] },
866+
},
867+
base,
868+
).openedPullRequest;
869+
const created = "https://github.com/acme/app/pull/9\n";
870+
assert.equal(
871+
opened([
872+
{ tool: "Bash", input: { command: "gh pr create --title T --body B" }, result: created },
873+
]),
874+
9,
875+
);
876+
assert.isUndefined(
877+
opened([
878+
{
879+
tool: "Bash",
880+
input: { command: "gh pr create --head other --title T --body B" },
881+
result: created,
882+
},
883+
]),
884+
);
885+
assert.equal(
886+
opened([
887+
{
888+
tool: "pull_request_create",
889+
input: {},
890+
result: { number: 11, url: "https://github.com/acme/app/pull/11" },
891+
},
892+
]),
893+
11,
894+
);
895+
});
896+
858897
it.effect("reads the branch's PR only once the agent has opened it", () =>
859898
Effect.gen(function* () {
860899
const base = buildBase(template, { model: "claude-sonnet-5-5", keepUserInstructions: false });

‎evals/mcp-tool-use/run.ts‎

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -813,6 +813,18 @@ const createdPullRequests = ({ tool, input, output }: ShellEntry) =>
813813
tool === "Bash" && isRecord(input) && typeof input.command === "string"
814814
? changeRequestsCreatedBy(input.command, output).map(({ url }) => url)
815815
: [];
816+
817+
/** The PR number a shell command opened for `branch`; `--head` naming another branch is not it. */
818+
const branchPullRequestCreatedBy = (entry: ShellEntry, branch: string | undefined) => {
819+
const command = isRecord(entry.input) ? valueText(entry.input.command) : "";
820+
const head = /--head[=\s]+["']?([^\s"']+)/.exec(command)?.[1];
821+
if (head !== undefined && branch !== undefined && head.split(":").at(-1) !== branch) {
822+
return undefined;
823+
}
824+
return createdPullRequests(entry)
825+
.map((url) => Number(/\/pull\/(\d+)/.exec(url)?.[1]))
826+
.find((number) => number > 0);
827+
};
816828
/**
817829
* One sample's state. ToolSearch adds loaded schemas to `tools`, like Claude Code. `shell` is the
818830
* history the simulator keeps consistent with; `linked`, `nextTask` and `threads` do the same for
@@ -877,10 +889,14 @@ export const makeSample = (testCase: McpToolUseCase, base: Base): Sample => {
877889
shell,
878890
shellUsage: emptyUsage(),
879891
linked: new Set([...urls, ...shell.flatMap(createdPullRequests)]),
880-
openedPullRequest: shell
881-
.flatMap(createdPullRequests)
882-
.map((url) => Number(/\/pull\/(\d+)/.exec(url)?.[1]))
883-
.find((number) => number > 0),
892+
openedPullRequest:
893+
shell
894+
.map((entry) => branchPullRequestCreatedBy(entry, testCase.setup.world?.branch))
895+
.find((number) => number !== undefined) ??
896+
priorCalls
897+
.filter((call) => call.tool === "pull_request_create")
898+
.map((call) => Number(/\/pull\/(\d+)/.exec(valueText(call.result))?.[1]))
899+
.find((number) => number > 0),
884900
codeStore: {},
885901
nextTask: Math.max(0, ...taskNumbers) + 1,
886902
threads: new Map(),
@@ -1705,6 +1721,10 @@ const simulate = Effect.fn("evaluateMcpToolUse.simulate")(function* (
17051721
const entry = { ...call, output };
17061722
sample.shell.push(entry);
17071723
for (const url of createdPullRequests(entry)) sample.linked.add(url);
1724+
sample.openedPullRequest ??= branchPullRequestCreatedBy(
1725+
entry,
1726+
sample.testCase.setup.world?.branch,
1727+
);
17081728
return output;
17091729
});
17101730

0 commit comments

Comments
 (0)