Skip to content

Commit 7ec0192

Browse files
fix(evals): a mocked pull_request_create opens the PR it names
Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent c5ff1d0 commit 7ec0192

2 files changed

Lines changed: 27 additions & 3 deletions

File tree

‎evals/mcp-tool-use/run.test.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -878,6 +878,24 @@ it.effect("reads the branch's PR only once the agent has opened it", () =>
878878
assert.isTrue((yield* call("a", "pull_request_read", {})).is_error);
879879
yield* call("b", "pull_request_create", { title: "Feature", body: "Body" });
880880
assert.isNotTrue((yield* call("c", "pull_request_read", {})).is_error);
881+
882+
// A case that mocks the create still opens the PR it names.
883+
const url = "https://github.com/acme/app/pull/124";
884+
const mocks = { pull_request_create: { number: 124, url, created: true } };
885+
const mockedSample = makeSample(
886+
{ ...childCase, setup: { ...childCase.setup, world, mocks } },
887+
base,
888+
);
889+
mockedSample.tools.push(...sample.tools);
890+
const mockedCall = (id: string, name: string) =>
891+
answerToolCall(mockedSample, config, {
892+
type: "tool_use",
893+
id,
894+
name: `mcp__t3-code__${name}`,
895+
input: name === "pull_request_create" ? { title: "Feature", body: "Body" } : {},
896+
});
897+
yield* mockedCall("d", "pull_request_create");
898+
assert.isNotTrue((yield* mockedCall("e", "pull_request_read")).is_error);
881899
}).pipe(Effect.provide(FetchHttpClient.layer)),
882900
);
883901

‎evals/mcp-tool-use/run.ts‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1751,6 +1751,9 @@ export const answerToolCall = Effect.fn("evaluateMcpToolUse.answerToolCall")(fun
17511751
// A pasted secret is saved without a card, so a mocked card outcome does not apply.
17521752
if (mock !== undefined && mock !== "default" && !pastedSecret(name, input)) {
17531753
if (name === "request_secret") savedSecretFile(sample, mock);
1754+
if (name === "pull_request_create" && isRecord(mock) && typeof mock.url === "string") {
1755+
sample.linked.add(mock.url);
1756+
}
17541757
return toolResult(
17551758
call.id,
17561759
name === "orchestrator_capabilities"
@@ -1766,9 +1769,12 @@ export const answerToolCall = Effect.fn("evaluateMcpToolUse.answerToolCall")(fun
17661769
if (BRANCH_PULL_REQUEST_TOOLS.has(name) && input.number === undefined) {
17671770
// Until the agent opens it, the PR a create would open does not exist.
17681771
const own = branchPullRequest(sample);
1769-
if (own.created && !sample.linked.has(pullRequest(sample.repo, { number: own.number }).url)) {
1770-
return failed(BRANCH_HAS_NO_PR_ERROR);
1771-
}
1772+
const mocked = testCase.setup.mocks?.pull_request_create;
1773+
const opened = [
1774+
pullRequest(sample.repo, { number: own.number }).url,
1775+
isRecord(mocked) ? mocked.url : undefined,
1776+
].some((url) => typeof url === "string" && sample.linked.has(url));
1777+
if (own.created && !opened) return failed(BRANCH_HAS_NO_PR_ERROR);
17721778
}
17731779
const value = t3Default(sample, name, input);
17741780
if (name === "request_secret") savedSecretFile(sample, value);

0 commit comments

Comments
 (0)