@@ -826,6 +826,8 @@ type Sample = {
826826 readonly shell : Array < ShellEntry > ;
827827 readonly shellUsage : Usage ;
828828 readonly linked : Set < string > ;
829+ /** The branch PR the agent opened in this sample, which links alone do not prove. */
830+ openedPullRequest : number | undefined ;
829831 readonly codeStore : Record < string , unknown > ;
830832 nextTask : number ;
831833 /** Threads the agent launched in this sample, by id, so reading one shows its own brief. */
@@ -875,6 +877,10 @@ export const makeSample = (testCase: McpToolUseCase, base: Base): Sample => {
875877 shell,
876878 shellUsage : emptyUsage ( ) ,
877879 linked : new Set ( [ ...urls , ...shell . flatMap ( createdPullRequests ) ] ) ,
880+ openedPullRequest : shell
881+ . flatMap ( createdPullRequests )
882+ . map ( ( url ) => Number ( / \/ p u l l \/ ( \d + ) / . exec ( url ) ?. [ 1 ] ) )
883+ . find ( ( number ) => number > 0 ) ,
878884 codeStore : { } ,
879885 nextTask : Math . max ( 0 , ...taskNumbers ) + 1 ,
880886 threads : new Map ( ) ,
@@ -1117,6 +1123,9 @@ const branchPullRequest = (sample: Sample) => {
11171123 const prs = worldPullRequests ( sample ) ;
11181124 const own = prs . find ( ( pr ) => pr . headRefName === branch ) ;
11191125 if ( own !== undefined ) return { number : Number ( own . number ) , created : false , pr : own } ;
1126+ if ( sample . openedPullRequest !== undefined ) {
1127+ return { number : sample . openedPullRequest , created : false , pr : undefined } ;
1128+ }
11201129 const mocked = ( sample . testCase . setup . shellMocks ?? [ ] ) . find ( ( mock ) =>
11211130 / \b ( p r | m r ) c r e a t e \b / . test ( mock . match ) ,
11221131 ) ;
@@ -1226,6 +1235,7 @@ export const t3Default = (
12261235 const { number, created } = branchPullRequest ( sample ) ;
12271236 const target = pullRequest ( sample . repo , { number } ) ;
12281237 sample . linked . add ( target . url ) ;
1238+ if ( created ) sample . openedPullRequest = number ;
12291239 return { ...target , created } ;
12301240 }
12311241 case "pull_request_read" : {
@@ -1753,6 +1763,8 @@ export const answerToolCall = Effect.fn("evaluateMcpToolUse.answerToolCall")(fun
17531763 if ( name === "request_secret" ) savedSecretFile ( sample , mock ) ;
17541764 if ( name === "pull_request_create" && isRecord ( mock ) && typeof mock . url === "string" ) {
17551765 sample . linked . add ( mock . url ) ;
1766+ const number = Number ( mock . number ) ;
1767+ if ( number > 0 ) sample . openedPullRequest = number ;
17561768 }
17571769 return toolResult (
17581770 call . id ,
@@ -1768,13 +1780,7 @@ export const answerToolCall = Effect.fn("evaluateMcpToolUse.answerToolCall")(fun
17681780 }
17691781 if ( BRANCH_PULL_REQUEST_TOOLS . has ( name ) && input . number === undefined ) {
17701782 // Until the agent opens it, the PR a create would open does not exist.
1771- const own = branchPullRequest ( sample ) ;
1772- const mocked = testCase . setup . mocks ?. pull_request_create ;
1773- const opened = [
1774- pullRequest ( sample . repo , { number : own . number } ) . url ,
1775- isRecord ( mocked ) ? mocked . url : undefined ,
1776- ] . some ( ( url ) => typeof url === "string" && sample . linked . has ( url ) ) ;
1777- if ( own . created && ! opened ) return failed ( BRANCH_HAS_NO_PR_ERROR ) ;
1783+ if ( branchPullRequest ( sample ) . created ) return failed ( BRANCH_HAS_NO_PR_ERROR ) ;
17781784 }
17791785 const value = t3Default ( sample , name , input ) ;
17801786 if ( name === "request_secret" ) savedSecretFile ( sample , value ) ;
0 commit comments