@@ -19,6 +19,7 @@ import {
1919 ThreadId ,
2020 type ModelSelection ,
2121 type OrchestrationV2AppThread ,
22+ type OrchestrationV2ProviderFailure ,
2223 type OrchestrationV2ProviderThread ,
2324 type OrchestrationV2ProviderTurn ,
2425 type ProviderReplayEntry ,
@@ -2965,7 +2966,11 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => {
29652966 ) ;
29662967 } ) . pipe ( Effect . scoped , Effect . provide ( TestClock . layer ( ) ) ) ,
29672968 ) ;
2968- const reconnected = ( active : Record < string , unknown > , ended ?: "succeeded" ) => [
2969+ const reconnected = (
2970+ active : Record < string , unknown > ,
2971+ ended ?: "succeeded" ,
2972+ assistant : Record < string , unknown > = { } ,
2973+ ) => [
29692974 { type : "runtime_exit" , status : "success" } as const ,
29702975 out ( "event.subscribe" ) ,
29712976 out ( "session.active" ) ,
@@ -3000,6 +3005,7 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => {
30003005 } ,
30013006 ] ,
30023007 finish : "stop" ,
3008+ ...assistant ,
30033009 } ,
30043010 {
30053011 id : "msg_0eb735d41001NJee1EvVePJAK5" ,
@@ -3890,6 +3896,263 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => {
38903896 } ) . pipe ( Effect . scoped ) ,
38913897 ) ;
38923898
3899+ const ASSISTANT = "msg_0eb735d5b001oAFVeY5jz3WD4Z" ;
3900+ const rateLimited = {
3901+ type : "provider.rate-limit" ,
3902+ message : "Rate limit exceeded: free-models-per-min." ,
3903+ status : 429 ,
3904+ } ;
3905+ /**
3906+ * OpenCode numbers the request it retries with, so its first retry is
3907+ * attempt 2, and stamps `at` from the clock behind the event's `created` (1 here).
3908+ */
3909+ const retryScheduled = ( attempt : number , delayMs : number ) =>
3910+ event ( "session.retry.scheduled" , {
3911+ sessionID : SESSION ,
3912+ assistantMessageID : ASSISTANT ,
3913+ attempt,
3914+ at : 1 + delayMs ,
3915+ error : rateLimited ,
3916+ } ) ;
3917+ const retrying = {
3918+ class : "provider_error" ,
3919+ message : rateLimited . message ,
3920+ code : rateLimited . type ,
3921+ retryable : true ,
3922+ } satisfies OrchestrationV2ProviderFailure ;
3923+ const firstRetry = { attempt : 1 , maxAttempts : null , retryDelayMs : 2_000 } ;
3924+ const errorRows = ( collected : ReadonlyArray < ProviderAdapter . ProviderAdapterV2Event > ) =>
3925+ collected . flatMap ( ( event ) =>
3926+ event . type === "turn_item.updated" && event . turnItem . type === "error" ? [ event . turnItem ] : [ ] ,
3927+ ) ;
3928+ const runRetryTurn = ( entries : ReadonlyArray < ProviderReplayEntry > , gate ?: ProviderReplayGate ) =>
3929+ Effect . gen ( function * ( ) {
3930+ const { runtime, thread } = yield * resumed (
3931+ [
3932+ out ( "session.prompt" , { sessionID : SESSION , text : "<any>" } ) ,
3933+ promptAccepted ,
3934+ event ( "session.execution.started" , { sessionID : SESSION } ) ,
3935+ ...entries ,
3936+ ] ,
3937+ gate === undefined ? undefined : { replayGate : gate } ,
3938+ ) ;
3939+ const retryShown = yield * Deferred . make < void > ( ) ;
3940+ const collected = yield * runtime . events . pipe (
3941+ Stream . tap ( ( event ) =>
3942+ event . type === "turn_item.updated" && event . turnItem . type === "error"
3943+ ? Deferred . succeed ( retryShown , undefined )
3944+ : Effect . void ,
3945+ ) ,
3946+ Stream . takeUntil ( ( event ) => event . type === "turn.terminal" ) ,
3947+ Stream . runCollect ,
3948+ Effect . forkScoped ,
3949+ ) ;
3950+ yield * runtime . startTurn ( turnInput ( thread ) ) ;
3951+ if ( gate !== undefined ) {
3952+ yield * Deferred . await ( retryShown ) ;
3953+ yield * TestClock . adjust ( "5 seconds" ) ;
3954+ gate . releaseAll ( ) ;
3955+ }
3956+ return yield * Fiber . join ( collected ) ;
3957+ } ) ;
3958+
3959+ it . effect . each ( [
3960+ {
3961+ name : "recovers a provider retry at the provider's first output" ,
3962+ after : [
3963+ event ( "session.text.started" , {
3964+ sessionID : SESSION ,
3965+ assistantMessageID : ASSISTANT ,
3966+ ordinal : 0 ,
3967+ } ) ,
3968+ event ( "session.text.ended" , {
3969+ sessionID : SESSION ,
3970+ assistantMessageID : ASSISTANT ,
3971+ ordinal : 0 ,
3972+ text : "DONE" ,
3973+ } ) ,
3974+ event ( "session.execution.succeeded" , { sessionID : SESSION } ) ,
3975+ ] ,
3976+ items : [ "error:running" , "error:completed" , "assistant_message:completed" ] ,
3977+ ended : "completed" as const ,
3978+ title : "Provider recovered" ,
3979+ } ,
3980+ {
3981+ name : "stops a provider retry when the user stops the turn" ,
3982+ after : [ event ( "session.execution.interrupted" , { sessionID : SESSION , reason : "user" } ) ] ,
3983+ items : [ "error:running" , "error:interrupted" ] ,
3984+ ended : "interrupted" as const ,
3985+ title : "Provider retry stopped" ,
3986+ } ,
3987+ ] ) ( "$name" , ( { after, items, ended, title } ) =>
3988+ Effect . gen ( function * ( ) {
3989+ const seen = yield * runRetryTurn ( [ retryScheduled ( 2 , 2_000 ) , ...after ] ) ;
3990+ assert . deepEqual ( turnItems ( seen ) , items ) ;
3991+ const rows = errorRows ( seen ) ;
3992+ assert . deepEqual (
3993+ rows . map ( ( { id, title, failure, retry } ) => ( { id, title, failure, retry } ) ) ,
3994+ [
3995+ { id : rows [ 0 ] ! . id , title : "Provider retry" , failure : retrying , retry : firstRetry } ,
3996+ { id : rows [ 0 ] ! . id , title, failure : retrying , retry : firstRetry } ,
3997+ ] ,
3998+ ) ;
3999+ assert . deepInclude ( seen . at ( - 1 ) , { type : "turn.terminal" , status : ended , failure : null } ) ;
4000+ } ) . pipe ( Effect . scoped ) ,
4001+ ) ;
4002+
4003+ it . effect . each ( [
4004+ { name : "restores retry pending at reconnect" , pending : true , streamed : false } ,
4005+ { name : "settles retry recovered during drop" , pending : false , streamed : false } ,
4006+ { name : "keeps retry pending when only streamed at reconnect" , pending : true , streamed : true } ,
4007+ ] ) ( "$name" , ( { pending, streamed } ) =>
4008+ Effect . gen ( function * ( ) {
4009+ const gate = makeProviderReplayGate ( [ "finish" ] ) ;
4010+ const { runtime, thread } = yield * resumed (
4011+ [
4012+ out ( "session.prompt" , { sessionID : SESSION , id : PROMPT_ID , text : "<any>" } ) ,
4013+ promptAccepted ,
4014+ event ( "session.execution.started" , { sessionID : SESSION } ) ,
4015+ ...( pending && ! streamed ? [ ] : [ retryScheduled ( 2 , 2_000 ) ] ) ,
4016+ ...reconnected ( { [ SESSION ] : { type : "running" } } , undefined , {
4017+ id : ASSISTANT ,
4018+ finish : undefined ,
4019+ content : pending ? [ ] : [ { type : "text" , text : "Recovered during the drop." } ] ,
4020+ ...( pending && ! streamed
4021+ ? { retry : { attempt : 2 , at : 2_001 , error : rateLimited } }
4022+ : { } ) ,
4023+ ...( streamed ? { time : { created : 2 , streamed : 3 } } : { } ) ,
4024+ } ) ,
4025+ ...openRequests ( [ shellAsk . data ] ) ,
4026+ event ( "session.execution.succeeded" , { sessionID : SESSION } , "finish" ) ,
4027+ ] ,
4028+ { supervised : true , replayGate : gate } ,
4029+ ) ;
4030+ const events = yield * runtime . events . pipe (
4031+ Stream . takeUntil ( ( event ) => event . type === "turn.terminal" ) ,
4032+ Stream . runCollect ,
4033+ Effect . forkScoped ,
4034+ ) ;
4035+ yield * runtime . startTurn ( turnInput ( thread , bigPickle , "approval-required" ) ) ;
4036+ yield * Effect . promise ( ( ) => gate . waitForReached ( "finish" ) ) ;
4037+ gate . release ( "finish" ) ;
4038+ const collected = yield * Fiber . join ( events ) ;
4039+ // Reconciled requests follow backfill, before the terminal can settle the retry row.
4040+ const reconciled = collected . findIndex ( ( event ) => event . type === "runtime_request.updated" ) ;
4041+ assert . isAtLeast ( reconciled , 0 ) ;
4042+ const seen = collected . slice ( 0 , reconciled ) ;
4043+ const rows = errorRows ( seen ) ;
4044+ assert . deepEqual (
4045+ rows . map ( ( { status, title } ) => ( { status, title } ) ) ,
4046+ pending
4047+ ? [ { status : "running" , title : "Provider retry" } ]
4048+ : [
4049+ { status : "running" , title : "Provider retry" } ,
4050+ { status : "completed" , title : "Provider recovered" } ,
4051+ ] ,
4052+ ) ;
4053+ for ( const row of rows ) {
4054+ assert . deepEqual ( row . failure , retrying ) ;
4055+ assert . deepInclude ( row . retry , { attempt : 1 , maxAttempts : null } ) ;
4056+ assert . equal ( row . id , rows [ 0 ] ! . id ) ;
4057+ assert . equal ( row . ordinal , rows [ 0 ] ! . ordinal ) ;
4058+ assert . deepEqual ( row . startedAt , rows [ 0 ] ! . startedAt ) ;
4059+ }
4060+ if ( ! pending ) {
4061+ assert . isTrue (
4062+ seen . some (
4063+ ( event ) =>
4064+ event . type === "turn_item.updated" &&
4065+ event . turnItem . type === "assistant_message" &&
4066+ event . turnItem . text === "Recovered during the drop." ,
4067+ ) ,
4068+ ) ;
4069+ }
4070+ } ) . pipe ( Effect . scoped ) ,
4071+ ) ;
4072+
4073+ it . effect ( "replaces a provider retry with the error that failed the turn" , ( ) =>
4074+ Effect . gen ( function * ( ) {
4075+ const step = {
4076+ sessionID : SESSION ,
4077+ assistantMessageID : ASSISTANT ,
4078+ agent : "build" ,
4079+ model : { providerID : "opencode" , id : "big-pickle" , variant : "default" } ,
4080+ started : 1 ,
4081+ } ;
4082+ const gate = makeProviderReplayGate ( [ "second retry" ] ) ;
4083+ // OpenCode starts the step of each attempt that fails, so a step start is no recovery.
4084+ const seen = yield * runRetryTurn (
4085+ [
4086+ event ( "session.step.started" , step ) ,
4087+ retryScheduled ( 2 , 2_000 ) ,
4088+ event ( "session.step.started" , step , "second retry" ) ,
4089+ retryScheduled ( 3 , 4_000 ) ,
4090+ event ( "session.step.started" , step ) ,
4091+ event ( "session.step.failed" , {
4092+ sessionID : SESSION ,
4093+ assistantMessageID : ASSISTANT ,
4094+ error : rateLimited ,
4095+ } ) ,
4096+ event ( "session.execution.failed" , { sessionID : SESSION , error : rateLimited } ) ,
4097+ ] ,
4098+ gate ,
4099+ ) ;
4100+ const failed = { ...retrying , class : "usage_limit" as const , retryable : null } ;
4101+ const lastRetry = { attempt : 2 , maxAttempts : null , retryDelayMs : 4_000 } ;
4102+ const rows = errorRows ( seen ) ;
4103+ assert . deepEqual (
4104+ rows . map ( ( { id, status, title, failure, retry } ) => ( {
4105+ id,
4106+ status,
4107+ title,
4108+ failure,
4109+ retry,
4110+ } ) ) ,
4111+ [
4112+ {
4113+ id : rows [ 0 ] ! . id ,
4114+ status : "running" ,
4115+ title : "Provider retry" ,
4116+ failure : retrying ,
4117+ retry : firstRetry ,
4118+ } ,
4119+ {
4120+ id : rows [ 0 ] ! . id ,
4121+ status : "running" ,
4122+ title : "Provider retry" ,
4123+ failure : retrying ,
4124+ retry : lastRetry ,
4125+ } ,
4126+ {
4127+ id : rows [ 0 ] ! . id ,
4128+ status : "failed" ,
4129+ title : "Usage limit reached" ,
4130+ failure : failed ,
4131+ retry : lastRetry ,
4132+ } ,
4133+ ] ,
4134+ ) ;
4135+ const { startedAt, ordinal } = rows [ 0 ] ! ;
4136+ assert . ok ( startedAt ) ;
4137+ assert . deepEqual (
4138+ rows . map ( ( row ) => ( { startedAt : row . startedAt , ordinal : row . ordinal } ) ) ,
4139+ rows . map ( ( ) => ( { startedAt, ordinal } ) ) ,
4140+ ) ;
4141+ assert . isAtLeast (
4142+ DateTime . toEpochMillis ( rows [ 2 ] ! . updatedAt ) - DateTime . toEpochMillis ( startedAt ) ,
4143+ 5_000 ,
4144+ ) ;
4145+ assert . deepInclude ( seen . at ( - 1 ) , {
4146+ type : "turn.terminal" ,
4147+ status : "failed" ,
4148+ failure : failed ,
4149+ failureItemOrdinal : ordinal ,
4150+ retry : lastRetry ,
4151+ retryStartedAt : startedAt ,
4152+ } ) ;
4153+ } ) . pipe ( Effect . scoped ) ,
4154+ ) ;
4155+
38934156 it . effect ( "reports a model's input limit as its context window" , ( ) =>
38944157 Effect . gen ( function * ( ) {
38954158 const runtime = yield * openCode2ReplayRuntime ( [ ...opening ] ) ;
0 commit comments