@@ -36,6 +36,7 @@ import * as Deferred from "effect/Deferred";
3636import * as Effect from "effect/Effect" ;
3737import * as Fiber from "effect/Fiber" ;
3838import * as Layer from "effect/Layer" ;
39+ import * as Logger from "effect/Logger" ;
3940import * as Option from "effect/Option" ;
4041import * as Ref from "effect/Ref" ;
4142import * as Stream from "effect/Stream" ;
@@ -4104,6 +4105,230 @@ it.effect("keeps completed runs completed when pull request refresh fails", () =
41044105 } ) ,
41054106) ;
41064107
4108+ it . effect (
4109+ "recovers ingestion failure through typed errors and defects with bounded backoff" ,
4110+ ( ) => {
4111+ const warnings : Array < unknown > = [ ] ;
4112+ const logger = Logger . make ( ( { message } ) => {
4113+ const text = Array . isArray ( message ) ? message [ 0 ] : message ;
4114+ if ( text === "orchestration V2 run failure finalization retry" ) warnings . push ( text ) ;
4115+ } ) ;
4116+ return Effect . gen ( function * ( ) {
4117+ const failIngestion = yield * Deferred . make < void > ( ) ;
4118+ const failedWrite = yield * Deferred . make < void > ( ) ;
4119+ const writeTimes : Array < number > = [ ] ;
4120+ let storageAvailable = false ;
4121+ let failedAt = yield * DateTime . now ;
4122+ const { finalEvents, observed, written } = yield * captureRootRunTermination ( {
4123+ key : "recovery-backoff" ,
4124+ shouldFinalizeRun : ( ) => Effect . succeed ( true ) ,
4125+ ingestNormalized : ( ) =>
4126+ Deferred . await ( failIngestion ) . pipe ( Effect . andThen ( Effect . die ( "ingestion failed" ) ) ) ,
4127+ beforeTerminalWrite : ( ) =>
4128+ Effect . gen ( function * ( ) {
4129+ writeTimes . push ( DateTime . toEpochMillis ( yield * DateTime . now ) ) ;
4130+ if ( storageAvailable ) return true ;
4131+ yield * Deferred . succeed ( failedWrite , undefined ) ;
4132+ const error = new EventSink . EventSinkWriteError ( { eventCount : 5 } ) ;
4133+ return yield * writeTimes . length % 2 === 1 ? Effect . fail ( error ) : Effect . die ( error ) ;
4134+ } ) ,
4135+ afterStart : ( ) =>
4136+ Effect . gen ( function * ( ) {
4137+ // Separate startup, ingestion failure, and eventual storage recovery.
4138+ yield * TestClock . adjust ( "5 seconds" ) ;
4139+ failedAt = yield * DateTime . now ;
4140+ yield * Deferred . succeed ( failIngestion , undefined ) ;
4141+ yield * Deferred . await ( failedWrite ) ;
4142+ yield * TestClock . adjust ( "999 millis" ) ;
4143+ assert . lengthOf ( writeTimes , 1 ) ;
4144+ yield * TestClock . adjust ( "1 millis" ) ;
4145+ assert . lengthOf ( writeTimes , 2 ) ;
4146+ assert . lengthOf ( warnings , 1 ) ;
4147+ yield * TestClock . adjust ( "598999 millis" ) ;
4148+ const offsets = writeTimes . map ( ( time ) => time - DateTime . toEpochMillis ( failedAt ) ) ;
4149+ assert . deepEqual ( offsets , [
4150+ 0 ,
4151+ 1000 ,
4152+ 3000 ,
4153+ 7000 ,
4154+ 15000 ,
4155+ 31000 ,
4156+ ...Array . from ( { length : 18 } , ( _ , index ) => 61000 + index * 30000 ) ,
4157+ ] ) ;
4158+ assert . lengthOf ( warnings , 1 ) ;
4159+ yield * TestClock . adjust ( "1001 millis" ) ;
4160+ assert . lengthOf ( warnings , 2 ) ;
4161+ storageAvailable = true ;
4162+ yield * TestClock . adjust ( "30 seconds" ) ;
4163+ } ) ,
4164+ } ) ;
4165+ assert . deepEqual ( observed , [ "run:failed" , "pull-requests-refreshed" ] ) ;
4166+ assert . lengthOf ( warnings , 2 ) ;
4167+ assert . sameMembers (
4168+ finalEvents . map ( ( event ) => event . type ) ,
4169+ [
4170+ "run-attempt.updated" ,
4171+ "turn-item.updated" ,
4172+ "run.updated" ,
4173+ "node.updated" ,
4174+ "provider-thread.updated" ,
4175+ ] ,
4176+ ) ;
4177+ for ( const event of finalEvents ) {
4178+ assert . deepEqual ( event . occurredAt , failedAt ) ;
4179+ if (
4180+ event . type === "run.updated" ||
4181+ event . type === "run-attempt.updated" ||
4182+ event . type === "node.updated"
4183+ ) {
4184+ assert . equal ( event . payload . status , "failed" ) ;
4185+ assert . deepEqual ( event . payload . completedAt , failedAt ) ;
4186+ }
4187+ if ( event . type === "provider-thread.updated" )
4188+ assert . deepEqual ( event . payload . updatedAt , failedAt ) ;
4189+ }
4190+ assert . lengthOf ( written , 1 ) ;
4191+ assert . equal ( written [ 0 ] ?. type , "error" ) ;
4192+ assert . deepEqual ( written [ 0 ] ?. completedAt , failedAt ) ;
4193+ if ( written [ 0 ] ?. type === "error" )
4194+ assert . equal ( written [ 0 ] . failure . message , "Provider turn failed." ) ;
4195+ } ) . pipe ( Effect . provide ( Logger . layer ( [ logger ] , { mergeWithExisting : false } ) ) ) ;
4196+ } ,
4197+ ) ;
4198+
4199+ it . effect ( "stops recovery when the atomic gate rejects a superseded attempt" , ( ) =>
4200+ Effect . gen ( function * ( ) {
4201+ const key = "recovery-successor" ;
4202+ const ids = backgroundScenarioIds ( key ) ;
4203+ const failedWrite = yield * Deferred . make < void > ( ) ;
4204+ let activeAttemptId = ids . attemptId ;
4205+ let writes = 0 ;
4206+ const { finalEvents, observed, written } = yield * captureRootRunTermination ( {
4207+ key,
4208+ shouldFinalizeRun : ( ) => Effect . succeed ( true ) ,
4209+ ingestNormalized : ( ) => Effect . die ( "ingestion failed" ) ,
4210+ beforeTerminalWrite : ( request ) =>
4211+ Effect . gen ( function * ( ) {
4212+ writes ++ ;
4213+ assert . equal ( request . runId , ids . runId ) ;
4214+ assert . equal ( request . activeAttemptId , ids . attemptId ) ;
4215+ assert . equal ( request . expectedStatus , "running" ) ;
4216+ if ( writes > 1 ) return request . activeAttemptId === activeAttemptId ;
4217+ yield * Deferred . succeed ( failedWrite , undefined ) ;
4218+ return yield * new EventSink . EventSinkWriteError ( { eventCount : 5 } ) ;
4219+ } ) ,
4220+ afterStart : ( ) =>
4221+ Effect . gen ( function * ( ) {
4222+ yield * Deferred . await ( failedWrite ) ;
4223+ activeAttemptId = RunAttemptId . make ( "attempt:successor" ) ;
4224+ yield * TestClock . adjust ( "1 second" ) ;
4225+ assert . equal ( writes , 2 ) ;
4226+ yield * TestClock . adjust ( "10 minutes" ) ;
4227+ assert . equal ( writes , 2 ) ;
4228+ } ) ,
4229+ } ) ;
4230+ assert . deepEqual ( finalEvents , [ ] ) ;
4231+ assert . deepEqual ( written , [ ] ) ;
4232+ assert . deepEqual ( observed , [ ] ) ;
4233+ } ) ,
4234+ ) ;
4235+
4236+ it . effect . each ( [ true , false ] ) (
4237+ "keeps recovery alive after startup cancels its consumer (startup terminal persisted: %s)" ,
4238+ ( startupPersisted ) =>
4239+ Effect . gen ( function * ( ) {
4240+ const failedWrite = yield * Deferred . make < void > ( ) ;
4241+ let writes = 0 ;
4242+ let status : OrchestrationV2Run [ "status" ] = "running" ;
4243+ const { observed, finalEvents, written } = yield * captureRootRunTermination ( {
4244+ key : `recovery-startup:${ startupPersisted } ` ,
4245+ shouldFinalizeRun : ( ) => Effect . succeed ( true ) ,
4246+ ingestNormalized : ( ) => Effect . die ( "ingestion failed" ) ,
4247+ beforeTerminalWrite : ( request ) =>
4248+ Effect . gen ( function * ( ) {
4249+ writes ++ ;
4250+ if ( writes === 1 || ( writes === 2 && ! startupPersisted ) ) {
4251+ yield * Deferred . succeed ( failedWrite , undefined ) ;
4252+ return yield * new EventSink . EventSinkWriteError ( { eventCount : 5 } ) ;
4253+ }
4254+ if ( request . expectedStatus !== status ) return false ;
4255+ status = "failed" ;
4256+ return true ;
4257+ } ) ,
4258+ startTurn : ( input ) =>
4259+ Deferred . await ( failedWrite ) . pipe (
4260+ Effect . andThen (
4261+ Effect . fail (
4262+ new ProviderAdapter . ProviderAdapterTurnStartError ( {
4263+ driver,
4264+ threadId : input . threadId ,
4265+ providerThreadId : input . providerThread . id ,
4266+ runId : input . runId ,
4267+ cause : "provider rejected startup" ,
4268+ } ) ,
4269+ ) ,
4270+ ) ,
4271+ ) ,
4272+ onStartFailure : ( error ) =>
4273+ Effect . sync ( ( ) => {
4274+ assert . isFalse ( startupPersisted ) ;
4275+ assert . instanceOf ( error , RunExecutionService . RunExecutionStartError ) ;
4276+ } ) ,
4277+ afterStart : ( consumerClosed ) =>
4278+ Effect . gen ( function * ( ) {
4279+ yield * Deferred . await ( consumerClosed ) ;
4280+ assert . equal ( writes , 2 ) ;
4281+ yield * TestClock . adjust ( "1 second" ) ;
4282+ assert . equal ( writes , 3 ) ;
4283+ yield * TestClock . adjust ( "10 minutes" ) ;
4284+ assert . equal ( writes , 3 ) ;
4285+ } ) ,
4286+ } ) ;
4287+ assert . deepEqual ( observed , [ "run:failed" , "pull-requests-refreshed" ] ) ;
4288+ assert . lengthOf (
4289+ finalEvents . filter (
4290+ ( event ) => event . type === "run-attempt.updated" && event . payload . status === "failed" ,
4291+ ) ,
4292+ 1 ,
4293+ ) ;
4294+ assert . lengthOf ( written , 1 ) ;
4295+ assert . equal ( written [ 0 ] ?. type , "error" ) ;
4296+ if ( written [ 0 ] ?. type === "error" )
4297+ assert . include (
4298+ written [ 0 ] . failure . message ,
4299+ startupPersisted ? "provider could not start this turn" : "Provider turn failed." ,
4300+ ) ;
4301+ } ) ,
4302+ ) ;
4303+
4304+ it . effect ( "stops pending recovery at service shutdown" , ( ) =>
4305+ Effect . gen ( function * ( ) {
4306+ const failedWrite = yield * Deferred . make < void > ( ) ;
4307+ let writes = 0 ;
4308+ const exit = yield * captureRootRunTermination ( {
4309+ key : "recovery-shutdown" ,
4310+ shouldFinalizeRun : ( ) => Effect . succeed ( true ) ,
4311+ ingestNormalized : ( ) => Effect . die ( "ingestion failed" ) ,
4312+ beforeTerminalWrite : ( ) =>
4313+ Effect . gen ( function * ( ) {
4314+ writes ++ ;
4315+ yield * Deferred . succeed ( failedWrite , undefined ) ;
4316+ return yield * new EventSink . EventSinkWriteError ( { eventCount : 5 } ) ;
4317+ } ) ,
4318+ afterStart : ( ) =>
4319+ Effect . gen ( function * ( ) {
4320+ yield * Deferred . await ( failedWrite ) ;
4321+ yield * TestClock . adjust ( "1 second" ) ;
4322+ assert . equal ( writes , 2 ) ;
4323+ return yield * Effect . interrupt ;
4324+ } ) ,
4325+ } ) . pipe ( Effect . exit ) ;
4326+ assert . isTrue ( Exit . isFailure ( exit ) && Cause . hasInterruptsOnly ( exit . cause ) ) ;
4327+ yield * TestClock . adjust ( "10 minutes" ) ;
4328+ assert . equal ( writes , 2 ) ;
4329+ } ) ,
4330+ ) ;
4331+
41074332function captureRootRunTermination ( input : {
41084333 readonly key : string ;
41094334 readonly shouldFinalizeRun : ( ) => Effect . Effect < boolean , ProjectionStore . ProjectionStoreV2Error > ;
@@ -4117,10 +4342,18 @@ function captureRootRunTermination(input: {
41174342 ProviderAdapter . ProviderAdapterV2Error
41184343 > ;
41194344 readonly startTurn ?: ProviderAdapter . ProviderAdapterV2SessionRuntime [ "startTurn" ] ;
4345+ readonly onStartFailure ?: (
4346+ error : RunExecutionService . RunExecutionServiceV2Error ,
4347+ ) => Effect . Effect < void > ;
41204348 readonly refreshAfterTurn ?: Effect . Effect < void > ;
41214349 readonly onSubscriptionClose ?: Effect . Effect < void > ;
41224350 readonly shouldStartProviderTurn ?: RunExecutionService . RunExecutionServiceV2StartRootRunInput [ "shouldStartProviderTurn" ] ;
41234351 readonly appContext ?: ReturnType < McpAppModelContext . McpAppModelContext [ "Service" ] [ "forThread" ] > ;
4352+ readonly ingestNormalized ?: ProviderEventIngestor . ProviderEventIngestorV2 [ "Service" ] [ "ingestNormalized" ] ;
4353+ readonly beforeTerminalWrite ?: (
4354+ request : Parameters < EventSink . EventSinkV2 [ "Service" ] [ "writeIfRunCurrent" ] > [ 0 ] ,
4355+ ) => Effect . Effect < boolean , EventSink . EventSinkV2Error > ;
4356+ readonly afterStart ?: ( closed : Deferred . Deferred < void > ) => Effect . Effect < void > ;
41244357} ) {
41254358 return Effect . gen ( function * ( ) {
41264359 const ids = backgroundScenarioIds ( input . key ) ;
@@ -4181,7 +4414,11 @@ function captureRootRunTermination(input: {
41814414 Effect . gen ( function * ( ) {
41824415 const effects = payload . effects ?? [ ] ;
41834416 yield * Ref . update ( submittedEffects , ( current ) => [ ...current , ...effects ] ) ;
4184- if ( input . rejectTerminalWrite === true ) {
4417+ if (
4418+ input . rejectTerminalWrite === true ||
4419+ ( input . beforeTerminalWrite !== undefined &&
4420+ ! ( yield * input . beforeTerminalWrite ( payload ) ) )
4421+ ) {
41854422 return { committed : false , storedEvents : [ ] } ;
41864423 }
41874424 yield * Ref . update ( committedEffects , ( current ) => [ ...current , ...effects ] ) ;
@@ -4191,7 +4428,7 @@ function captureRootRunTermination(input: {
41914428 } ) ,
41924429 IdAllocator . layer ,
41934430 Layer . mock ( ProviderEventIngestor . ProviderEventIngestorV2 ) ( {
4194- ingestNormalized : ( ) => Effect . succeed ( [ ] ) ,
4431+ ingestNormalized : input . ingestNormalized ?? ( ( ) => Effect . succeed ( [ ] ) ) ,
41954432 } ) ,
41964433 ServerSettings . layerTest ( ) ,
41974434 Layer . succeed ( RunFinalizationService . RunFinalizationObserver , {
@@ -4207,7 +4444,7 @@ function captureRootRunTermination(input: {
42074444
42084445 yield * Effect . gen ( function * ( ) {
42094446 const runExecution = yield * RunExecutionService . RunExecutionServiceV2 ;
4210- yield * runExecution . startRootRun ( {
4447+ const start = runExecution . startRootRun ( {
42114448 commandId : CommandId . make ( `command:${ input . key } ` ) ,
42124449 appThread : { id : ids . threadId } as OrchestrationV2AppThread ,
42134450 providerSessionId : ProviderSessionId . make ( `session:${ input . key } ` ) ,
@@ -4298,9 +4535,13 @@ function captureRootRunTermination(input: {
42984535 } ,
42994536 } ,
43004537 } ) ;
4538+ yield * start . pipe (
4539+ Effect . catch ( ( error ) => input . onStartFailure ?.( error ) ?? Effect . fail ( error ) ) ,
4540+ ) ;
4541+ yield * input . afterStart ?.( ingestionDone ) ?? Effect . void ;
4542+ yield * Deferred . await ( ingestionDone ) ;
43014543 } ) . pipe ( Effect . provide ( layerTest ) ) ;
43024544
4303- yield * Deferred . await ( ingestionDone ) ;
43044545 return {
43054546 written : yield * Ref . get ( writtenItems ) ,
43064547 finalEvents : yield * Ref . get ( finalEvents ) ,
0 commit comments