@@ -33,9 +33,11 @@ import * as DateTime from "effect/DateTime";
3333import * as Deferred from "effect/Deferred" ;
3434import * as Effect from "effect/Effect" ;
3535import * as Layer from "effect/Layer" ;
36+ import * as Logger from "effect/Logger" ;
3637import * as Option from "effect/Option" ;
3738import * as Ref from "effect/Ref" ;
3839import * as Stream from "effect/Stream" ;
40+ import * as TestClock from "effect/testing/TestClock" ;
3941
4042import * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts" ;
4143import * as ServerSettings from "../serverSettings.ts" ;
@@ -3849,6 +3851,228 @@ it.effect("keeps completed runs completed when pull request refresh fails", () =
38493851 } ) ,
38503852) ;
38513853
3854+ it . effect . each ( [ "typed" , "defect" ] as const ) (
3855+ "settles failed runs after transient %s terminal write failures" ,
3856+ ( failureKind ) =>
3857+ Effect . gen ( function * ( ) {
3858+ const failedAt = yield * DateTime . now ;
3859+ const failedWrite = yield * Deferred . make < void > ( ) ;
3860+ const writeAttempts = yield * Ref . make ( 0 ) ;
3861+ const { observed, attempts, written } = yield * captureRootRunTermination ( {
3862+ key : `failure-finalization-retry:${ failureKind } ` ,
3863+ shouldFinalizeRun : ( ) => Effect . succeed ( true ) ,
3864+ ingestNormalized : ( ) => Effect . die ( "ingestion failed" ) ,
3865+ beforeTerminalWrite : ( ) =>
3866+ Effect . gen ( function * ( ) {
3867+ const count = yield * Ref . updateAndGet ( writeAttempts , ( count ) => count + 1 ) ;
3868+ if ( count > 2 ) return true ;
3869+ yield * Deferred . succeed ( failedWrite , undefined ) ;
3870+ const error = new EventSink . EventSinkWriteError ( { eventCount : 5 } ) ;
3871+ return yield * failureKind === "typed" ? Effect . fail ( error ) : Effect . die ( error ) ;
3872+ } ) ,
3873+ afterStart : ( closed ) =>
3874+ Effect . gen ( function * ( ) {
3875+ yield * Deferred . await ( failedWrite ) ;
3876+ yield * TestClock . adjust ( "1 second" ) ;
3877+ assert . isFalse ( yield * Deferred . isDone ( closed ) ) ;
3878+ assert . equal ( yield * Ref . get ( writeAttempts ) , 2 ) ;
3879+ yield * TestClock . adjust ( "2 seconds" ) ;
3880+ } ) ,
3881+ } ) ;
3882+ assert . deepEqual ( observed , [ "run:failed" , "pull-requests-refreshed" ] ) ;
3883+ assert . deepEqual ( attempts , [ "failed" ] ) ;
3884+ assert . deepEqual (
3885+ written . map ( ( item ) => item . completedAt ) ,
3886+ [ failedAt ] ,
3887+ ) ;
3888+ assert . equal ( yield * Ref . get ( writeAttempts ) , 3 ) ;
3889+ } ) ,
3890+ ) ;
3891+
3892+ it . effect ( "throttles failure-finalization warnings without slowing recovery" , ( ) => {
3893+ const warnings : Array < unknown > = [ ] ;
3894+ const logger = Logger . make ( ( { message } ) => {
3895+ const text = Array . isArray ( message ) ? message [ 0 ] : message ;
3896+ if ( text === "orchestration V2 run failure finalization retry" ) warnings . push ( text ) ;
3897+ } ) ;
3898+ return Effect . gen ( function * ( ) {
3899+ const failedWrite = yield * Deferred . make < void > ( ) ;
3900+ const storageAvailable = yield * Ref . make ( false ) ;
3901+ const writeAttempts = yield * Ref . make ( 0 ) ;
3902+ const { observed } = yield * captureRootRunTermination ( {
3903+ key : "failure-finalization-warning-throttle" ,
3904+ shouldFinalizeRun : ( ) => Effect . succeed ( true ) ,
3905+ ingestNormalized : ( ) => Effect . die ( "ingestion failed" ) ,
3906+ beforeTerminalWrite : ( ) =>
3907+ Effect . gen ( function * ( ) {
3908+ yield * Ref . update ( writeAttempts , ( count ) => count + 1 ) ;
3909+ if ( yield * Ref . get ( storageAvailable ) ) return true ;
3910+ yield * Deferred . succeed ( failedWrite , undefined ) ;
3911+ return yield * new EventSink . EventSinkWriteError ( { eventCount : 5 } ) ;
3912+ } ) ,
3913+ afterStart : ( ) =>
3914+ Effect . gen ( function * ( ) {
3915+ yield * Deferred . await ( failedWrite ) ;
3916+ yield * TestClock . adjust ( "1 minute" ) ;
3917+ assert . isAbove ( yield * Ref . get ( writeAttempts ) , 2 ) ;
3918+ assert . lengthOf ( warnings , 1 ) ;
3919+ yield * TestClock . adjust ( "10 minutes" ) ;
3920+ assert . lengthOf ( warnings , 2 ) ;
3921+ yield * Ref . set ( storageAvailable , true ) ;
3922+ yield * TestClock . adjust ( "30 seconds" ) ;
3923+ } ) ,
3924+ } ) ;
3925+ assert . deepEqual ( observed , [ "run:failed" , "pull-requests-refreshed" ] ) ;
3926+ assert . lengthOf ( warnings , 2 ) ;
3927+ } ) . pipe ( Effect . provide ( Logger . layer ( [ logger ] , { mergeWithExisting : false } ) ) ) ;
3928+ } ) ;
3929+
3930+ it . effect ( "does not overwrite a superseding attempt when failure finalization retries" , ( ) =>
3931+ Effect . gen ( function * ( ) {
3932+ const failedWrite = yield * Deferred . make < void > ( ) ;
3933+ const writeAttempts = yield * Ref . make ( 0 ) ;
3934+ const { observed, attempts, written } = yield * captureRootRunTermination ( {
3935+ key : "failure-finalization-superseded" ,
3936+ shouldFinalizeRun : ( ) => Effect . succeed ( true ) ,
3937+ ingestNormalized : ( ) => Effect . die ( "ingestion failed" ) ,
3938+ beforeTerminalWrite : ( ) =>
3939+ Effect . gen ( function * ( ) {
3940+ const count = yield * Ref . updateAndGet ( writeAttempts , ( count ) => count + 1 ) ;
3941+ if ( count > 1 ) return false ;
3942+ yield * Deferred . succeed ( failedWrite , undefined ) ;
3943+ return yield * new EventSink . EventSinkWriteError ( { eventCount : 5 } ) ;
3944+ } ) ,
3945+ afterStart : ( closed ) =>
3946+ Effect . gen ( function * ( ) {
3947+ yield * Deferred . await ( failedWrite ) ;
3948+ yield * TestClock . adjust ( "1 second" ) ;
3949+ assert . equal ( yield * Ref . get ( writeAttempts ) , 2 ) ;
3950+ yield * Deferred . await ( closed ) ;
3951+ yield * TestClock . adjust ( "30 seconds" ) ;
3952+ } ) ,
3953+ } ) ;
3954+ assert . deepEqual ( observed , [ ] ) ;
3955+ assert . deepEqual ( attempts , [ ] ) ;
3956+ assert . deepEqual ( written , [ ] ) ;
3957+ assert . equal ( yield * Ref . get ( writeAttempts ) , 2 ) ;
3958+ } ) ,
3959+ ) ;
3960+
3961+ it . effect ( "does not overwrite a startup terminal when its consumer interrupts recovery" , ( ) =>
3962+ Effect . gen ( function * ( ) {
3963+ const failedWrite = yield * Deferred . make < void > ( ) ;
3964+ const writeAttempts = yield * Ref . make ( 0 ) ;
3965+ const { observed, attempts } = yield * captureRootRunTermination ( {
3966+ key : "failure-finalization-start-rejected" ,
3967+ shouldFinalizeRun : ( ) => Effect . succeed ( true ) ,
3968+ ingestNormalized : ( ) => Effect . die ( "ingestion failed" ) ,
3969+ beforeTerminalWrite : ( ) =>
3970+ Effect . gen ( function * ( ) {
3971+ const count = yield * Ref . updateAndGet ( writeAttempts , ( count ) => count + 1 ) ;
3972+ if ( count > 1 ) return count === 2 ;
3973+ yield * Deferred . succeed ( failedWrite , undefined ) ;
3974+ return yield * new EventSink . EventSinkWriteError ( { eventCount : 5 } ) ;
3975+ } ) ,
3976+ startTurn : ( input ) =>
3977+ Deferred . await ( failedWrite ) . pipe (
3978+ Effect . andThen (
3979+ Effect . fail (
3980+ new ProviderAdapter . ProviderAdapterTurnStartError ( {
3981+ driver,
3982+ threadId : input . threadId ,
3983+ providerThreadId : input . providerThread . id ,
3984+ runId : input . runId ,
3985+ cause : "provider rejected the turn" ,
3986+ } ) ,
3987+ ) ,
3988+ ) ,
3989+ ) ,
3990+ afterStart : ( closed ) =>
3991+ Effect . gen ( function * ( ) {
3992+ yield * Deferred . await ( closed ) ;
3993+ // The service is still open here; only the consumer was interrupted.
3994+ yield * TestClock . adjust ( "30 seconds" ) ;
3995+ assert . equal ( yield * Ref . get ( writeAttempts ) , 3 ) ;
3996+ } ) ,
3997+ } ) ;
3998+ assert . deepEqual ( observed , [ "run:failed" , "pull-requests-refreshed" ] ) ;
3999+ assert . deepEqual ( attempts , [ "failed" ] ) ;
4000+ } ) ,
4001+ ) ;
4002+
4003+ it . effect ( "settles ingestion failure after startup finalization also fails" , ( ) =>
4004+ Effect . gen ( function * ( ) {
4005+ const failedWrite = yield * Deferred . make < void > ( ) ;
4006+ const writeAttempts = yield * Ref . make ( 0 ) ;
4007+ const { observed, attempts } = yield * captureRootRunTermination ( {
4008+ key : "failure-finalization-start-write-failed" ,
4009+ shouldFinalizeRun : ( ) => Effect . succeed ( true ) ,
4010+ ingestNormalized : ( ) => Effect . die ( "ingestion failed" ) ,
4011+ beforeTerminalWrite : ( ) =>
4012+ Effect . gen ( function * ( ) {
4013+ const count = yield * Ref . updateAndGet ( writeAttempts , ( count ) => count + 1 ) ;
4014+ if ( count > 2 ) return true ;
4015+ yield * Deferred . succeed ( failedWrite , undefined ) ;
4016+ return yield * new EventSink . EventSinkWriteError ( { eventCount : 5 } ) ;
4017+ } ) ,
4018+ startTurn : ( input ) =>
4019+ Deferred . await ( failedWrite ) . pipe (
4020+ Effect . andThen (
4021+ Effect . fail (
4022+ new ProviderAdapter . ProviderAdapterTurnStartError ( {
4023+ driver,
4024+ threadId : input . threadId ,
4025+ providerThreadId : input . providerThread . id ,
4026+ runId : input . runId ,
4027+ cause : "provider rejected the turn" ,
4028+ } ) ,
4029+ ) ,
4030+ ) ,
4031+ ) ,
4032+ onStartFailure : ( error ) =>
4033+ Effect . sync ( ( ) => assert . instanceOf ( error , RunExecutionService . RunExecutionStartError ) ) ,
4034+ afterStart : ( closed ) =>
4035+ Effect . gen ( function * ( ) {
4036+ yield * Deferred . await ( closed ) ;
4037+ assert . equal ( yield * Ref . get ( writeAttempts ) , 2 ) ;
4038+ // Startup has failed and its consumer is gone, but the service is open.
4039+ yield * TestClock . adjust ( "1 second" ) ;
4040+ assert . equal ( yield * Ref . get ( writeAttempts ) , 3 ) ;
4041+ } ) ,
4042+ } ) ;
4043+ assert . deepEqual ( observed , [ "run:failed" , "pull-requests-refreshed" ] ) ;
4044+ assert . deepEqual ( attempts , [ "failed" ] ) ;
4045+ } ) ,
4046+ ) ;
4047+
4048+ it . effect ( "interrupts failure-finalization retries when the service scope closes" , ( ) =>
4049+ Effect . gen ( function * ( ) {
4050+ const failedWrite = yield * Deferred . make < void > ( ) ;
4051+ const writeAttempts = yield * Ref . make ( 0 ) ;
4052+ const exit = yield * captureRootRunTermination ( {
4053+ key : "failure-finalization-shutdown" ,
4054+ shouldFinalizeRun : ( ) => Effect . succeed ( true ) ,
4055+ ingestNormalized : ( ) => Effect . die ( "ingestion failed" ) ,
4056+ beforeTerminalWrite : ( ) =>
4057+ Ref . update ( writeAttempts , ( count ) => count + 1 ) . pipe (
4058+ Effect . andThen ( Deferred . succeed ( failedWrite , undefined ) ) ,
4059+ Effect . andThen ( Effect . fail ( new EventSink . EventSinkWriteError ( { eventCount : 5 } ) ) ) ,
4060+ ) ,
4061+ afterStart : ( closed ) =>
4062+ Effect . gen ( function * ( ) {
4063+ yield * Deferred . await ( failedWrite ) ;
4064+ yield * TestClock . adjust ( "1 second" ) ;
4065+ assert . isFalse ( yield * Deferred . isDone ( closed ) ) ;
4066+ assert . equal ( yield * Ref . get ( writeAttempts ) , 2 ) ;
4067+ return yield * Effect . interrupt ;
4068+ } ) ,
4069+ } ) . pipe ( Effect . exit ) ;
4070+ assert . isTrue ( Exit . isFailure ( exit ) && Cause . hasInterruptsOnly ( exit . cause ) ) ;
4071+ yield * TestClock . adjust ( "30 seconds" ) ;
4072+ assert . equal ( yield * Ref . get ( writeAttempts ) , 2 ) ;
4073+ } ) ,
4074+ ) ;
4075+
38524076function captureRootRunTermination ( input : {
38534077 readonly key : string ;
38544078 readonly shouldFinalizeRun : ( ) => Effect . Effect < boolean , ProjectionStore . ProjectionStoreV2Error > ;
@@ -3862,9 +4086,15 @@ function captureRootRunTermination(input: {
38624086 ProviderAdapter . ProviderAdapterV2Error
38634087 > ;
38644088 readonly startTurn ?: ProviderAdapter . ProviderAdapterV2SessionRuntime [ "startTurn" ] ;
4089+ readonly onStartFailure ?: (
4090+ error : RunExecutionService . RunExecutionServiceV2Error ,
4091+ ) => Effect . Effect < void > ;
38654092 readonly refreshAfterTurn ?: Effect . Effect < void > ;
38664093 readonly shouldStartProviderTurn ?: RunExecutionService . RunExecutionServiceV2StartRootRunInput [ "shouldStartProviderTurn" ] ;
38674094 readonly appContext ?: ReturnType < McpAppModelContext . McpAppModelContext [ "Service" ] [ "forThread" ] > ;
4095+ readonly ingestNormalized ?: ProviderEventIngestor . ProviderEventIngestorV2 [ "Service" ] [ "ingestNormalized" ] ;
4096+ readonly beforeTerminalWrite ?: ( ) => Effect . Effect < boolean , EventSink . EventSinkV2Error > ;
4097+ readonly afterStart ?: ( closed : Deferred . Deferred < void > ) => Effect . Effect < void > ;
38684098} ) {
38694099 return Effect . gen ( function * ( ) {
38704100 const ids = backgroundScenarioIds ( input . key ) ;
@@ -3879,6 +4109,7 @@ function captureRootRunTermination(input: {
38794109 const writtenItems = yield * Ref . make < ReadonlyArray < OrchestrationV2TurnItem > > ( [ ] ) ;
38804110 const finalEvents = yield * Ref . make < ReadonlyArray < OrchestrationV2DomainEvent > > ( [ ] ) ;
38814111 const observed = yield * Ref . make < ReadonlyArray < string > > ( [ ] ) ;
4112+ const attempts = yield * Ref . make < ReadonlyArray < OrchestrationV2RunAttempt [ "status" ] > > ( [ ] ) ;
38824113 const submittedEffects = yield * Ref . make < ReadonlyArray < PendingOrchestrationEffectV2 > > ( [ ] ) ;
38834114 const committedEffects = yield * Ref . make < ReadonlyArray < PendingOrchestrationEffectV2 > > ( [ ] ) ;
38844115 const ingestionDone = yield * Deferred . make < void > ( ) ;
@@ -3894,6 +4125,9 @@ function captureRootRunTermination(input: {
38944125 if ( event . type === "run.updated" ) {
38954126 yield * Ref . update ( observed , ( current ) => [ ...current , `run:${ event . payload . status } ` ] ) ;
38964127 }
4128+ if ( event . type === "run-attempt.updated" ) {
4129+ yield * Ref . update ( attempts , ( current ) => [ ...current , event . payload . status ] ) ;
4130+ }
38974131 }
38984132 } ) ;
38994133 const layerTest = RunExecutionService . layer . pipe (
@@ -3925,7 +4159,10 @@ function captureRootRunTermination(input: {
39254159 Effect . gen ( function * ( ) {
39264160 const effects = payload . effects ?? [ ] ;
39274161 yield * Ref . update ( submittedEffects , ( current ) => [ ...current , ...effects ] ) ;
3928- if ( input . rejectTerminalWrite === true ) {
4162+ if (
4163+ input . rejectTerminalWrite === true ||
4164+ ( input . beforeTerminalWrite !== undefined && ! ( yield * input . beforeTerminalWrite ( ) ) )
4165+ ) {
39294166 return { committed : false , storedEvents : [ ] } ;
39304167 }
39314168 yield * Ref . update ( committedEffects , ( current ) => [ ...current , ...effects ] ) ;
@@ -3935,7 +4172,7 @@ function captureRootRunTermination(input: {
39354172 } ) ,
39364173 IdAllocator . layer ,
39374174 Layer . mock ( ProviderEventIngestor . ProviderEventIngestorV2 ) ( {
3938- ingestNormalized : ( ) => Effect . succeed ( [ ] ) ,
4175+ ingestNormalized : input . ingestNormalized ?? ( ( ) => Effect . succeed ( [ ] ) ) ,
39394176 } ) ,
39404177 ServerSettings . layerTest ( ) ,
39414178 Layer . succeed ( RunFinalizationService . RunFinalizationObserver , {
@@ -3951,7 +4188,7 @@ function captureRootRunTermination(input: {
39514188
39524189 yield * Effect . gen ( function * ( ) {
39534190 const runExecution = yield * RunExecutionService . RunExecutionServiceV2 ;
3954- yield * runExecution . startRootRun ( {
4191+ const start = runExecution . startRootRun ( {
39554192 commandId : CommandId . make ( `command:${ input . key } ` ) ,
39564193 appThread : { id : ids . threadId } as OrchestrationV2AppThread ,
39574194 providerSessionId : ProviderSessionId . make ( `session:${ input . key } ` ) ,
@@ -4040,13 +4277,18 @@ function captureRootRunTermination(input: {
40404277 } ,
40414278 } ,
40424279 } ) ;
4280+ yield * start . pipe (
4281+ Effect . catch ( ( error ) => input . onStartFailure ?.( error ) ?? Effect . fail ( error ) ) ,
4282+ ) ;
4283+ yield * input . afterStart ?.( ingestionDone ) ?? Effect . void ;
4284+ yield * Deferred . await ( ingestionDone ) ;
40434285 } ) . pipe ( Effect . provide ( layerTest ) ) ;
40444286
4045- yield * Deferred . await ( ingestionDone ) ;
40464287 return {
40474288 written : yield * Ref . get ( writtenItems ) ,
40484289 finalEvents : yield * Ref . get ( finalEvents ) ,
40494290 observed : yield * Ref . get ( observed ) ,
4291+ attempts : yield * Ref . get ( attempts ) ,
40504292 submittedEffects : yield * Ref . get ( submittedEffects ) ,
40514293 committedEffects : yield * Ref . get ( committedEffects ) ,
40524294 } ;
0 commit comments