@@ -33,9 +33,11 @@ import * as DateTime from "effect/DateTime";
3333import * as Deferred from "effect/Deferred" ;
3434import * as Effect from "effect/Effect" ;
3535import * as Layer from "effect/Layer" ;
36+ import * as Logger from "effect/Logger" ;
3637import * as Option from "effect/Option" ;
3738import * as Ref from "effect/Ref" ;
3839import * as Stream from "effect/Stream" ;
40+ import * as TestClock from "effect/testing/TestClock" ;
3941
4042import * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts" ;
4143import * as ServerSettings from "../serverSettings.ts" ;
@@ -3831,6 +3833,228 @@ it.effect("keeps completed runs completed when pull request refresh fails", () =
38313833 } ) ,
38323834) ;
38333835
3836+ it . effect . each ( [ "typed" , "defect" ] as const ) (
3837+ "settles failed runs after transient %s terminal write failures" ,
3838+ ( failureKind ) =>
3839+ Effect . gen ( function * ( ) {
3840+ const failedAt = yield * DateTime . now ;
3841+ const failedWrite = yield * Deferred . make < void > ( ) ;
3842+ const writeAttempts = yield * Ref . make ( 0 ) ;
3843+ const { observed, attempts, written } = yield * captureRootRunTermination ( {
3844+ key : `failure-finalization-retry:${ failureKind } ` ,
3845+ shouldFinalizeRun : ( ) => Effect . succeed ( true ) ,
3846+ ingestNormalized : ( ) => Effect . die ( "ingestion failed" ) ,
3847+ beforeTerminalWrite : ( ) =>
3848+ Effect . gen ( function * ( ) {
3849+ const count = yield * Ref . updateAndGet ( writeAttempts , ( count ) => count + 1 ) ;
3850+ if ( count > 2 ) return true ;
3851+ yield * Deferred . succeed ( failedWrite , undefined ) ;
3852+ const error = new EventSink . EventSinkWriteError ( { eventCount : 5 } ) ;
3853+ return yield * failureKind === "typed" ? Effect . fail ( error ) : Effect . die ( error ) ;
3854+ } ) ,
3855+ afterStart : ( closed ) =>
3856+ Effect . gen ( function * ( ) {
3857+ yield * Deferred . await ( failedWrite ) ;
3858+ yield * TestClock . adjust ( "1 second" ) ;
3859+ assert . isFalse ( yield * Deferred . isDone ( closed ) ) ;
3860+ assert . equal ( yield * Ref . get ( writeAttempts ) , 2 ) ;
3861+ yield * TestClock . adjust ( "2 seconds" ) ;
3862+ } ) ,
3863+ } ) ;
3864+ assert . deepEqual ( observed , [ "run:failed" , "pull-requests-refreshed" ] ) ;
3865+ assert . deepEqual ( attempts , [ "failed" ] ) ;
3866+ assert . deepEqual (
3867+ written . map ( ( item ) => item . completedAt ) ,
3868+ [ failedAt ] ,
3869+ ) ;
3870+ assert . equal ( yield * Ref . get ( writeAttempts ) , 3 ) ;
3871+ } ) ,
3872+ ) ;
3873+
3874+ it . effect ( "throttles failure-finalization warnings without slowing recovery" , ( ) => {
3875+ const warnings : Array < unknown > = [ ] ;
3876+ const logger = Logger . make ( ( { message } ) => {
3877+ const text = Array . isArray ( message ) ? message [ 0 ] : message ;
3878+ if ( text === "orchestration V2 run failure finalization retry" ) warnings . push ( text ) ;
3879+ } ) ;
3880+ return Effect . gen ( function * ( ) {
3881+ const failedWrite = yield * Deferred . make < void > ( ) ;
3882+ const storageAvailable = yield * Ref . make ( false ) ;
3883+ const writeAttempts = yield * Ref . make ( 0 ) ;
3884+ const { observed } = yield * captureRootRunTermination ( {
3885+ key : "failure-finalization-warning-throttle" ,
3886+ shouldFinalizeRun : ( ) => Effect . succeed ( true ) ,
3887+ ingestNormalized : ( ) => Effect . die ( "ingestion failed" ) ,
3888+ beforeTerminalWrite : ( ) =>
3889+ Effect . gen ( function * ( ) {
3890+ yield * Ref . update ( writeAttempts , ( count ) => count + 1 ) ;
3891+ if ( yield * Ref . get ( storageAvailable ) ) return true ;
3892+ yield * Deferred . succeed ( failedWrite , undefined ) ;
3893+ return yield * new EventSink . EventSinkWriteError ( { eventCount : 5 } ) ;
3894+ } ) ,
3895+ afterStart : ( ) =>
3896+ Effect . gen ( function * ( ) {
3897+ yield * Deferred . await ( failedWrite ) ;
3898+ yield * TestClock . adjust ( "1 minute" ) ;
3899+ assert . isAbove ( yield * Ref . get ( writeAttempts ) , 2 ) ;
3900+ assert . lengthOf ( warnings , 1 ) ;
3901+ yield * TestClock . adjust ( "10 minutes" ) ;
3902+ assert . lengthOf ( warnings , 2 ) ;
3903+ yield * Ref . set ( storageAvailable , true ) ;
3904+ yield * TestClock . adjust ( "30 seconds" ) ;
3905+ } ) ,
3906+ } ) ;
3907+ assert . deepEqual ( observed , [ "run:failed" , "pull-requests-refreshed" ] ) ;
3908+ assert . lengthOf ( warnings , 2 ) ;
3909+ } ) . pipe ( Effect . provide ( Logger . layer ( [ logger ] , { mergeWithExisting : false } ) ) ) ;
3910+ } ) ;
3911+
3912+ it . effect ( "does not overwrite a superseding attempt when failure finalization retries" , ( ) =>
3913+ Effect . gen ( function * ( ) {
3914+ const failedWrite = yield * Deferred . make < void > ( ) ;
3915+ const writeAttempts = yield * Ref . make ( 0 ) ;
3916+ const { observed, attempts, written } = yield * captureRootRunTermination ( {
3917+ key : "failure-finalization-superseded" ,
3918+ shouldFinalizeRun : ( ) => Effect . succeed ( true ) ,
3919+ ingestNormalized : ( ) => Effect . die ( "ingestion failed" ) ,
3920+ beforeTerminalWrite : ( ) =>
3921+ Effect . gen ( function * ( ) {
3922+ const count = yield * Ref . updateAndGet ( writeAttempts , ( count ) => count + 1 ) ;
3923+ if ( count > 1 ) return false ;
3924+ yield * Deferred . succeed ( failedWrite , undefined ) ;
3925+ return yield * new EventSink . EventSinkWriteError ( { eventCount : 5 } ) ;
3926+ } ) ,
3927+ afterStart : ( closed ) =>
3928+ Effect . gen ( function * ( ) {
3929+ yield * Deferred . await ( failedWrite ) ;
3930+ yield * TestClock . adjust ( "1 second" ) ;
3931+ assert . equal ( yield * Ref . get ( writeAttempts ) , 2 ) ;
3932+ yield * Deferred . await ( closed ) ;
3933+ yield * TestClock . adjust ( "30 seconds" ) ;
3934+ } ) ,
3935+ } ) ;
3936+ assert . deepEqual ( observed , [ ] ) ;
3937+ assert . deepEqual ( attempts , [ ] ) ;
3938+ assert . deepEqual ( written , [ ] ) ;
3939+ assert . equal ( yield * Ref . get ( writeAttempts ) , 2 ) ;
3940+ } ) ,
3941+ ) ;
3942+
3943+ it . effect ( "does not overwrite a startup terminal when its consumer interrupts recovery" , ( ) =>
3944+ Effect . gen ( function * ( ) {
3945+ const failedWrite = yield * Deferred . make < void > ( ) ;
3946+ const writeAttempts = yield * Ref . make ( 0 ) ;
3947+ const { observed, attempts } = yield * captureRootRunTermination ( {
3948+ key : "failure-finalization-start-rejected" ,
3949+ shouldFinalizeRun : ( ) => Effect . succeed ( true ) ,
3950+ ingestNormalized : ( ) => Effect . die ( "ingestion failed" ) ,
3951+ beforeTerminalWrite : ( ) =>
3952+ Effect . gen ( function * ( ) {
3953+ const count = yield * Ref . updateAndGet ( writeAttempts , ( count ) => count + 1 ) ;
3954+ if ( count > 1 ) return count === 2 ;
3955+ yield * Deferred . succeed ( failedWrite , undefined ) ;
3956+ return yield * new EventSink . EventSinkWriteError ( { eventCount : 5 } ) ;
3957+ } ) ,
3958+ startTurn : ( input ) =>
3959+ Deferred . await ( failedWrite ) . pipe (
3960+ Effect . andThen (
3961+ Effect . fail (
3962+ new ProviderAdapter . ProviderAdapterTurnStartError ( {
3963+ driver,
3964+ threadId : input . threadId ,
3965+ providerThreadId : input . providerThread . id ,
3966+ runId : input . runId ,
3967+ cause : "provider rejected the turn" ,
3968+ } ) ,
3969+ ) ,
3970+ ) ,
3971+ ) ,
3972+ afterStart : ( closed ) =>
3973+ Effect . gen ( function * ( ) {
3974+ yield * Deferred . await ( closed ) ;
3975+ // The service is still open here; only the consumer was interrupted.
3976+ yield * TestClock . adjust ( "30 seconds" ) ;
3977+ assert . equal ( yield * Ref . get ( writeAttempts ) , 3 ) ;
3978+ } ) ,
3979+ } ) ;
3980+ assert . deepEqual ( observed , [ "run:failed" , "pull-requests-refreshed" ] ) ;
3981+ assert . deepEqual ( attempts , [ "failed" ] ) ;
3982+ } ) ,
3983+ ) ;
3984+
3985+ it . effect ( "settles ingestion failure after startup finalization also fails" , ( ) =>
3986+ Effect . gen ( function * ( ) {
3987+ const failedWrite = yield * Deferred . make < void > ( ) ;
3988+ const writeAttempts = yield * Ref . make ( 0 ) ;
3989+ const { observed, attempts } = yield * captureRootRunTermination ( {
3990+ key : "failure-finalization-start-write-failed" ,
3991+ shouldFinalizeRun : ( ) => Effect . succeed ( true ) ,
3992+ ingestNormalized : ( ) => Effect . die ( "ingestion failed" ) ,
3993+ beforeTerminalWrite : ( ) =>
3994+ Effect . gen ( function * ( ) {
3995+ const count = yield * Ref . updateAndGet ( writeAttempts , ( count ) => count + 1 ) ;
3996+ if ( count > 2 ) return true ;
3997+ yield * Deferred . succeed ( failedWrite , undefined ) ;
3998+ return yield * new EventSink . EventSinkWriteError ( { eventCount : 5 } ) ;
3999+ } ) ,
4000+ startTurn : ( input ) =>
4001+ Deferred . await ( failedWrite ) . pipe (
4002+ Effect . andThen (
4003+ Effect . fail (
4004+ new ProviderAdapter . ProviderAdapterTurnStartError ( {
4005+ driver,
4006+ threadId : input . threadId ,
4007+ providerThreadId : input . providerThread . id ,
4008+ runId : input . runId ,
4009+ cause : "provider rejected the turn" ,
4010+ } ) ,
4011+ ) ,
4012+ ) ,
4013+ ) ,
4014+ onStartFailure : ( error ) =>
4015+ Effect . sync ( ( ) => assert . instanceOf ( error , RunExecutionService . RunExecutionStartError ) ) ,
4016+ afterStart : ( closed ) =>
4017+ Effect . gen ( function * ( ) {
4018+ yield * Deferred . await ( closed ) ;
4019+ assert . equal ( yield * Ref . get ( writeAttempts ) , 2 ) ;
4020+ // Startup has failed and its consumer is gone, but the service is open.
4021+ yield * TestClock . adjust ( "1 second" ) ;
4022+ assert . equal ( yield * Ref . get ( writeAttempts ) , 3 ) ;
4023+ } ) ,
4024+ } ) ;
4025+ assert . deepEqual ( observed , [ "run:failed" , "pull-requests-refreshed" ] ) ;
4026+ assert . deepEqual ( attempts , [ "failed" ] ) ;
4027+ } ) ,
4028+ ) ;
4029+
4030+ it . effect ( "interrupts failure-finalization retries when the service scope closes" , ( ) =>
4031+ Effect . gen ( function * ( ) {
4032+ const failedWrite = yield * Deferred . make < void > ( ) ;
4033+ const writeAttempts = yield * Ref . make ( 0 ) ;
4034+ const exit = yield * captureRootRunTermination ( {
4035+ key : "failure-finalization-shutdown" ,
4036+ shouldFinalizeRun : ( ) => Effect . succeed ( true ) ,
4037+ ingestNormalized : ( ) => Effect . die ( "ingestion failed" ) ,
4038+ beforeTerminalWrite : ( ) =>
4039+ Ref . update ( writeAttempts , ( count ) => count + 1 ) . pipe (
4040+ Effect . andThen ( Deferred . succeed ( failedWrite , undefined ) ) ,
4041+ Effect . andThen ( Effect . fail ( new EventSink . EventSinkWriteError ( { eventCount : 5 } ) ) ) ,
4042+ ) ,
4043+ afterStart : ( closed ) =>
4044+ Effect . gen ( function * ( ) {
4045+ yield * Deferred . await ( failedWrite ) ;
4046+ yield * TestClock . adjust ( "1 second" ) ;
4047+ assert . isFalse ( yield * Deferred . isDone ( closed ) ) ;
4048+ assert . equal ( yield * Ref . get ( writeAttempts ) , 2 ) ;
4049+ return yield * Effect . interrupt ;
4050+ } ) ,
4051+ } ) . pipe ( Effect . exit ) ;
4052+ assert . isTrue ( Exit . isFailure ( exit ) && Cause . hasInterruptsOnly ( exit . cause ) ) ;
4053+ yield * TestClock . adjust ( "30 seconds" ) ;
4054+ assert . equal ( yield * Ref . get ( writeAttempts ) , 2 ) ;
4055+ } ) ,
4056+ ) ;
4057+
38344058function captureRootRunTermination ( input : {
38354059 readonly key : string ;
38364060 readonly shouldFinalizeRun : ( ) => Effect . Effect < boolean , ProjectionStore . ProjectionStoreV2Error > ;
@@ -3844,9 +4068,15 @@ function captureRootRunTermination(input: {
38444068 ProviderAdapter . ProviderAdapterV2Error
38454069 > ;
38464070 readonly startTurn ?: ProviderAdapter . ProviderAdapterV2SessionRuntime [ "startTurn" ] ;
4071+ readonly onStartFailure ?: (
4072+ error : RunExecutionService . RunExecutionServiceV2Error ,
4073+ ) => Effect . Effect < void > ;
38474074 readonly refreshAfterTurn ?: Effect . Effect < void > ;
38484075 readonly shouldStartProviderTurn ?: RunExecutionService . RunExecutionServiceV2StartRootRunInput [ "shouldStartProviderTurn" ] ;
38494076 readonly appContext ?: ReturnType < McpAppModelContext . McpAppModelContext [ "Service" ] [ "forThread" ] > ;
4077+ readonly ingestNormalized ?: ProviderEventIngestor . ProviderEventIngestorV2 [ "Service" ] [ "ingestNormalized" ] ;
4078+ readonly beforeTerminalWrite ?: ( ) => Effect . Effect < boolean , EventSink . EventSinkV2Error > ;
4079+ readonly afterStart ?: ( closed : Deferred . Deferred < void > ) => Effect . Effect < void > ;
38504080} ) {
38514081 return Effect . gen ( function * ( ) {
38524082 const ids = backgroundScenarioIds ( input . key ) ;
@@ -3861,6 +4091,7 @@ function captureRootRunTermination(input: {
38614091 const writtenItems = yield * Ref . make < ReadonlyArray < OrchestrationV2TurnItem > > ( [ ] ) ;
38624092 const finalEvents = yield * Ref . make < ReadonlyArray < OrchestrationV2DomainEvent > > ( [ ] ) ;
38634093 const observed = yield * Ref . make < ReadonlyArray < string > > ( [ ] ) ;
4094+ const attempts = yield * Ref . make < ReadonlyArray < OrchestrationV2RunAttempt [ "status" ] > > ( [ ] ) ;
38644095 const submittedEffects = yield * Ref . make < ReadonlyArray < PendingOrchestrationEffectV2 > > ( [ ] ) ;
38654096 const committedEffects = yield * Ref . make < ReadonlyArray < PendingOrchestrationEffectV2 > > ( [ ] ) ;
38664097 const ingestionDone = yield * Deferred . make < void > ( ) ;
@@ -3876,6 +4107,9 @@ function captureRootRunTermination(input: {
38764107 if ( event . type === "run.updated" ) {
38774108 yield * Ref . update ( observed , ( current ) => [ ...current , `run:${ event . payload . status } ` ] ) ;
38784109 }
4110+ if ( event . type === "run-attempt.updated" ) {
4111+ yield * Ref . update ( attempts , ( current ) => [ ...current , event . payload . status ] ) ;
4112+ }
38794113 }
38804114 } ) ;
38814115 const layerTest = RunExecutionService . layer . pipe (
@@ -3906,7 +4140,10 @@ function captureRootRunTermination(input: {
39064140 Effect . gen ( function * ( ) {
39074141 const effects = payload . effects ?? [ ] ;
39084142 yield * Ref . update ( submittedEffects , ( current ) => [ ...current , ...effects ] ) ;
3909- if ( input . rejectTerminalWrite === true ) {
4143+ if (
4144+ input . rejectTerminalWrite === true ||
4145+ ( input . beforeTerminalWrite !== undefined && ! ( yield * input . beforeTerminalWrite ( ) ) )
4146+ ) {
39104147 return { committed : false , storedEvents : [ ] } ;
39114148 }
39124149 yield * Ref . update ( committedEffects , ( current ) => [ ...current , ...effects ] ) ;
@@ -3916,7 +4153,7 @@ function captureRootRunTermination(input: {
39164153 } ) ,
39174154 IdAllocator . layer ,
39184155 Layer . mock ( ProviderEventIngestor . ProviderEventIngestorV2 ) ( {
3919- ingestNormalized : ( ) => Effect . succeed ( [ ] ) ,
4156+ ingestNormalized : input . ingestNormalized ?? ( ( ) => Effect . succeed ( [ ] ) ) ,
39204157 } ) ,
39214158 ServerSettings . layerTest ( ) ,
39224159 Layer . succeed ( RunFinalizationService . RunFinalizationObserver , {
@@ -3932,7 +4169,7 @@ function captureRootRunTermination(input: {
39324169
39334170 yield * Effect . gen ( function * ( ) {
39344171 const runExecution = yield * RunExecutionService . RunExecutionServiceV2 ;
3935- yield * runExecution . startRootRun ( {
4172+ const start = runExecution . startRootRun ( {
39364173 commandId : CommandId . make ( `command:${ input . key } ` ) ,
39374174 appThread : { id : ids . threadId } as OrchestrationV2AppThread ,
39384175 providerSessionId : ProviderSessionId . make ( `session:${ input . key } ` ) ,
@@ -4021,13 +4258,18 @@ function captureRootRunTermination(input: {
40214258 } ,
40224259 } ,
40234260 } ) ;
4261+ yield * start . pipe (
4262+ Effect . catch ( ( error ) => input . onStartFailure ?.( error ) ?? Effect . fail ( error ) ) ,
4263+ ) ;
4264+ yield * input . afterStart ?.( ingestionDone ) ?? Effect . void ;
4265+ yield * Deferred . await ( ingestionDone ) ;
40244266 } ) . pipe ( Effect . provide ( layerTest ) ) ;
40254267
4026- yield * Deferred . await ( ingestionDone ) ;
40274268 return {
40284269 written : yield * Ref . get ( writtenItems ) ,
40294270 finalEvents : yield * Ref . get ( finalEvents ) ,
40304271 observed : yield * Ref . get ( observed ) ,
4272+ attempts : yield * Ref . get ( attempts ) ,
40314273 submittedEffects : yield * Ref . get ( submittedEffects ) ,
40324274 committedEffects : yield * Ref . get ( committedEffects ) ,
40334275 } ;
0 commit comments