@@ -36,6 +36,7 @@ import * as Layer from "effect/Layer";
3636import * as Option from "effect/Option" ;
3737import * as Ref from "effect/Ref" ;
3838import * as Stream from "effect/Stream" ;
39+ import * as TestClock from "effect/testing/TestClock" ;
3940
4041import * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts" ;
4142import * as ServerSettings from "../serverSettings.ts" ;
@@ -3814,6 +3815,145 @@ it.effect("keeps completed runs completed when pull request refresh fails", () =
38143815 } ) ,
38153816) ;
38163817
3818+ it . effect . each ( [ "typed" , "defect" ] as const ) (
3819+ "settles failed runs after transient %s terminal write failures" ,
3820+ ( failureKind ) =>
3821+ Effect . gen ( function * ( ) {
3822+ const failedAt = yield * DateTime . now ;
3823+ const failedWrite = yield * Deferred . make < void > ( ) ;
3824+ const writeAttempts = yield * Ref . make ( 0 ) ;
3825+ const { observed, attempts, written } = yield * captureRootRunTermination ( {
3826+ key : `failure-finalization-retry:${ failureKind } ` ,
3827+ shouldFinalizeRun : ( ) => Effect . succeed ( true ) ,
3828+ ingestNormalized : ( ) => Effect . die ( "ingestion failed" ) ,
3829+ beforeTerminalWrite : ( ) =>
3830+ Effect . gen ( function * ( ) {
3831+ const count = yield * Ref . updateAndGet ( writeAttempts , ( count ) => count + 1 ) ;
3832+ if ( count > 2 ) return true ;
3833+ yield * Deferred . succeed ( failedWrite , undefined ) ;
3834+ const error = new EventSink . EventSinkWriteError ( { eventCount : 5 } ) ;
3835+ return yield * failureKind === "typed" ? Effect . fail ( error ) : Effect . die ( error ) ;
3836+ } ) ,
3837+ afterStart : ( closed ) =>
3838+ Effect . gen ( function * ( ) {
3839+ yield * Deferred . await ( failedWrite ) ;
3840+ yield * TestClock . adjust ( "1 second" ) ;
3841+ assert . isFalse ( yield * Deferred . isDone ( closed ) ) ;
3842+ assert . equal ( yield * Ref . get ( writeAttempts ) , 2 ) ;
3843+ yield * TestClock . adjust ( "2 seconds" ) ;
3844+ } ) ,
3845+ } ) ;
3846+ assert . deepEqual ( observed , [ "run:failed" , "pull-requests-refreshed" ] ) ;
3847+ assert . deepEqual ( attempts , [ "failed" ] ) ;
3848+ assert . deepEqual (
3849+ written . map ( ( item ) => item . completedAt ) ,
3850+ [ failedAt ] ,
3851+ ) ;
3852+ assert . equal ( yield * Ref . get ( writeAttempts ) , 3 ) ;
3853+ } ) ,
3854+ ) ;
3855+
3856+ it . effect ( "does not overwrite a superseding attempt when failure finalization retries" , ( ) =>
3857+ Effect . gen ( function * ( ) {
3858+ const failedWrite = yield * Deferred . make < void > ( ) ;
3859+ const writeAttempts = yield * Ref . make ( 0 ) ;
3860+ const { observed, attempts, written } = yield * captureRootRunTermination ( {
3861+ key : "failure-finalization-superseded" ,
3862+ shouldFinalizeRun : ( ) => Effect . succeed ( true ) ,
3863+ ingestNormalized : ( ) => Effect . die ( "ingestion failed" ) ,
3864+ beforeTerminalWrite : ( ) =>
3865+ Effect . gen ( function * ( ) {
3866+ const count = yield * Ref . updateAndGet ( writeAttempts , ( count ) => count + 1 ) ;
3867+ if ( count > 1 ) return false ;
3868+ yield * Deferred . succeed ( failedWrite , undefined ) ;
3869+ return yield * new EventSink . EventSinkWriteError ( { eventCount : 5 } ) ;
3870+ } ) ,
3871+ afterStart : ( closed ) =>
3872+ Effect . gen ( function * ( ) {
3873+ yield * Deferred . await ( failedWrite ) ;
3874+ yield * TestClock . adjust ( "1 second" ) ;
3875+ assert . equal ( yield * Ref . get ( writeAttempts ) , 2 ) ;
3876+ yield * Deferred . await ( closed ) ;
3877+ yield * TestClock . adjust ( "30 seconds" ) ;
3878+ } ) ,
3879+ } ) ;
3880+ assert . deepEqual ( observed , [ ] ) ;
3881+ assert . deepEqual ( attempts , [ ] ) ;
3882+ assert . deepEqual ( written , [ ] ) ;
3883+ assert . equal ( yield * Ref . get ( writeAttempts ) , 2 ) ;
3884+ } ) ,
3885+ ) ;
3886+
3887+ it . effect ( "stops ingestion recovery when a rejected provider start interrupts its consumer" , ( ) =>
3888+ Effect . gen ( function * ( ) {
3889+ const failedWrite = yield * Deferred . make < void > ( ) ;
3890+ const writeAttempts = yield * Ref . make ( 0 ) ;
3891+ const { observed, attempts } = yield * captureRootRunTermination ( {
3892+ key : "failure-finalization-start-rejected" ,
3893+ shouldFinalizeRun : ( ) => Effect . succeed ( true ) ,
3894+ ingestNormalized : ( ) => Effect . die ( "ingestion failed" ) ,
3895+ beforeTerminalWrite : ( ) =>
3896+ Effect . gen ( function * ( ) {
3897+ const count = yield * Ref . updateAndGet ( writeAttempts , ( count ) => count + 1 ) ;
3898+ if ( count > 1 ) return true ;
3899+ yield * Deferred . succeed ( failedWrite , undefined ) ;
3900+ return yield * new EventSink . EventSinkWriteError ( { eventCount : 5 } ) ;
3901+ } ) ,
3902+ startTurn : ( input ) =>
3903+ Deferred . await ( failedWrite ) . pipe (
3904+ Effect . andThen (
3905+ Effect . fail (
3906+ new ProviderAdapter . ProviderAdapterTurnStartError ( {
3907+ driver,
3908+ threadId : input . threadId ,
3909+ providerThreadId : input . providerThread . id ,
3910+ runId : input . runId ,
3911+ cause : "provider rejected the turn" ,
3912+ } ) ,
3913+ ) ,
3914+ ) ,
3915+ ) ,
3916+ afterStart : ( closed ) =>
3917+ Effect . gen ( function * ( ) {
3918+ yield * Deferred . await ( closed ) ;
3919+ // The service is still open here; only the consumer was interrupted.
3920+ yield * TestClock . adjust ( "30 seconds" ) ;
3921+ assert . equal ( yield * Ref . get ( writeAttempts ) , 2 ) ;
3922+ } ) ,
3923+ } ) ;
3924+ assert . deepEqual ( observed , [ "run:failed" , "pull-requests-refreshed" ] ) ;
3925+ assert . deepEqual ( attempts , [ "failed" ] ) ;
3926+ } ) ,
3927+ ) ;
3928+
3929+ it . effect ( "interrupts failure-finalization retries when the service scope closes" , ( ) =>
3930+ Effect . gen ( function * ( ) {
3931+ const failedWrite = yield * Deferred . make < void > ( ) ;
3932+ const writeAttempts = yield * Ref . make ( 0 ) ;
3933+ const exit = yield * captureRootRunTermination ( {
3934+ key : "failure-finalization-shutdown" ,
3935+ shouldFinalizeRun : ( ) => Effect . succeed ( true ) ,
3936+ ingestNormalized : ( ) => Effect . die ( "ingestion failed" ) ,
3937+ beforeTerminalWrite : ( ) =>
3938+ Ref . update ( writeAttempts , ( count ) => count + 1 ) . pipe (
3939+ Effect . andThen ( Deferred . succeed ( failedWrite , undefined ) ) ,
3940+ Effect . andThen ( Effect . fail ( new EventSink . EventSinkWriteError ( { eventCount : 5 } ) ) ) ,
3941+ ) ,
3942+ afterStart : ( closed ) =>
3943+ Effect . gen ( function * ( ) {
3944+ yield * Deferred . await ( failedWrite ) ;
3945+ yield * TestClock . adjust ( "1 second" ) ;
3946+ assert . isFalse ( yield * Deferred . isDone ( closed ) ) ;
3947+ assert . equal ( yield * Ref . get ( writeAttempts ) , 2 ) ;
3948+ return yield * Effect . interrupt ;
3949+ } ) ,
3950+ } ) . pipe ( Effect . exit ) ;
3951+ assert . isTrue ( Exit . isFailure ( exit ) && Cause . hasInterruptsOnly ( exit . cause ) ) ;
3952+ yield * TestClock . adjust ( "30 seconds" ) ;
3953+ assert . equal ( yield * Ref . get ( writeAttempts ) , 2 ) ;
3954+ } ) ,
3955+ ) ;
3956+
38173957function captureRootRunTermination ( input : {
38183958 readonly key : string ;
38193959 readonly shouldFinalizeRun : ( ) => Effect . Effect < boolean , ProjectionStore . ProjectionStoreV2Error > ;
@@ -3828,6 +3968,9 @@ function captureRootRunTermination(input: {
38283968 > ;
38293969 readonly startTurn ?: ProviderAdapter . ProviderAdapterV2SessionRuntime [ "startTurn" ] ;
38303970 readonly refreshAfterTurn ?: Effect . Effect < void > ;
3971+ readonly ingestNormalized ?: ProviderEventIngestor . ProviderEventIngestorV2 [ "Service" ] [ "ingestNormalized" ] ;
3972+ readonly beforeTerminalWrite ?: ( ) => Effect . Effect < boolean , EventSink . EventSinkV2Error > ;
3973+ readonly afterStart ?: ( closed : Deferred . Deferred < void > ) => Effect . Effect < void > ;
38313974} ) {
38323975 return Effect . gen ( function * ( ) {
38333976 const ids = backgroundScenarioIds ( input . key ) ;
@@ -3841,6 +3984,7 @@ function captureRootRunTermination(input: {
38413984 } ) ;
38423985 const writtenItems = yield * Ref . make < ReadonlyArray < OrchestrationV2TurnItem > > ( [ ] ) ;
38433986 const observed = yield * Ref . make < ReadonlyArray < string > > ( [ ] ) ;
3987+ const attempts = yield * Ref . make < ReadonlyArray < OrchestrationV2RunAttempt [ "status" ] > > ( [ ] ) ;
38443988 const submittedEffects = yield * Ref . make < ReadonlyArray < PendingOrchestrationEffectV2 > > ( [ ] ) ;
38453989 const committedEffects = yield * Ref . make < ReadonlyArray < PendingOrchestrationEffectV2 > > ( [ ] ) ;
38463990 const ingestionDone = yield * Deferred . make < void > ( ) ;
@@ -3855,6 +3999,9 @@ function captureRootRunTermination(input: {
38553999 if ( event . type === "run.updated" ) {
38564000 yield * Ref . update ( observed , ( current ) => [ ...current , `run:${ event . payload . status } ` ] ) ;
38574001 }
4002+ if ( event . type === "run-attempt.updated" ) {
4003+ yield * Ref . update ( attempts , ( current ) => [ ...current , event . payload . status ] ) ;
4004+ }
38584005 }
38594006 } ) ;
38604007 const layerTest = RunExecutionService . layer . pipe (
@@ -3883,7 +4030,10 @@ function captureRootRunTermination(input: {
38834030 Effect . gen ( function * ( ) {
38844031 const effects = payload . effects ?? [ ] ;
38854032 yield * Ref . update ( submittedEffects , ( current ) => [ ...current , ...effects ] ) ;
3886- if ( input . rejectTerminalWrite === true ) {
4033+ if (
4034+ input . rejectTerminalWrite === true ||
4035+ ( input . beforeTerminalWrite !== undefined && ! ( yield * input . beforeTerminalWrite ( ) ) )
4036+ ) {
38874037 return { committed : false , storedEvents : [ ] } ;
38884038 }
38894039 yield * Ref . update ( committedEffects , ( current ) => [ ...current , ...effects ] ) ;
@@ -3893,7 +4043,7 @@ function captureRootRunTermination(input: {
38934043 } ) ,
38944044 IdAllocator . layer ,
38954045 Layer . mock ( ProviderEventIngestor . ProviderEventIngestorV2 ) ( {
3896- ingestNormalized : ( ) => Effect . succeed ( [ ] ) ,
4046+ ingestNormalized : input . ingestNormalized ?? ( ( ) => Effect . succeed ( [ ] ) ) ,
38974047 } ) ,
38984048 ServerSettings . layerTest ( ) ,
38994049 Layer . succeed ( RunFinalizationService . RunFinalizationObserver , {
@@ -3995,12 +4145,14 @@ function captureRootRunTermination(input: {
39954145 } ,
39964146 } ,
39974147 } ) ;
4148+ yield * input . afterStart ?.( ingestionDone ) ?? Effect . void ;
4149+ yield * Deferred . await ( ingestionDone ) ;
39984150 } ) . pipe ( Effect . provide ( layerTest ) ) ;
39994151
4000- yield * Deferred . await ( ingestionDone ) ;
40014152 return {
40024153 written : yield * Ref . get ( writtenItems ) ,
40034154 observed : yield * Ref . get ( observed ) ,
4155+ attempts : yield * Ref . get ( attempts ) ,
40044156 submittedEffects : yield * Ref . get ( submittedEffects ) ,
40054157 committedEffects : yield * Ref . get ( committedEffects ) ,
40064158 } ;
0 commit comments