@@ -82,6 +82,9 @@ function layerExecutorFor(input: {
8282 readonly threads ?: Partial < ThreadManagementService . ThreadManagementService [ "Service" ] > ;
8383 readonly continueAfterRestart ?: boolean ;
8484 readonly interrupt ?: ProviderTurnControlService . ProviderTurnControlServiceV2Shape [ "interrupt" ] ;
85+ readonly failRunFailure ?: boolean ;
86+ readonly interruptFailure ?: boolean ;
87+ readonly startInterrupted ?: boolean ;
8588} ) {
8689 const record = ( event : string ) => Ref . update ( input . events , ( events ) => [ ...events , event ] ) ;
8790 const layerDependencies = Layer . mergeAll (
@@ -95,6 +98,19 @@ function layerExecutorFor(input: {
9598 request . replacementProviderSessionId === undefined
9699 ? "interrupt"
97100 : `interrupt:${ request . replacementProviderSessionId } ` ,
101+ ) . pipe (
102+ Effect . andThen (
103+ input . interruptFailure === true
104+ ? Effect . fail (
105+ new ProviderTurnControlService . ProviderTurnControlError ( {
106+ threadId,
107+ operation : "interrupt" ,
108+ providerTurnId,
109+ cause : "simulated interrupt failure" ,
110+ } ) ,
111+ )
112+ : Effect . void ,
113+ ) ,
98114 ) ,
99115 } ) ,
100116 ) ,
@@ -116,6 +132,7 @@ function layerExecutorFor(input: {
116132 start : ( ) =>
117133 Effect . gen ( function * ( ) {
118134 yield * record ( "start" ) ;
135+ if ( input . startInterrupted === true ) return yield * Effect . interrupt ;
119136 if (
120137 input . failFirstStart !== undefined &&
121138 ( yield * Ref . getAndSet ( input . failFirstStart , false ) )
@@ -126,6 +143,19 @@ function layerExecutorFor(input: {
126143 } ) ;
127144 }
128145 } ) ,
146+ failStartingRun : ( ) =>
147+ record ( "fail-run" ) . pipe (
148+ Effect . andThen (
149+ input . failRunFailure === true
150+ ? Effect . fail (
151+ new ProviderTurnStartService . ProviderTurnStartError ( {
152+ runId,
153+ cause : "simulated run failure write" ,
154+ } ) ,
155+ )
156+ : Effect . void ,
157+ ) ,
158+ ) ,
129159 } ) ,
130160 ) ,
131161 Layer . succeed (
@@ -775,7 +805,7 @@ it.effect("safely retries after replacement cleanup succeeds and start fails", (
775805
776806 const first = yield * Effect . gen ( function * ( ) {
777807 const executor = yield * EffectWorker . OrchestrationEffectExecutorV2 ;
778- return yield * Effect . exit ( executor . execute ( effect ) ) ;
808+ return yield * Effect . exit ( executor . execute ( effect , { willRetry : true } ) ) ;
779809 } ) . pipe ( Effect . provide ( layer ) ) ;
780810 assert . isTrue ( Exit . isFailure ( first ) ) ;
781811
@@ -837,3 +867,135 @@ it.effect("settles a delegated child once its restart continuation fails for goo
837867 } ) . pipe ( Effect . provide ( layer ) ) ;
838868 } ) ,
839869) ;
870+
871+ function startEffect ( now : DateTime . Utc ) : EffectOutbox . OrchestrationEffectV2 {
872+ return {
873+ ...restartEffect ( now , { type : "detach" } ) ,
874+ id : "effect:start" ,
875+ request : { type : "provider-turn.start" , runId } ,
876+ } ;
877+ }
878+
879+ it . effect ( "fails the run when the last start attempt fails" , ( ) =>
880+ Effect . gen ( function * ( ) {
881+ const now = yield * DateTime . now ;
882+ const events = yield * Ref . make < ReadonlyArray < string > > ( [ ] ) ;
883+ const failFirstStart = yield * Ref . make ( true ) ;
884+
885+ yield * Effect . gen ( function * ( ) {
886+ const executor = yield * EffectWorker . OrchestrationEffectExecutorV2 ;
887+ yield * executor . execute ( startEffect ( now ) , { willRetry : false } ) ;
888+ } ) . pipe ( Effect . provide ( layerExecutorFor ( { events, failFirstStart } ) ) ) ;
889+
890+ assert . deepEqual ( yield * Ref . get ( events ) , [ "start" , "fail-run" ] ) ;
891+ } ) ,
892+ ) ;
893+
894+ it . effect ( "fails the run when a restart's last attempt cannot interrupt the old turn" , ( ) =>
895+ Effect . gen ( function * ( ) {
896+ const now = yield * DateTime . now ;
897+ const events = yield * Ref . make < ReadonlyArray < string > > ( [ ] ) ;
898+ const effect = restartEffect ( now , { type : "detach" } ) ;
899+
900+ yield * Effect . gen ( function * ( ) {
901+ const executor = yield * EffectWorker . OrchestrationEffectExecutorV2 ;
902+ yield * executor . execute ( effect , { willRetry : false } ) ;
903+ } ) . pipe ( Effect . provide ( layerExecutorFor ( { events, interruptFailure : true } ) ) ) ;
904+
905+ assert . deepEqual ( yield * Ref . get ( events ) , [ "interrupt" , "fail-run" ] ) ;
906+ } ) ,
907+ ) ;
908+
909+ it . effect ( "does not fail the run when its last start attempt is interrupted" , ( ) =>
910+ Effect . gen ( function * ( ) {
911+ const now = yield * DateTime . now ;
912+ const events = yield * Ref . make < ReadonlyArray < string > > ( [ ] ) ;
913+
914+ const exit = yield * Effect . gen ( function * ( ) {
915+ const executor = yield * EffectWorker . OrchestrationEffectExecutorV2 ;
916+ return yield * Effect . exit ( executor . execute ( startEffect ( now ) , { willRetry : false } ) ) ;
917+ } ) . pipe ( Effect . provide ( layerExecutorFor ( { events, startInterrupted : true } ) ) ) ;
918+
919+ assert . isTrue ( Exit . isFailure ( exit ) && Cause . hasInterruptsOnly ( exit . cause ) ) ;
920+ assert . deepEqual ( yield * Ref . get ( events ) , [ "start" ] ) ;
921+ } ) ,
922+ ) ;
923+
924+ it . effect ( "returns the start failure when the run cannot be failed either" , ( ) =>
925+ Effect . gen ( function * ( ) {
926+ const now = yield * DateTime . now ;
927+ const events = yield * Ref . make < ReadonlyArray < string > > ( [ ] ) ;
928+ const failFirstStart = yield * Ref . make ( true ) ;
929+ const effect = restartEffect ( now , { type : "detach" } ) ;
930+
931+ const error = yield * Effect . gen ( function * ( ) {
932+ const executor = yield * EffectWorker . OrchestrationEffectExecutorV2 ;
933+ return yield * executor . execute ( effect , { willRetry : false } ) . pipe ( Effect . flip ) ;
934+ } ) . pipe ( Effect . provide ( layerExecutorFor ( { events, failFirstStart, failRunFailure : true } ) ) ) ;
935+
936+ assert . deepEqual ( yield * Ref . get ( events ) , [ "interrupt" , "detach" , "start" , "fail-run" ] ) ;
937+ assert . include ( Cause . pretty ( Cause . fail ( error ) ) , "simulated first start failure" ) ;
938+ } ) ,
939+ ) ;
940+
941+ it . effect . each ( [ "provider-turn.start" , "provider-turn.restart" ] as const ) (
942+ "keeps retrying a %s effect past its attempt budget" ,
943+ ( effectType ) =>
944+ Effect . gen ( function * ( ) {
945+ const now = yield * DateTime . now ;
946+ const workerId = "worker-run-start-past-budget" ;
947+ const claimedEffect : EffectOutbox . OrchestrationEffectV2 = {
948+ ...restartEffect ( now , { type : "detach" } ) ,
949+ ...( effectType === "provider-turn.start"
950+ ? { request : { type : "provider-turn.start" as const , runId } }
951+ : { } ) ,
952+ attemptCount : 5 ,
953+ leaseOwner : workerId ,
954+ } ;
955+ const retryDelays = yield * Ref . make < ReadonlyArray < number > > ( [ ] ) ;
956+ const terminalizations = yield * Ref . make ( 0 ) ;
957+ const willRetry = yield * Ref . make < ReadonlyArray < boolean | undefined > > ( [ ] ) ;
958+ const outboxLayer = Layer . mock ( EffectOutbox . EffectOutboxV2 ) ( {
959+ claimNext : ( ) => Effect . succeed ( Option . some ( claimedEffect ) ) ,
960+ get : ( ) => Effect . succeed ( Option . some ( claimedEffect ) ) ,
961+ awaitCancellation : ( ) => Effect . never ,
962+ clearCancellation : ( ) => Effect . void ,
963+ retry : ( { delayMs } ) =>
964+ Ref . update ( retryDelays , ( delays ) => [ ...delays , delayMs ] ) . pipe ( Effect . as ( true ) ) ,
965+ fail : ( ) => Ref . update ( terminalizations , ( count ) => count + 1 ) . pipe ( Effect . as ( true ) ) ,
966+ } ) ;
967+ const executorLayer = Layer . succeed (
968+ EffectWorker . OrchestrationEffectExecutorV2 ,
969+ EffectWorker . OrchestrationEffectExecutorV2 . of ( {
970+ execute : ( _effect , options ) =>
971+ Ref . update ( willRetry , ( seen ) => [ ...seen , options ?. willRetry ] ) . pipe (
972+ Effect . andThen (
973+ Effect . fail (
974+ new EffectWorker . OrchestrationEffectExecutionError ( {
975+ effectId : claimedEffect . id ,
976+ effectType,
977+ cause : "simulated storage outage" ,
978+ } ) ,
979+ ) ,
980+ ) ,
981+ ) ,
982+ } ) ,
983+ ) ;
984+ const workerLayer = EffectWorker . layerWithOptions ( { workerId, maxAttempts : 5 } ) . pipe (
985+ Layer . provide ( Layer . merge ( outboxLayer , executorLayer ) ) ,
986+ ) ;
987+
988+ assert . isTrue (
989+ yield * EffectWorker . OrchestrationEffectWorkerV2 . pipe (
990+ Effect . flatMap ( ( worker ) => worker . runOnce ) ,
991+ Effect . provide ( workerLayer ) ,
992+ ) ,
993+ ) ;
994+
995+ // Each attempt past the budget is a last attempt: it starts the run or
996+ // fails it, and the effect itself is never abandoned.
997+ assert . deepEqual ( yield * Ref . get ( willRetry ) , [ false ] ) ;
998+ assert . deepEqual ( yield * Ref . get ( retryDelays ) , [ 1_600 ] ) ;
999+ assert . equal ( yield * Ref . get ( terminalizations ) , 0 ) ;
1000+ } ) ,
1001+ ) ;
0 commit comments