@@ -83,10 +83,6 @@ function layerExecutorFor(input: {
8383 readonly threads ?: Partial < ThreadManagementService . ThreadManagementService [ "Service" ] > ;
8484 readonly continueAfterRestart ?: boolean ;
8585 readonly interrupt ?: ProviderTurnControlService . ProviderTurnControlServiceV2Shape [ "interrupt" ] ;
86- readonly failRunFailure ?: boolean ;
87- readonly failRunStateMissing ?: boolean ;
88- readonly interruptFailure ?: boolean ;
89- readonly startInterrupted ?: boolean ;
9086} ) {
9187 const record = ( event : string ) => Ref . update ( input . events , ( events ) => [ ...events , event ] ) ;
9288 const layerDependencies = Layer . mergeAll (
@@ -100,19 +96,6 @@ function layerExecutorFor(input: {
10096 request . replacementProviderSessionId === undefined
10197 ? "interrupt"
10298 : `interrupt:${ request . replacementProviderSessionId } ` ,
103- ) . pipe (
104- Effect . andThen (
105- input . interruptFailure === true
106- ? Effect . fail (
107- new ProviderTurnControlService . ProviderTurnControlError ( {
108- threadId,
109- operation : "interrupt" ,
110- providerTurnId,
111- cause : "simulated interrupt failure" ,
112- } ) ,
113- )
114- : Effect . void ,
115- ) ,
11699 ) ,
117100 } ) ,
118101 ) ,
@@ -132,10 +115,10 @@ function layerExecutorFor(input: {
132115 Layer . succeed (
133116 ProviderTurnStartService . ProviderTurnStartServiceV2 ,
134117 ProviderTurnStartService . ProviderTurnStartServiceV2 . of ( {
118+ failStartingRun : ( ) => Effect . void ,
135119 start : ( ) =>
136120 Effect . gen ( function * ( ) {
137121 yield * record ( "start" ) ;
138- if ( input . startInterrupted === true ) return yield * Effect . interrupt ;
139122 if (
140123 input . failFirstStart !== undefined &&
141124 ( yield * Ref . getAndSet ( input . failFirstStart , false ) )
@@ -146,21 +129,6 @@ function layerExecutorFor(input: {
146129 } ) ;
147130 }
148131 } ) ,
149- failStartingRun : ( ) =>
150- Effect . gen ( function * ( ) {
151- yield * record ( "fail-run" ) ;
152- if ( input . failRunStateMissing === true ) {
153- return yield * new ProviderTurnStartService . ProviderTurnStartRunStateMissingError ( {
154- runId,
155- } ) ;
156- }
157- if ( input . failRunFailure === true ) {
158- return yield * new ProviderTurnStartService . ProviderTurnStartError ( {
159- runId,
160- cause : "simulated run failure write" ,
161- } ) ;
162- }
163- } ) ,
164132 } ) ,
165133 ) ,
166134 Layer . succeed (
@@ -937,201 +905,3 @@ it.effect.each(["confirmed", "unconfirmed", "failed"] as const)(
937905 ) ;
938906 } ) ,
939907) ;
940-
941- function startEffect ( now : DateTime . Utc ) : EffectOutbox . OrchestrationEffectV2 {
942- return {
943- ...restartEffect ( now , { type : "detach" } ) ,
944- id : "effect:start" ,
945- request : { type : "provider-turn.start" , runId } ,
946- } ;
947- }
948-
949- it . effect ( "fails the run when the last start attempt fails" , ( ) =>
950- Effect . gen ( function * ( ) {
951- const now = yield * DateTime . now ;
952- const events = yield * Ref . make < ReadonlyArray < string > > ( [ ] ) ;
953- const failFirstStart = yield * Ref . make ( true ) ;
954-
955- yield * Effect . gen ( function * ( ) {
956- const executor = yield * EffectWorker . OrchestrationEffectExecutorV2 ;
957- yield * executor . execute ( startEffect ( now ) , { willRetry : false } ) ;
958- } ) . pipe ( Effect . provide ( layerExecutorFor ( { events, failFirstStart } ) ) ) ;
959-
960- assert . deepEqual ( yield * Ref . get ( events ) , [ "start" , "fail-run" ] ) ;
961- } ) ,
962- ) ;
963-
964- it . effect ( "fails the run when a restart's last attempt cannot interrupt the old turn" , ( ) =>
965- Effect . gen ( function * ( ) {
966- const now = yield * DateTime . now ;
967- const events = yield * Ref . make < ReadonlyArray < string > > ( [ ] ) ;
968- const effect = restartEffect ( now , { type : "detach" } ) ;
969-
970- yield * Effect . gen ( function * ( ) {
971- const executor = yield * EffectWorker . OrchestrationEffectExecutorV2 ;
972- yield * executor . execute ( effect , { willRetry : false } ) ;
973- } ) . pipe ( Effect . provide ( layerExecutorFor ( { events, interruptFailure : true } ) ) ) ;
974-
975- assert . deepEqual ( yield * Ref . get ( events ) , [ "interrupt" , "fail-run" ] ) ;
976- } ) ,
977- ) ;
978-
979- it . effect ( "does not fail the run when its last start attempt is interrupted" , ( ) =>
980- Effect . gen ( function * ( ) {
981- const now = yield * DateTime . now ;
982- const events = yield * Ref . make < ReadonlyArray < string > > ( [ ] ) ;
983-
984- const exit = yield * Effect . gen ( function * ( ) {
985- const executor = yield * EffectWorker . OrchestrationEffectExecutorV2 ;
986- return yield * Effect . exit ( executor . execute ( startEffect ( now ) , { willRetry : false } ) ) ;
987- } ) . pipe ( Effect . provide ( layerExecutorFor ( { events, startInterrupted : true } ) ) ) ;
988-
989- assert . isTrue ( Exit . isFailure ( exit ) && Cause . hasInterruptsOnly ( exit . cause ) ) ;
990- assert . deepEqual ( yield * Ref . get ( events ) , [ "start" ] ) ;
991- } ) ,
992- ) ;
993-
994- it . effect ( "returns the start failure when the run cannot be failed either" , ( ) =>
995- Effect . gen ( function * ( ) {
996- const now = yield * DateTime . now ;
997- const events = yield * Ref . make < ReadonlyArray < string > > ( [ ] ) ;
998- const failFirstStart = yield * Ref . make ( true ) ;
999- const effect = restartEffect ( now , { type : "detach" } ) ;
1000-
1001- const error = yield * Effect . gen ( function * ( ) {
1002- const executor = yield * EffectWorker . OrchestrationEffectExecutorV2 ;
1003- return yield * executor . execute ( effect , { willRetry : false } ) . pipe ( Effect . flip ) ;
1004- } ) . pipe ( Effect . provide ( layerExecutorFor ( { events, failFirstStart, failRunFailure : true } ) ) ) ;
1005-
1006- assert . deepEqual ( yield * Ref . get ( events ) , [ "interrupt" , "detach" , "start" , "fail-run" ] ) ;
1007- assert . include ( Cause . pretty ( Cause . fail ( error ) ) , "simulated first start failure" ) ;
1008- } ) ,
1009- ) ;
1010-
1011- it . effect ( "returns an unsettleable run instead of the start failure" , ( ) =>
1012- Effect . gen ( function * ( ) {
1013- const now = yield * DateTime . now ;
1014- const events = yield * Ref . make < ReadonlyArray < string > > ( [ ] ) ;
1015- const failFirstStart = yield * Ref . make ( true ) ;
1016-
1017- const error = yield * Effect . gen ( function * ( ) {
1018- const executor = yield * EffectWorker . OrchestrationEffectExecutorV2 ;
1019- return yield * executor . execute ( startEffect ( now ) , { willRetry : false } ) . pipe ( Effect . flip ) ;
1020- } ) . pipe (
1021- Effect . provide ( layerExecutorFor ( { events, failFirstStart, failRunStateMissing : true } ) ) ,
1022- ) ;
1023-
1024- assert . deepEqual ( yield * Ref . get ( events ) , [ "start" , "fail-run" ] ) ;
1025- assert . isTrue ( ProviderTurnStartService . isProviderTurnStartRunStateMissingError ( error . cause ) ) ;
1026- } ) ,
1027- ) ;
1028-
1029- it . effect . each ( [ "provider-turn.start" , "provider-turn.restart" ] as const ) (
1030- "keeps retrying a %s effect past its attempt budget" ,
1031- ( effectType ) =>
1032- Effect . gen ( function * ( ) {
1033- const now = yield * DateTime . now ;
1034- const workerId = "worker-run-start-past-budget" ;
1035- const claimedEffect : EffectOutbox . OrchestrationEffectV2 = {
1036- ...restartEffect ( now , { type : "detach" } ) ,
1037- ...( effectType === "provider-turn.start"
1038- ? { request : { type : "provider-turn.start" as const , runId } }
1039- : { } ) ,
1040- attemptCount : 5 ,
1041- leaseOwner : workerId ,
1042- } ;
1043- const retryDelays = yield * Ref . make < ReadonlyArray < number > > ( [ ] ) ;
1044- const terminalizations = yield * Ref . make ( 0 ) ;
1045- const willRetry = yield * Ref . make < ReadonlyArray < boolean | undefined > > ( [ ] ) ;
1046- const outboxLayer = Layer . mock ( EffectOutbox . EffectOutboxV2 ) ( {
1047- claimNext : ( ) => Effect . succeed ( Option . some ( claimedEffect ) ) ,
1048- get : ( ) => Effect . succeed ( Option . some ( claimedEffect ) ) ,
1049- awaitCancellation : ( ) => Effect . never ,
1050- clearCancellation : ( ) => Effect . void ,
1051- retry : ( { delayMs } ) =>
1052- Ref . update ( retryDelays , ( delays ) => [ ...delays , delayMs ] ) . pipe ( Effect . as ( true ) ) ,
1053- fail : ( ) => Ref . update ( terminalizations , ( count ) => count + 1 ) . pipe ( Effect . as ( true ) ) ,
1054- } ) ;
1055- const executorLayer = Layer . succeed (
1056- EffectWorker . OrchestrationEffectExecutorV2 ,
1057- EffectWorker . OrchestrationEffectExecutorV2 . of ( {
1058- execute : ( _effect , options ) =>
1059- Ref . update ( willRetry , ( seen ) => [ ...seen , options ?. willRetry ] ) . pipe (
1060- Effect . andThen (
1061- Effect . fail (
1062- new EffectWorker . OrchestrationEffectExecutionError ( {
1063- effectId : claimedEffect . id ,
1064- effectType,
1065- cause : "simulated storage outage" ,
1066- } ) ,
1067- ) ,
1068- ) ,
1069- ) ,
1070- } ) ,
1071- ) ;
1072- const workerLayer = EffectWorker . layerWithOptions ( { workerId, maxAttempts : 5 } ) . pipe (
1073- Layer . provide ( Layer . merge ( outboxLayer , executorLayer ) ) ,
1074- ) ;
1075-
1076- assert . isTrue (
1077- yield * EffectWorker . OrchestrationEffectWorkerV2 . pipe (
1078- Effect . flatMap ( ( worker ) => worker . runOnce ) ,
1079- Effect . provide ( workerLayer ) ,
1080- ) ,
1081- ) ;
1082-
1083- // Each attempt past the budget is a last attempt: it starts the run or
1084- // fails it, and the effect itself is never abandoned.
1085- assert . deepEqual ( yield * Ref . get ( willRetry ) , [ false ] ) ;
1086- assert . deepEqual ( yield * Ref . get ( retryDelays ) , [ 1_600 ] ) ;
1087- assert . equal ( yield * Ref . get ( terminalizations ) , 0 ) ;
1088- } ) ,
1089- ) ;
1090-
1091- it . effect ( "fails a run-start effect for good when its run has no state left to settle" , ( ) =>
1092- Effect . gen ( function * ( ) {
1093- const now = yield * DateTime . now ;
1094- const workerId = "worker-run-start-unsettleable" ;
1095- const claimedEffect : EffectOutbox . OrchestrationEffectV2 = {
1096- ...startEffect ( now ) ,
1097- attemptCount : 5 ,
1098- leaseOwner : workerId ,
1099- } ;
1100- const retries = yield * Ref . make ( 0 ) ;
1101- const terminalizations = yield * Ref . make ( 0 ) ;
1102- const outboxLayer = Layer . mock ( EffectOutbox . EffectOutboxV2 ) ( {
1103- claimNext : ( ) => Effect . succeed ( Option . some ( claimedEffect ) ) ,
1104- get : ( ) => Effect . succeed ( Option . some ( claimedEffect ) ) ,
1105- awaitCancellation : ( ) => Effect . never ,
1106- clearCancellation : ( ) => Effect . void ,
1107- retry : ( ) => Ref . update ( retries , ( count ) => count + 1 ) . pipe ( Effect . as ( true ) ) ,
1108- fail : ( ) => Ref . update ( terminalizations , ( count ) => count + 1 ) . pipe ( Effect . as ( true ) ) ,
1109- } ) ;
1110- const executorLayer = Layer . succeed (
1111- EffectWorker . OrchestrationEffectExecutorV2 ,
1112- EffectWorker . OrchestrationEffectExecutorV2 . of ( {
1113- execute : ( ) =>
1114- Effect . fail (
1115- new EffectWorker . OrchestrationEffectExecutionError ( {
1116- effectId : claimedEffect . id ,
1117- effectType : claimedEffect . request . type ,
1118- cause : new ProviderTurnStartService . ProviderTurnStartRunStateMissingError ( { runId } ) ,
1119- } ) ,
1120- ) ,
1121- } ) ,
1122- ) ;
1123- const workerLayer = EffectWorker . layerWithOptions ( { workerId, maxAttempts : 5 } ) . pipe (
1124- Layer . provide ( Layer . merge ( outboxLayer , executorLayer ) ) ,
1125- ) ;
1126-
1127- assert . isTrue (
1128- yield * EffectWorker . OrchestrationEffectWorkerV2 . pipe (
1129- Effect . flatMap ( ( worker ) => worker . runOnce ) ,
1130- Effect . provide ( workerLayer ) ,
1131- ) ,
1132- ) ;
1133-
1134- assert . equal ( yield * Ref . get ( retries ) , 0 ) ;
1135- assert . equal ( yield * Ref . get ( terminalizations ) , 1 ) ;
1136- } ) ,
1137- ) ;
0 commit comments