@@ -9,8 +9,10 @@ import {
99 type OrchestrationV2Notification ,
1010} from "@t3tools/contracts" ;
1111import * as Deferred from "effect/Deferred" ;
12+ import * as Cause from "effect/Cause" ;
1213import * as DateTime from "effect/DateTime" ;
1314import * as Effect from "effect/Effect" ;
15+ import * as Exit from "effect/Exit" ;
1416import * as Layer from "effect/Layer" ;
1517import * as Option from "effect/Option" ;
1618import * as Queue from "effect/Queue" ;
@@ -103,6 +105,106 @@ function testLayer(input: {
103105}
104106
105107describe ( "ProviderContinuationService" , ( ) => {
108+ it . effect . each ( [ "dispatch" , "projection" ] as const ) (
109+ "bounds delegated completion retries after persistent %s failure" ,
110+ ( failure ) =>
111+ Effect . gen ( function * ( ) {
112+ const attempts = yield * Ref . make ( 0 ) ;
113+ const attemptEvents = yield * Queue . unbounded < number > ( ) ;
114+ const fail = Ref . updateAndGet ( attempts , ( count ) => count + 1 ) . pipe (
115+ Effect . tap ( ( count ) => Queue . offer ( attemptEvents , count ) ) ,
116+ Effect . andThen ( Effect . die ( "persistent continuation failure" ) ) ,
117+ ) ;
118+ const threads = Layer . mock ( ThreadManagementService . ThreadManagementService ) ( {
119+ getThreadRecords : ( ) =>
120+ failure === "projection" ? fail : Effect . succeed ( delegatedProjection ( ) ) ,
121+ dispatch : ( ) => fail ,
122+ } ) ;
123+ const worker = ProviderContinuationService . workerLive . pipe (
124+ Layer . provide (
125+ Layer . mergeAll ( IdAllocator . layer , ProviderContinuationRequests . layer , threads ) ,
126+ ) ,
127+ ) ;
128+ yield * Effect . gen ( function * ( ) {
129+ const requests = yield * ProviderContinuationRequests . ProviderContinuationRequests ;
130+ yield * requests . offer ( {
131+ threadId,
132+ providerThreadId,
133+ driver,
134+ detail : null ,
135+ delegatedCompletion : { parentRunId, generation : 1 , messageId : delegatedMessageId } ,
136+ } ) ;
137+ assert . equal ( yield * Queue . take ( attemptEvents ) , 1 ) ;
138+ for ( const delay of [ 100 , 200 , 400 , 800 , 1600 , 3200 , 5000 , 5000 ] ) {
139+ yield * TestClock . adjust ( `${ delay } millis` ) ;
140+ if ( failure === "projection" ) yield * Queue . take ( attemptEvents ) ;
141+ yield * Queue . take ( attemptEvents ) ;
142+ }
143+ const exhaustedCount = yield * Ref . get ( attempts ) ;
144+ assert . equal ( exhaustedCount , failure === "projection" ? 17 : 9 ) ;
145+ yield * TestClock . adjust ( "1 day" ) ;
146+ assert . equal ( yield * Ref . get ( attempts ) , exhaustedCount ) ;
147+ // A durable mailbox re-offer starts another bounded retry chain.
148+ yield * requests . offer ( {
149+ threadId,
150+ providerThreadId,
151+ driver,
152+ detail : null ,
153+ delegatedCompletion : { parentRunId, generation : 1 , messageId : delegatedMessageId } ,
154+ } ) ;
155+ assert . equal ( yield * Queue . take ( attemptEvents ) , exhaustedCount + 1 ) ;
156+ yield * TestClock . adjust ( "100 millis" ) ;
157+ if ( failure === "projection" ) yield * Queue . take ( attemptEvents ) ;
158+ yield * Queue . take ( attemptEvents ) ;
159+ assert . equal (
160+ yield * Ref . get ( attempts ) ,
161+ exhaustedCount + ( failure === "projection" ? 3 : 2 ) ,
162+ ) ;
163+ } ) . pipe (
164+ Effect . provide ( Layer . merge ( ProviderContinuationRequests . layer , worker ) ) ,
165+ Effect . scoped ,
166+ ) ;
167+ } ) ,
168+ ) ;
169+ it . effect ( "interrupts an in-flight delegated delivery when the worker scope closes" , ( ) =>
170+ Effect . gen ( function * ( ) {
171+ const attempts = yield * Ref . make ( 0 ) ;
172+ const started = yield * Deferred . make < void > ( ) ;
173+ const stopped = yield * Deferred . make < Exit . Exit < never > > ( ) ;
174+ const threads = Layer . mock ( ThreadManagementService . ThreadManagementService ) ( {
175+ getThreadRecords : ( ) => Effect . succeed ( delegatedProjection ( ) ) ,
176+ dispatch : ( ) =>
177+ Ref . update ( attempts , ( count ) => count + 1 ) . pipe (
178+ Effect . andThen ( Deferred . succeed ( started , undefined ) ) ,
179+ Effect . andThen ( Effect . never ) ,
180+ Effect . onExit ( ( exit ) => Deferred . succeed ( stopped , exit ) ) ,
181+ ) ,
182+ } ) ;
183+ const worker = ProviderContinuationService . workerLive . pipe (
184+ Layer . provide (
185+ Layer . mergeAll ( IdAllocator . layer , ProviderContinuationRequests . layer , threads ) ,
186+ ) ,
187+ ) ;
188+ yield * Effect . gen ( function * ( ) {
189+ const requests = yield * ProviderContinuationRequests . ProviderContinuationRequests ;
190+ yield * requests . offer ( {
191+ threadId,
192+ providerThreadId,
193+ driver,
194+ detail : null ,
195+ delegatedCompletion : { parentRunId, generation : 1 , messageId : delegatedMessageId } ,
196+ } ) ;
197+ yield * Deferred . await ( started ) ;
198+ } ) . pipe (
199+ Effect . provide ( Layer . merge ( ProviderContinuationRequests . layer , worker ) ) ,
200+ Effect . scoped ,
201+ ) ;
202+ const exit = yield * Deferred . await ( stopped ) ;
203+ assert . isTrue ( Exit . isFailure ( exit ) && Cause . hasInterruptsOnly ( exit . cause ) ) ;
204+ yield * TestClock . adjust ( "1 day" ) ;
205+ assert . equal ( yield * Ref . get ( attempts ) , 1 ) ;
206+ } ) ,
207+ ) ;
106208 it . effect ( "recovers an unaccepted persisted steer using the same delivery identity" , ( ) =>
107209 Effect . gen ( function * ( ) {
108210 const dispatched = yield * Queue . unbounded < unknown > ( ) ;
0 commit comments