@@ -11,6 +11,7 @@ import {
1111 VcsProcessTimeoutError ,
1212 VcsUnsupportedOperationError ,
1313} from "@t3tools/contracts" ;
14+ import * as Cause from "effect/Cause" ;
1415import * as DateTime from "effect/DateTime" ;
1516import * as Deferred from "effect/Deferred" ;
1617import * as Effect from "effect/Effect" ;
@@ -164,12 +165,23 @@ it.effect(
164165 assert . strictEqual ( ( yield * Fiber . join ( postTurn ) ) . status , "ready" ) ;
165166 assert . strictEqual ( ( yield * Fiber . join ( waitingBaseline ) ) . _tag , "Success" ) ;
166167
167- const slowBaseline = yield * checkpoints
168- . captureBaseline ( { scope, ordinalWithinScope : 2 } )
168+ for ( const ordinalWithinScope of [ 2 , 3 ] ) {
169+ const slowBaseline = yield * checkpoints
170+ . captureBaseline ( { scope, ordinalWithinScope } )
171+ . pipe ( Effect . result , Effect . forkChild ) ;
172+ yield * TestClock . adjust ( "5 seconds" ) ;
173+ const result = yield * Fiber . join ( slowBaseline ) ;
174+ assert . strictEqual ( result . _tag , "Failure" ) ;
175+ if ( result . _tag === "Failure" ) {
176+ assert . isTrue ( Cause . isTimeoutError ( result . failure . cause ) ) ;
177+ }
178+ }
179+ const backedOffBaseline = yield * checkpoints
180+ . captureBaseline ( { scope, ordinalWithinScope : 4 } )
169181 . pipe ( Effect . result , Effect . forkChild ) ;
170182 yield * TestClock . adjust ( "5 seconds" ) ;
171- assert . strictEqual ( ( yield * Fiber . join ( slowBaseline ) ) . _tag , "Failure " ) ;
172- assert . strictEqual ( yield * Ref . get ( snapshots ) , 2 ) ;
183+ assert . strictEqual ( ( yield * Fiber . join ( backedOffBaseline ) ) . _tag , "Success " ) ;
184+ assert . strictEqual ( yield * Ref . get ( snapshots ) , 3 ) ;
173185 assert . strictEqual ( ( yield * Ref . get ( refs ) ) . size , 1 ) ;
174186 } ) . pipe ( Effect . provide ( layerTest ) ) ;
175187 } ) ,
@@ -233,3 +245,113 @@ it.effect("records a repository probe failure as an error checkpoint, not a miss
233245 assert . deepEqual ( statuses , { baseline : "error" , turn : "error" } ) ;
234246 } ) ,
235247) ;
248+
249+ it . effect ( "backs off only baseline retries after repeated workspace capture timeouts" , ( ) =>
250+ Effect . gen ( function * ( ) {
251+ const scope : OrchestrationV2CheckpointScope = {
252+ id : CheckpointScopeId . make ( "checkpoint-scope:slow-capture" ) ,
253+ threadId : ThreadId . make ( "thread:slow-capture" ) ,
254+ runId : RunId . make ( "run:slow-capture:1" ) ,
255+ nodeId : NodeId . make ( "node:slow-capture:1" ) ,
256+ parentScopeId : null ,
257+ providerThreadId : ProviderThreadId . make ( "provider-thread:slow-capture" ) ,
258+ kind : "root_run" ,
259+ ordinalWithinParent : 0 ,
260+ advancesAppRunCount : true ,
261+ cwd : "/repo/slow" ,
262+ createdAt : DateTime . makeUnsafe ( "2026-07-28T00:00:00.000Z" ) ,
263+ } ;
264+ const timeout = new VcsProcessTimeoutError ( {
265+ operation : "GitVcsDriver.checkpoints.captureCheckpoint" ,
266+ command : "git" ,
267+ cwd : scope . cwd ,
268+ timeoutMs : 30_000 ,
269+ } ) ;
270+ let attempts = 0 ;
271+ let healthy = false ;
272+ let nonTimeoutFailure = false ;
273+ const failure = new VcsProcessExitError ( {
274+ operation : "test.capture" ,
275+ command : "git" ,
276+ cwd : scope . cwd ,
277+ exitCode : 128 ,
278+ detail : "capture failed" ,
279+ } ) ;
280+ const layerTest = CheckpointService . layer . pipe (
281+ Layer . provide (
282+ Layer . mergeAll (
283+ IdAllocator . layer ,
284+ Layer . mock ( CheckpointStore . CheckpointStore ) ( {
285+ isGitRepository : ( ) => Effect . succeed ( true ) ,
286+ hasCheckpointRef : ( ) => Effect . succeed ( false ) ,
287+ captureCheckpoint : ( ) =>
288+ Effect . suspend ( ( ) => {
289+ attempts ++ ;
290+ return healthy ? Effect . void : Effect . fail ( nonTimeoutFailure ? failure : timeout ) ;
291+ } ) ,
292+ } ) ,
293+ ) ,
294+ ) ,
295+ Layer . provideMerge ( NodeCrypto . layer ) ,
296+ ) ;
297+ yield * Effect . gen ( function * ( ) {
298+ const checkpoints = yield * CheckpointService . CheckpointServiceV2 ;
299+ const baseline = { scope, ordinalWithinScope : 0 } ;
300+ const turn = {
301+ scope,
302+ ordinalWithinScope : 1 ,
303+ runId : scope . runId ,
304+ nodeId : scope . nodeId ,
305+ appRunOrdinal : 1 ,
306+ capturedAt : scope . createdAt ,
307+ } ;
308+ yield * Effect . result ( checkpoints . captureBaseline ( baseline ) ) ;
309+ assert . strictEqual ( ( yield * checkpoints . capture ( turn ) ) . status , "error" ) ;
310+ assert . strictEqual ( attempts , 2 ) ;
311+
312+ const otherThread = { ...scope , id : CheckpointScopeId . make ( "checkpoint-scope:other" ) } ;
313+ const skipped = yield * Effect . result (
314+ checkpoints . captureBaseline ( { scope : otherThread , ordinalWithinScope : 0 } ) ,
315+ ) ;
316+ assert . strictEqual ( attempts , 2 , "baseline retried a repeatedly timing-out workspace" ) ;
317+ assert . strictEqual ( skipped . _tag , "Success" ) ;
318+
319+ yield * Effect . result (
320+ checkpoints . captureBaseline ( {
321+ scope : { ...otherThread , cwd : "/repo/other" } ,
322+ ordinalWithinScope : 0 ,
323+ } ) ,
324+ ) ;
325+ assert . strictEqual ( attempts , 3 , "another workspace must still attempt capture" ) ;
326+
327+ const failedTurn = yield * checkpoints . capture ( turn ) ;
328+ assert . strictEqual ( failedTurn . status , "error" ) ;
329+ assert . strictEqual ( attempts , 4 , "post-turn capture must not back off" ) ;
330+
331+ yield * checkpoints . captureBaseline ( baseline ) ;
332+ assert . strictEqual ( attempts , 4 , "baseline retries must wait for post-turn recovery" ) ;
333+ healthy = true ;
334+ assert . strictEqual ( ( yield * checkpoints . capture ( turn ) ) . status , "ready" ) ;
335+ assert . strictEqual ( attempts , 5 ) ;
336+ yield * checkpoints . captureBaseline ( baseline ) ;
337+ assert . strictEqual ( attempts , 6 , "a successful post-turn capture must clear backoff" ) ;
338+
339+ healthy = false ;
340+ yield * Effect . result ( checkpoints . captureBaseline ( baseline ) ) ;
341+ yield * Effect . result ( checkpoints . captureBaseline ( baseline ) ) ;
342+ yield * checkpoints . captureBaseline ( baseline ) ;
343+ assert . strictEqual ( attempts , 8 ) ;
344+ nonTimeoutFailure = true ;
345+ assert . strictEqual ( ( yield * checkpoints . capture ( turn ) ) . status , "error" ) ;
346+ nonTimeoutFailure = false ;
347+ yield * Effect . result ( checkpoints . captureBaseline ( baseline ) ) ;
348+ assert . strictEqual ( attempts , 10 , "a non-timeout failure must clear backoff" ) ;
349+ healthy = true ;
350+ yield * checkpoints . captureBaseline ( baseline ) ;
351+ healthy = false ;
352+ yield * Effect . result ( checkpoints . captureBaseline ( baseline ) ) ;
353+ yield * Effect . result ( checkpoints . captureBaseline ( baseline ) ) ;
354+ assert . strictEqual ( attempts , 13 , "a successful baseline must clear the timeout streak" ) ;
355+ } ) . pipe ( Effect . provide ( layerTest ) ) ;
356+ } ) ,
357+ ) ;
0 commit comments