Skip to content

Commit ffd0336

Browse files
committed
fix(server): skip baseline retries after repeated capture timeouts
1 parent 4e48d9e commit ffd0336

2 files changed

Lines changed: 178 additions & 32 deletions

File tree

‎apps/server/src/orchestration-v2/CheckpointService.test.ts‎

Lines changed: 126 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import {
1111
VcsProcessTimeoutError,
1212
VcsUnsupportedOperationError,
1313
} from "@t3tools/contracts";
14+
import * as Cause from "effect/Cause";
1415
import * as DateTime from "effect/DateTime";
1516
import * as Deferred from "effect/Deferred";
1617
import * as Effect from "effect/Effect";
@@ -164,12 +165,23 @@ it.effect(
164165
assert.strictEqual((yield* Fiber.join(postTurn)).status, "ready");
165166
assert.strictEqual((yield* Fiber.join(waitingBaseline))._tag, "Success");
166167

167-
const slowBaseline = yield* checkpoints
168-
.captureBaseline({ scope, ordinalWithinScope: 2 })
168+
for (const ordinalWithinScope of [2, 3]) {
169+
const slowBaseline = yield* checkpoints
170+
.captureBaseline({ scope, ordinalWithinScope })
171+
.pipe(Effect.result, Effect.forkChild);
172+
yield* TestClock.adjust("5 seconds");
173+
const result = yield* Fiber.join(slowBaseline);
174+
assert.strictEqual(result._tag, "Failure");
175+
if (result._tag === "Failure") {
176+
assert.isTrue(Cause.isTimeoutError(result.failure.cause));
177+
}
178+
}
179+
const backedOffBaseline = yield* checkpoints
180+
.captureBaseline({ scope, ordinalWithinScope: 4 })
169181
.pipe(Effect.result, Effect.forkChild);
170182
yield* TestClock.adjust("5 seconds");
171-
assert.strictEqual((yield* Fiber.join(slowBaseline))._tag, "Failure");
172-
assert.strictEqual(yield* Ref.get(snapshots), 2);
183+
assert.strictEqual((yield* Fiber.join(backedOffBaseline))._tag, "Success");
184+
assert.strictEqual(yield* Ref.get(snapshots), 3);
173185
assert.strictEqual((yield* Ref.get(refs)).size, 1);
174186
}).pipe(Effect.provide(layerTest));
175187
}),
@@ -233,3 +245,113 @@ it.effect("records a repository probe failure as an error checkpoint, not a miss
233245
assert.deepEqual(statuses, { baseline: "error", turn: "error" });
234246
}),
235247
);
248+
249+
it.effect("backs off only baseline retries after repeated workspace capture timeouts", () =>
250+
Effect.gen(function* () {
251+
const scope: OrchestrationV2CheckpointScope = {
252+
id: CheckpointScopeId.make("checkpoint-scope:slow-capture"),
253+
threadId: ThreadId.make("thread:slow-capture"),
254+
runId: RunId.make("run:slow-capture:1"),
255+
nodeId: NodeId.make("node:slow-capture:1"),
256+
parentScopeId: null,
257+
providerThreadId: ProviderThreadId.make("provider-thread:slow-capture"),
258+
kind: "root_run",
259+
ordinalWithinParent: 0,
260+
advancesAppRunCount: true,
261+
cwd: "/repo/slow",
262+
createdAt: DateTime.makeUnsafe("2026-07-28T00:00:00.000Z"),
263+
};
264+
const timeout = new VcsProcessTimeoutError({
265+
operation: "GitVcsDriver.checkpoints.captureCheckpoint",
266+
command: "git",
267+
cwd: scope.cwd,
268+
timeoutMs: 30_000,
269+
});
270+
let attempts = 0;
271+
let healthy = false;
272+
let nonTimeoutFailure = false;
273+
const failure = new VcsProcessExitError({
274+
operation: "test.capture",
275+
command: "git",
276+
cwd: scope.cwd,
277+
exitCode: 128,
278+
detail: "capture failed",
279+
});
280+
const layerTest = CheckpointService.layer.pipe(
281+
Layer.provide(
282+
Layer.mergeAll(
283+
IdAllocator.layer,
284+
Layer.mock(CheckpointStore.CheckpointStore)({
285+
isGitRepository: () => Effect.succeed(true),
286+
hasCheckpointRef: () => Effect.succeed(false),
287+
captureCheckpoint: () =>
288+
Effect.suspend(() => {
289+
attempts++;
290+
return healthy ? Effect.void : Effect.fail(nonTimeoutFailure ? failure : timeout);
291+
}),
292+
}),
293+
),
294+
),
295+
Layer.provideMerge(NodeCrypto.layer),
296+
);
297+
yield* Effect.gen(function* () {
298+
const checkpoints = yield* CheckpointService.CheckpointServiceV2;
299+
const baseline = { scope, ordinalWithinScope: 0 };
300+
const turn = {
301+
scope,
302+
ordinalWithinScope: 1,
303+
runId: scope.runId,
304+
nodeId: scope.nodeId,
305+
appRunOrdinal: 1,
306+
capturedAt: scope.createdAt,
307+
};
308+
yield* Effect.result(checkpoints.captureBaseline(baseline));
309+
assert.strictEqual((yield* checkpoints.capture(turn)).status, "error");
310+
assert.strictEqual(attempts, 2);
311+
312+
const otherThread = { ...scope, id: CheckpointScopeId.make("checkpoint-scope:other") };
313+
const skipped = yield* Effect.result(
314+
checkpoints.captureBaseline({ scope: otherThread, ordinalWithinScope: 0 }),
315+
);
316+
assert.strictEqual(attempts, 2, "baseline retried a repeatedly timing-out workspace");
317+
assert.strictEqual(skipped._tag, "Success");
318+
319+
yield* Effect.result(
320+
checkpoints.captureBaseline({
321+
scope: { ...otherThread, cwd: "/repo/other" },
322+
ordinalWithinScope: 0,
323+
}),
324+
);
325+
assert.strictEqual(attempts, 3, "another workspace must still attempt capture");
326+
327+
const failedTurn = yield* checkpoints.capture(turn);
328+
assert.strictEqual(failedTurn.status, "error");
329+
assert.strictEqual(attempts, 4, "post-turn capture must not back off");
330+
331+
yield* checkpoints.captureBaseline(baseline);
332+
assert.strictEqual(attempts, 4, "baseline retries must wait for post-turn recovery");
333+
healthy = true;
334+
assert.strictEqual((yield* checkpoints.capture(turn)).status, "ready");
335+
assert.strictEqual(attempts, 5);
336+
yield* checkpoints.captureBaseline(baseline);
337+
assert.strictEqual(attempts, 6, "a successful post-turn capture must clear backoff");
338+
339+
healthy = false;
340+
yield* Effect.result(checkpoints.captureBaseline(baseline));
341+
yield* Effect.result(checkpoints.captureBaseline(baseline));
342+
yield* checkpoints.captureBaseline(baseline);
343+
assert.strictEqual(attempts, 8);
344+
nonTimeoutFailure = true;
345+
assert.strictEqual((yield* checkpoints.capture(turn)).status, "error");
346+
nonTimeoutFailure = false;
347+
yield* Effect.result(checkpoints.captureBaseline(baseline));
348+
assert.strictEqual(attempts, 10, "a non-timeout failure must clear backoff");
349+
healthy = true;
350+
yield* checkpoints.captureBaseline(baseline);
351+
healthy = false;
352+
yield* Effect.result(checkpoints.captureBaseline(baseline));
353+
yield* Effect.result(checkpoints.captureBaseline(baseline));
354+
assert.strictEqual(attempts, 13, "a successful baseline must clear the timeout streak");
355+
}).pipe(Effect.provide(layerTest));
356+
}),
357+
);

‎apps/server/src/orchestration-v2/CheckpointService.ts‎

Lines changed: 52 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import {
99
RunId,
1010
ThreadId,
1111
} from "@t3tools/contracts";
12+
import * as Cache from "effect/Cache";
1213
import * as Context from "effect/Context";
1314
import * as Digest from "effect/crypto/Digest";
1415
import * as DateTime from "effect/DateTime";
@@ -17,6 +18,7 @@ import * as Base64Url from "effect/encoding/Base64Url";
1718
import * as Hex from "effect/encoding/Hex";
1819
import * as KeyedLock from "@t3tools/shared/KeyedLock";
1920
import * as Layer from "effect/Layer";
21+
import * as Option from "effect/Option";
2022
import * as Schema from "effect/Schema";
2123

2224
import { isGitImport, parseTurnDiffFilesFromNumstat } from "../checkpointing/Diffs.ts";
@@ -250,6 +252,28 @@ export const layer: Layer.Layer<
250252
const withWorkspaceLock = <A, E, R>(cwd: string, effect: Effect.Effect<A, E, R>) =>
251253
workspaceLocks.withLock(cwd, effect);
252254

255+
const captureTimeouts = yield* Cache.make<string, number>({
256+
capacity: 128,
257+
lookup: () => Effect.succeed(0),
258+
});
259+
// Both callers hold the workspace lock; post-turn captures always probe for recovery.
260+
const captureWorkspaceCheckpoint = Effect.fn("CheckpointService.captureWorkspaceCheckpoint")(
261+
function* (input: CheckpointStore.CaptureCheckpointInput, baseline = false) {
262+
const capture = checkpointStore.captureCheckpoint(input);
263+
yield* (baseline ? capture.pipe(Effect.timeout(BASELINE_CAPTURE_BUDGET)) : capture).pipe(
264+
Effect.tap(() => Cache.invalidate(captureTimeouts, input.cwd)),
265+
Effect.tapError((error) =>
266+
error._tag === "VcsProcessTimeoutError" || error._tag === "TimeoutError"
267+
? Effect.gen(function* () {
268+
const previous = yield* Cache.get(captureTimeouts, input.cwd);
269+
yield* Cache.set(captureTimeouts, input.cwd, previous + 1);
270+
})
271+
: Cache.invalidate(captureTimeouts, input.cwd),
272+
),
273+
);
274+
},
275+
);
276+
253277
// A failed probe is an operational error, not proof that the workspace has no repository.
254278
const probeRepository = (scope: OrchestrationV2CheckpointScope) =>
255279
checkpointStore.isGitRepository(scope.cwd).pipe(
@@ -284,17 +308,19 @@ export const layer: Layer.Layer<
284308
return;
285309
}
286310

287-
yield* checkpointStore
288-
.captureCheckpoint({
289-
cwd: input.scope.cwd,
290-
checkpointRef,
291-
})
292-
.pipe(
293-
Effect.timeout(BASELINE_CAPTURE_BUDGET),
294-
Effect.catchTags({
295-
VcsUnsupportedOperationError: () => Effect.void,
296-
}),
311+
const timeouts = yield* Cache.getOption(captureTimeouts, input.scope.cwd);
312+
if (Option.isSome(timeouts) && timeouts.value >= 2) {
313+
return yield* Effect.logWarning(
314+
"orchestration V2 checkpoint baseline skipped after repeated capture timeouts",
315+
{ scopeId: input.scope.id, ordinalWithinScope: input.ordinalWithinScope },
297316
);
317+
}
318+
319+
yield* captureWorkspaceCheckpoint({ cwd: input.scope.cwd, checkpointRef }, true).pipe(
320+
Effect.catchTags({
321+
VcsUnsupportedOperationError: () => Effect.void,
322+
}),
323+
);
298324
}),
299325
).pipe(
300326
Effect.mapError(
@@ -406,24 +432,22 @@ export const layer: Layer.Layer<
406432
});
407433
}
408434

409-
const captureStatus = yield* checkpointStore
410-
.captureCheckpoint({
411-
cwd: input.scope.cwd,
412-
checkpointRef,
413-
})
414-
.pipe(
415-
Effect.as("ready" as const),
416-
Effect.catchTags({
417-
VcsUnsupportedOperationError: () => Effect.succeed("missing" as const),
418-
}),
419-
Effect.catch((cause) =>
420-
Effect.logWarning("orchestration V2 checkpoint capture failed", {
421-
scopeId: input.scope.id,
422-
checkpointRef,
423-
cause: String(cause),
424-
}).pipe(Effect.as("error" as const)),
425-
),
426-
);
435+
const captureStatus = yield* captureWorkspaceCheckpoint({
436+
cwd: input.scope.cwd,
437+
checkpointRef,
438+
}).pipe(
439+
Effect.as("ready" as const),
440+
Effect.catchTags({
441+
VcsUnsupportedOperationError: () => Effect.succeed("missing" as const),
442+
}),
443+
Effect.catch((cause) =>
444+
Effect.logWarning("orchestration V2 checkpoint capture failed", {
445+
scopeId: input.scope.id,
446+
checkpointRef,
447+
cause: String(cause),
448+
}).pipe(Effect.as("error" as const)),
449+
),
450+
);
427451

428452
if (captureStatus !== "ready") {
429453
return makeCheckpoint({

0 commit comments

Comments
 (0)