Skip to content

Commit 7bc4764

Browse files
fix(server): retry only transient checkpoint Git commands
1 parent 13a9336 commit 7bc4764

6 files changed

Lines changed: 355 additions & 238 deletions

File tree

‎apps/server/src/checkpointing/CheckpointStore.test.ts‎

Lines changed: 1 addition & 192 deletions
Original file line numberDiff line numberDiff line change
@@ -3,20 +3,11 @@ import * as NodePath from "node:path";
33

44
import * as NodeServices from "@effect/platform-node/NodeServices";
55
import { it } from "@effect/vitest";
6-
import {
7-
ThreadId,
8-
VcsProcessExitError,
9-
VcsProcessSpawnError,
10-
VcsProcessTimeoutError,
11-
VcsUnsupportedOperationError,
12-
type VcsError,
13-
} from "@t3tools/contracts";
6+
import { ThreadId, type VcsError } from "@t3tools/contracts";
147
import { HostProcessPlatform } from "@t3tools/shared/hostProcess";
15-
import * as DateTime from "effect/DateTime";
168
import * as Effect from "effect/Effect";
179
import * as FileSystem from "effect/FileSystem";
1810
import * as Layer from "effect/Layer";
19-
import * as Option from "effect/Option";
2011
import * as PlatformError from "effect/PlatformError";
2112
import * as Scope from "effect/Scope";
2213
import { describe, expect } from "vite-plus/test";
@@ -25,7 +16,6 @@ import { checkpointRefForThreadTurn } from "./Utils.ts";
2516
import { parseTurnDiffFilesFromNumstat } from "./Diffs.ts";
2617
import * as CheckpointStore from "./CheckpointStore.ts";
2718
import * as VcsDriverRegistry from "../vcs/VcsDriverRegistry.ts";
28-
import * as VcsDriver from "../vcs/VcsDriver.ts";
2919
import * as VcsProcess from "../vcs/VcsProcess.ts";
3020
import * as ServerConfig from "../config.ts";
3121

@@ -442,184 +432,3 @@ it.layer(TestLayer)("CheckpointStore.layer", (it) => {
442432
);
443433
});
444434
});
445-
446-
const RetryCwd = "/tmp/checkpoint-retry-test";
447-
448-
function makeCaptureExitError(): VcsProcessExitError {
449-
return new VcsProcessExitError({
450-
operation: "CheckpointStore.captureCheckpoint",
451-
command: "git",
452-
cwd: RetryCwd,
453-
exitCode: 128,
454-
detail: "Process exited with a non-zero status.",
455-
});
456-
}
457-
458-
function makeFakeRegistryLayer(
459-
capture: () => Effect.Effect<void, VcsError>,
460-
): Layer.Layer<VcsDriverRegistry.VcsDriverRegistry> {
461-
const driver = {
462-
checkpoints: {
463-
captureCheckpoint: () => capture(),
464-
},
465-
} as unknown as VcsDriver.VcsDriver["Service"];
466-
return Layer.succeed(
467-
VcsDriverRegistry.VcsDriverRegistry,
468-
VcsDriverRegistry.VcsDriverRegistry.of({
469-
get: () => Effect.die("not used in this test"),
470-
detect: () => Effect.succeed(null),
471-
resolve: () =>
472-
Effect.succeed({
473-
kind: "git",
474-
repository: {
475-
kind: "git",
476-
rootPath: RetryCwd,
477-
metadataPath: null,
478-
freshness: {
479-
source: "live-local",
480-
observedAt: DateTime.makeUnsafe(0),
481-
expiresAt: Option.none(),
482-
},
483-
},
484-
driver,
485-
}),
486-
}),
487-
);
488-
}
489-
490-
function makeStoreWithCapture(
491-
capture: () => Effect.Effect<void, VcsError>,
492-
): Effect.Effect<CheckpointStore.CheckpointStore["Service"], never, Scope.Scope> {
493-
return Effect.provide(
494-
CheckpointStore.CheckpointStore,
495-
CheckpointStore.layer.pipe(Layer.provide(makeFakeRegistryLayer(capture))),
496-
);
497-
}
498-
499-
describe("captureCheckpoint retry", () => {
500-
it.live("retries transient capture failures until they succeed", () =>
501-
Effect.scoped(
502-
Effect.gen(function* () {
503-
let attempts = 0;
504-
const error = makeCaptureExitError();
505-
const checkpointStore = yield* makeStoreWithCapture(() => {
506-
attempts += 1;
507-
return attempts < 3 ? Effect.fail(error) : Effect.void;
508-
});
509-
510-
yield* checkpointStore.captureCheckpoint({
511-
cwd: RetryCwd,
512-
checkpointRef: checkpointRefForThreadTurn(ThreadId.make("thread-capture-retry"), 0),
513-
});
514-
515-
expect(attempts).toBe(3);
516-
}),
517-
),
518-
);
519-
520-
it.live("gives up after retries and keeps the last failure", () =>
521-
Effect.scoped(
522-
Effect.gen(function* () {
523-
let attempts = 0;
524-
const error = makeCaptureExitError();
525-
const checkpointStore = yield* makeStoreWithCapture(() => {
526-
attempts += 1;
527-
return Effect.fail(error);
528-
});
529-
530-
const failure = yield* Effect.flip(
531-
checkpointStore.captureCheckpoint({
532-
cwd: RetryCwd,
533-
checkpointRef: checkpointRefForThreadTurn(ThreadId.make("thread-capture-retry"), 0),
534-
}),
535-
);
536-
537-
expect(attempts).toBe(3);
538-
expect(failure).toBe(error);
539-
}),
540-
),
541-
);
542-
543-
it.live("does not retry permanent unsupported-driver failures", () =>
544-
Effect.scoped(
545-
Effect.gen(function* () {
546-
let attempts = 0;
547-
const error = new VcsUnsupportedOperationError({
548-
operation: "CheckpointStore.captureCheckpoint",
549-
kind: "unknown",
550-
detail: "unknown driver does not implement checkpoint operations.",
551-
});
552-
const checkpointStore = yield* makeStoreWithCapture(() => {
553-
attempts += 1;
554-
return Effect.fail(error);
555-
});
556-
557-
const failure = yield* Effect.flip(
558-
checkpointStore.captureCheckpoint({
559-
cwd: RetryCwd,
560-
checkpointRef: checkpointRefForThreadTurn(ThreadId.make("thread-capture-retry"), 0),
561-
}),
562-
);
563-
564-
expect(attempts).toBe(1);
565-
expect(failure).toBe(error);
566-
}),
567-
),
568-
);
569-
570-
it.live("does not retry spawn failures", () =>
571-
Effect.scoped(
572-
Effect.gen(function* () {
573-
let attempts = 0;
574-
const error = new VcsProcessSpawnError({
575-
operation: "CheckpointStore.captureCheckpoint",
576-
command: "git",
577-
cwd: RetryCwd,
578-
cause: new Error("spawn EAGAIN"),
579-
});
580-
const checkpointStore = yield* makeStoreWithCapture(() => {
581-
attempts += 1;
582-
return Effect.fail(error);
583-
});
584-
585-
const failure = yield* Effect.flip(
586-
checkpointStore.captureCheckpoint({
587-
cwd: RetryCwd,
588-
checkpointRef: checkpointRefForThreadTurn(ThreadId.make("thread-capture-retry"), 0),
589-
}),
590-
);
591-
592-
expect(attempts).toBe(1);
593-
expect(failure).toBe(error);
594-
}),
595-
),
596-
);
597-
598-
it.live("does not retry timeout failures", () =>
599-
Effect.scoped(
600-
Effect.gen(function* () {
601-
let attempts = 0;
602-
const error = new VcsProcessTimeoutError({
603-
operation: "CheckpointStore.captureCheckpoint",
604-
command: "git",
605-
cwd: RetryCwd,
606-
timeoutMs: 30_000,
607-
});
608-
const checkpointStore = yield* makeStoreWithCapture(() => {
609-
attempts += 1;
610-
return Effect.fail(error);
611-
});
612-
613-
const failure = yield* Effect.flip(
614-
checkpointStore.captureCheckpoint({
615-
cwd: RetryCwd,
616-
checkpointRef: checkpointRefForThreadTurn(ThreadId.make("thread-capture-retry"), 0),
617-
}),
618-
);
619-
620-
expect(attempts).toBe(1);
621-
expect(failure).toBe(error);
622-
}),
623-
),
624-
);
625-
});

‎apps/server/src/checkpointing/CheckpointStore.ts‎

Lines changed: 3 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -13,18 +13,10 @@
1313
*
1414
* @module CheckpointStore
1515
*/
16-
import {
17-
VcsProcessExitError,
18-
VcsUnsupportedOperationError,
19-
type CheckpointRef,
20-
type VcsError,
21-
} from "@t3tools/contracts";
16+
import { VcsUnsupportedOperationError, type CheckpointRef } from "@t3tools/contracts";
2217
import * as Context from "effect/Context";
23-
import * as Duration from "effect/Duration";
2418
import * as Effect from "effect/Effect";
2519
import * as Layer from "effect/Layer";
26-
import * as Schedule from "effect/Schedule";
27-
import * as Schema from "effect/Schema";
2820

2921
import type { CheckpointStoreError } from "./Errors.ts";
3022
import type { VcsCheckpointOps } from "../vcs/VcsDriver.ts";
@@ -106,18 +98,6 @@ export class CheckpointStore extends Context.Service<
10698
}
10799
>()("t3/checkpointing/CheckpointStore") {}
108100

109-
/**
110-
* Checkpoint capture runs alongside concurrent git activity in the workspace
111-
* (agents writing files, colocated VCS exports, lock churn), so a non-zero git
112-
* exit during capture can be transient. Exit failures are the only errors worth
113-
* retrying: timeouts are repo-size problems whose 30s budget would triple at
114-
* the sequential checkpoint worker, and unsupported-driver and detection
115-
* failures are permanent.
116-
*/
117-
const isVcsProcessExitError = Schema.is(VcsProcessExitError);
118-
119-
const isTransientCaptureError = (error: VcsError): boolean => isVcsProcessExitError(error);
120-
121101
/** @public Service construction is part of the canonical Effect module API. */
122102
export const make = Effect.gen(function* () {
123103
const vcsRegistry = yield* VcsDriverRegistry.VcsDriverRegistry;
@@ -145,28 +125,8 @@ export const make = Effect.gen(function* () {
145125
const captureCheckpoint: CheckpointStore["Service"]["captureCheckpoint"] = Effect.fn(
146126
"captureCheckpoint",
147127
)(function* (input) {
148-
return yield* Effect.retry(
149-
Effect.gen(function* () {
150-
const checkpoints = yield* resolveCheckpoints(
151-
"CheckpointStore.captureCheckpoint",
152-
input.cwd,
153-
);
154-
return yield* checkpoints.captureCheckpoint(input);
155-
}).pipe(
156-
Effect.tapError((error) =>
157-
Effect.logDebug("checkpoint capture attempt failed", {
158-
cwd: input.cwd,
159-
checkpointRef: input.checkpointRef,
160-
detail: error.message,
161-
}),
162-
),
163-
),
164-
{
165-
times: 2,
166-
while: isTransientCaptureError,
167-
schedule: Schedule.spaced(Duration.millis(75)),
168-
},
169-
);
128+
const checkpoints = yield* resolveCheckpoints("CheckpointStore.captureCheckpoint", input.cwd);
129+
return yield* checkpoints.captureCheckpoint(input);
170130
});
171131

172132
const hasCheckpointRef: CheckpointStore["Service"]["hasCheckpointRef"] = Effect.fn(

0 commit comments

Comments
 (0)