Skip to content

Commit 869347b

Browse files
saphidgithub-actions[bot]juliusmarminge
authored
fix(server): retry transient git failures during checkpoint capture (pingdotgg#11665)
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Julius Marminge <[email protected]>
1 parent e84f23a commit 869347b

5 files changed

Lines changed: 348 additions & 4 deletions

File tree

‎apps/server/src/vcs/GitVcsDriver.test.ts‎

Lines changed: 140 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
import * as NodeServices from "@effect/platform-node/NodeServices";
22
import * as Cause from "effect/Cause";
3+
import * as Clock from "effect/Clock";
4+
import * as Duration from "effect/Duration";
35
import * as Exit from "effect/Exit";
46
import * as Deferred from "effect/Deferred";
57
import * as Effect from "effect/Effect";
@@ -14,7 +16,11 @@ import { assert, it } from "@effect/vitest";
1416

1517
import { CheckpointRef, GitCommandError, VcsProcessExitError } from "@t3tools/contracts";
1618
import * as ServerConfig from "../config.ts";
19+
import * as CheckpointStore from "../checkpointing/CheckpointStore.ts";
20+
import * as ProcessRunner from "../processRunner.ts";
1721
import * as GitVcsDriver from "./GitVcsDriver.ts";
22+
import type * as VcsDriver from "./VcsDriver.ts";
23+
import * as VcsDriverRegistry from "./VcsDriverRegistry.ts";
1824
import * as VcsProcess from "./VcsProcess.ts";
1925
import { runVcsDriverContractSuite } from "./testing/VcsDriverContractHarness.ts";
2026

@@ -26,6 +32,10 @@ const GitContractLayer = Layer.mergeAll(GitVcsDriver.vcsLayer, GitVcsDriver.laye
2632
Layer.provideMerge(VcsProcess.layer),
2733
Layer.provideMerge(NodeServices.layer),
2834
);
35+
const GitCaptureContractLayer = Layer.merge(
36+
GitContractLayer,
37+
ProcessRunner.layer.pipe(Layer.provide(NodeServices.layer)),
38+
);
2939

3040
const runGit = (cwd: string, args: ReadonlyArray<string>) =>
3141
Effect.gen(function* () {
@@ -38,6 +48,22 @@ const runGit = (cwd: string, args: ReadonlyArray<string>) =>
3848
});
3949
});
4050

51+
const makeCaptureStore = Effect.fn("test.makeCaptureStore")(function* (
52+
driver: VcsDriver.VcsDriver["Service"],
53+
cwd: string,
54+
) {
55+
const repository = yield* driver.detectRepository(cwd);
56+
if (repository === null) return yield* Effect.die("Expected a test Git repository");
57+
const handle = { kind: repository.kind, repository, driver };
58+
return yield* CheckpointStore.make.pipe(
59+
Effect.provideService(VcsDriverRegistry.VcsDriverRegistry, {
60+
get: () => Effect.succeed(driver),
61+
detect: () => Effect.succeed(handle),
62+
resolve: () => Effect.succeed(handle),
63+
}),
64+
);
65+
});
66+
4167
type GitContractError = GitCommandError | PlatformError.PlatformError;
4268

4369
runVcsDriverContractSuite<GitVcsDriver.GitVcsDriver, GitContractError>({
@@ -260,6 +286,112 @@ it.effect("checkpoint recovery refuses excessive candidates before probing", ()
260286
}).pipe(Effect.scoped, Effect.provide(GitContractLayer)),
261287
);
262288

289+
it.effect.each([
290+
{ phase: "add", nestedRecovery: false, expireRecovery: false },
291+
{ phase: "update-ref", nestedRecovery: false, expireRecovery: false },
292+
{ phase: "update-ref", nestedRecovery: true, expireRecovery: false },
293+
{ phase: "add", nestedRecovery: true, expireRecovery: false },
294+
{ phase: "add", nestedRecovery: true, expireRecovery: true },
295+
])(
296+
"checkpoint handles a $phase lock with nestedRecovery=$nestedRecovery, expireRecovery=$expireRecovery",
297+
({ phase, nestedRecovery, expireRecovery }) =>
298+
Effect.gen(function* () {
299+
const fs = yield* FileSystem.FileSystem;
300+
const path = yield* Path.Path;
301+
const liveRunner = yield* ProcessRunner.ProcessRunner;
302+
const driver = yield* GitVcsDriver.makeVcsDriverShape();
303+
const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-checkpoint-ref-race-" });
304+
const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd);
305+
if (nestedRecovery) yield* git(["init", "empty"]);
306+
const originalIndex = yield* fs.readFile(path.join(cwd, ".git", "index"));
307+
const refLockPath = path.join(cwd, ".git", `${checkpointRef}.lock`);
308+
const failed = yield* Deferred.make<void>();
309+
const retryReached = yield* Deferred.make<void>();
310+
const allowRetry = yield* Deferred.make<void>();
311+
const clock = yield* Clock.Clock;
312+
const privateIndexes = new Set<string>();
313+
let racedAttempts = 0;
314+
let discoveries = 0;
315+
let stageError: VcsProcessExitError | undefined;
316+
const captureProcess = yield* VcsProcess.make.pipe(
317+
Effect.provideService(ProcessRunner.ProcessRunner, {
318+
run: (input) => {
319+
if (input.args.includes("--others")) discoveries += 1;
320+
if (input.env?.GIT_INDEX_FILE) privateIndexes.add(input.env.GIT_INDEX_FILE);
321+
const initialStage =
322+
phase === "add" &&
323+
nestedRecovery &&
324+
!input.args.some((arg) => arg.startsWith(":(exclude,literal)"));
325+
if (!input.args.includes(phase) || initialStage || ++racedAttempts !== 1) {
326+
return liveRunner.run(input);
327+
}
328+
const lockPath = phase === "add" ? `${input.env!.GIT_INDEX_FILE!}.lock` : refLockPath;
329+
return Effect.gen(function* () {
330+
yield* fs
331+
.makeDirectory(path.dirname(lockPath), { recursive: true })
332+
.pipe(Effect.orDie);
333+
yield* fs.writeFileString(lockPath, "concurrent ref writer").pipe(Effect.orDie);
334+
return yield* liveRunner.run(input).pipe(
335+
Effect.ensuring(fs.remove(lockPath).pipe(Effect.orDie)),
336+
Effect.tap(() => Deferred.succeed(failed, undefined)),
337+
);
338+
});
339+
},
340+
}),
341+
);
342+
const captureDriver = yield* GitVcsDriver.makeVcsDriverShape().pipe(
343+
Effect.provideService(VcsProcess.VcsProcess, {
344+
run: (input) =>
345+
captureProcess.run(input).pipe(
346+
Effect.tapError((error) => {
347+
if (input.args.includes("add") && error._tag === "VcsProcessExitError")
348+
stageError = error;
349+
return Effect.void;
350+
}),
351+
),
352+
}),
353+
);
354+
const captureStore = yield* makeCaptureStore(captureDriver, cwd);
355+
const fiber = yield* captureStore.captureCheckpoint({ cwd, checkpointRef }).pipe(
356+
Effect.provideService(Clock.Clock, {
357+
...clock,
358+
sleep: (duration) =>
359+
Duration.toMillis(duration) === 75
360+
? Deferred.succeed(retryReached, undefined).pipe(
361+
Effect.andThen(Deferred.await(allowRetry)),
362+
)
363+
: clock.sleep(duration),
364+
}),
365+
Effect.exit,
366+
Effect.forkScoped,
367+
);
368+
yield* Deferred.await(failed);
369+
yield* Deferred.await(retryReached);
370+
if (expireRecovery) yield* TestClock.adjust("5 seconds");
371+
else yield* Deferred.succeed(allowRetry, undefined);
372+
const result = yield* Fiber.join(fiber);
373+
if (expireRecovery) {
374+
if (Exit.isSuccess(result))
375+
return yield* Effect.die("Expected the recovery deadline to expire");
376+
const error = Cause.findErrorOption(result.cause);
377+
assert.isTrue(error._tag === "Some");
378+
if (error._tag === "Some") assert.strictEqual(error.value, stageError);
379+
} else assert.isTrue(Exit.isSuccess(result));
380+
assert.strictEqual(racedAttempts, expireRecovery ? 1 : 2);
381+
assert.strictEqual(discoveries, nestedRecovery ? 1 : 0);
382+
assert.strictEqual(privateIndexes.size, 1);
383+
assert.strictEqual(
384+
yield* driver.checkpoints.hasCheckpointRef({ cwd, checkpointRef }),
385+
!expireRecovery,
386+
);
387+
for (const index of privateIndexes) {
388+
assert.isFalse(yield* fs.exists(index));
389+
assert.isFalse(yield* fs.exists(`${index}.lock`));
390+
}
391+
assert.deepEqual(yield* fs.readFile(path.join(cwd, ".git", "index")), originalIndex);
392+
}).pipe(Effect.scoped, Effect.provide(GitCaptureContractLayer)),
393+
);
394+
263395
for (const blockedPhase of ["discovery", "probe", "retry"] as const) {
264396
it.effect(`checkpoint recovery has one deadline including ${blockedPhase}`, () =>
265397
Effect.gen(function* () {
@@ -276,11 +408,15 @@ for (const blockedPhase of ["discovery", "probe", "retry"] as const) {
276408
let stageError: VcsProcessExitError | undefined;
277409
let privateIndex: string | undefined;
278410
let interrupted = false;
411+
let stagingAttempts = 0;
279412
const captureDriver = yield* GitVcsDriver.makeVcsDriverShape().pipe(
280413
Effect.provideService(VcsProcess.VcsProcess, {
281414
run: (input) => {
282415
const staging = input.args.includes("add") && input.args.includes("-A");
283-
if (staging) privateIndex = input.env?.GIT_INDEX_FILE;
416+
if (staging) {
417+
privateIndex = input.env?.GIT_INDEX_FILE;
418+
stagingAttempts += 1;
419+
}
284420
const block =
285421
(blockedPhase === "discovery" && input.args.includes("--others")) ||
286422
(blockedPhase === "probe" && input.cwd !== cwd && input.args.includes("rev-parse")) ||
@@ -322,7 +458,8 @@ for (const blockedPhase of ["discovery", "probe", "retry"] as const) {
322458
},
323459
}),
324460
);
325-
const fiber = yield* captureDriver.checkpoints
461+
const captureStore = yield* makeCaptureStore(captureDriver, cwd);
462+
const fiber = yield* captureStore
326463
.captureCheckpoint({ cwd, checkpointRef })
327464
.pipe(Effect.flip, Effect.forkScoped);
328465
if (blockedPhase === "probe") {
@@ -333,6 +470,7 @@ for (const blockedPhase of ["discovery", "probe", "retry"] as const) {
333470
yield* TestClock.adjust(blockedPhase === "probe" ? "2 seconds" : "5 seconds");
334471
const error = yield* Fiber.join(fiber);
335472
assert.strictEqual(error, stageError);
473+
assert.strictEqual(stagingAttempts, blockedPhase === "retry" ? 2 : 1);
336474
assert.isTrue(interrupted);
337475
assert.isDefined(privateIndex);
338476
assert.isFalse(yield* fs.exists(privateIndex!));

‎apps/server/src/vcs/GitVcsDriver.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -766,7 +766,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
766766

767767
const checkpoints: VcsDriver.VcsCheckpointOps = {
768768
captureCheckpoint: Effect.fn("GitVcsDriver.checkpoints.captureCheckpoint")(function* (input) {
769-
const operation = "GitVcsDriver.checkpoints.captureCheckpoint";
769+
const operation = VcsProcess.CHECKPOINT_CAPTURE_OPERATION;
770770
const indexConfig = [
771771
"-c",
772772
"core.fsmonitor=false",

0 commit comments

Comments
 (0)