Skip to content

Commit 8481fae

Browse files
committed
fix(server): bound status fetches and preserve late failure backoff
1 parent a1494c7 commit 8481fae

3 files changed

Lines changed: 296 additions & 22 deletions

File tree

‎apps/server/src/vcs/GitVcsDriver.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ export interface ExecuteGitInput {
5050
readonly env?: NodeJS.ProcessEnv;
5151
readonly allowNonZeroExit?: boolean;
5252
readonly timeoutMs?: number | null;
53+
readonly forceKillAfterMs?: number;
5354
readonly maxOutputBytes?: number;
5455
readonly appendTruncationMarker?: boolean;
5556
/**

‎apps/server/src/vcs/GitVcsDriverCore.test.ts‎

Lines changed: 248 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
import * as NodeFS from "node:fs";
33
import * as NodeServices from "@effect/platform-node/NodeServices";
44
import * as HostProcess from "@t3tools/shared/HostProcess";
5+
import { writeFakeCli } from "@t3tools/provider-testing/fakeCli";
56
import { assert, it, describe } from "@effect/vitest";
67
import * as Deferred from "effect/Deferred";
78
import * as Effect from "effect/Effect";
@@ -39,6 +40,7 @@ import {
3940
} from "./GitVcsDriverCore.ts";
4041
import * as GitVcsDriver from "./GitVcsDriver.ts";
4142
import { gitConfigEnv } from "./VcsProcess.ts";
43+
import { makeProcessProbe, processProbeSource } from "../testUtils/processProbe.ts";
4244

4345
const encodeGitCommandError = Schema.encodeEffect(Schema.fromJsonString(GitCommandError));
4446

@@ -493,6 +495,252 @@ it.effect("coalesces concurrent ref pages into one repository snapshot", () =>
493495
);
494496

495497
describe("pullCurrentBranch", () => {
498+
const controlledStatusFetch = Effect.gen(function* () {
499+
const fs = yield* FileSystem.FileSystem;
500+
const cwd = yield* makeTmpDir();
501+
yield* fs.makeDirectory(`${cwd}/.git`);
502+
const started = yield* Queue.unbounded<{
503+
cwd: string;
504+
release: Deferred.Deferred<ChildProcessSpawner.ExitCode>;
505+
}>();
506+
const finalized = yield* Queue.unbounded<string>();
507+
const warnings = yield* Queue.unbounded<void>();
508+
const attempts = yield* Ref.make(0);
509+
const pulls = yield* Ref.make(0);
510+
const fetching = yield* Ref.make(false);
511+
const head = yield* Ref.make("before");
512+
const pullRemoteRead = yield* Deferred.make<void>();
513+
const logger = Logger.make<unknown, void>(() => {
514+
Queue.offerUnsafe(warnings, undefined);
515+
});
516+
const spawner = ChildProcessSpawner.make((command) =>
517+
Effect.gen(function* () {
518+
if (!ChildProcess.isStandardCommand(command)) return makeNonRepositoryHandle();
519+
const workingDirectory = command.options.cwd ?? cwd;
520+
const args = command.args;
521+
if (args.includes("fetch")) {
522+
const release = yield* Deferred.make<ChildProcessSpawner.ExitCode>();
523+
yield* Ref.set(fetching, true);
524+
yield* Ref.update(attempts, (count) => count + 1);
525+
yield* Queue.offer(started, { cwd: workingDirectory, release });
526+
yield* Effect.addFinalizer(() => Queue.offer(finalized, workingDirectory));
527+
yield* Effect.addFinalizer(() => Ref.set(fetching, false));
528+
return ChildProcessSpawner.makeHandle({
529+
...makeSuccessfulHandle(""),
530+
stderr: Stream.encodeText(Stream.make("fatal: remote unavailable\n")),
531+
exitCode: Deferred.await(release),
532+
});
533+
}
534+
if (args[0] === "pull") {
535+
yield* Ref.update(pulls, (count) => count + 1);
536+
if (yield* Ref.get(fetching)) {
537+
return ChildProcessSpawner.makeHandle({
538+
...makeSuccessfulHandle(""),
539+
exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(1)),
540+
stderr: Stream.encodeText(
541+
Stream.make("error: cannot lock ref 'refs/remotes/origin/main': File exists\n"),
542+
),
543+
});
544+
}
545+
yield* Ref.set(head, "after");
546+
return makeSuccessfulHandle("");
547+
}
548+
if (args.includes("branch.main.remote")) {
549+
yield* Deferred.succeed(pullRemoteRead, undefined);
550+
return makeSuccessfulHandle("origin\n");
551+
}
552+
if (args[0] === "status")
553+
return makeSuccessfulHandle(
554+
"# branch.oid before\0# branch.head main\0# branch.upstream origin/main\0# branch.ab +0 -1\0",
555+
);
556+
if (args[0] === "diff-index") return makeSuccessfulHandle("");
557+
if (args.length === 2 && args[0] === "rev-parse" && args[1] === "HEAD")
558+
return makeSuccessfulHandle(`${yield* Ref.get(head)}\n`);
559+
if (args.includes("--git-common-dir"))
560+
return makeSuccessfulHandle(`${workingDirectory}/.git\n`);
561+
if (args.includes("--show-toplevel")) return makeSuccessfulHandle(`${workingDirectory}\n`);
562+
if (args.includes("@{upstream}")) return makeSuccessfulHandle("origin/main\n");
563+
if (args[0] === "remote") return makeSuccessfulHandle("origin\n");
564+
if (args[0] === "rev-list") return makeSuccessfulHandle("0\t0\n");
565+
if (args.includes("refs/remotes/origin/HEAD"))
566+
return makeSuccessfulHandle("refs/remotes/origin/main\n");
567+
return makeSuccessfulHandle("main\n");
568+
}),
569+
);
570+
const driver = yield* makeGitVcsDriverCore().pipe(
571+
Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner),
572+
);
573+
const status = Effect.fn("test.controlledStatusFetch.status")((directory = cwd) =>
574+
driver
575+
.statusDetailsRemote(directory)
576+
.pipe(Effect.provideService(Logger.CurrentLoggers, new Set([logger]))),
577+
);
578+
return { cwd, driver, status, started, finalized, warnings, attempts, pulls, pullRemoteRead };
579+
});
580+
581+
it.effect.each([0, 128])(
582+
"waits for an existing status fetch before pulling after fetch exit %s",
583+
(exitCode) =>
584+
Effect.gen(function* () {
585+
const fixture = yield* controlledStatusFetch;
586+
const status = yield* fixture.status().pipe(Effect.forkScoped);
587+
const fetch = yield* Queue.take(fixture.started);
588+
yield* TestClock.adjust("5 seconds");
589+
yield* Fiber.join(status);
590+
591+
const pull = yield* fixture.driver.pullCurrentBranch(fixture.cwd).pipe(Effect.forkScoped);
592+
yield* Deferred.await(fixture.pullRemoteRead);
593+
yield* TestClock.adjust("30 seconds");
594+
assert.equal(yield* Ref.get(fixture.pulls), 0);
595+
yield* Deferred.succeed(fetch.release, ChildProcessSpawner.ExitCode(exitCode));
596+
597+
assert.equal((yield* Fiber.join(pull)).status, "pulled");
598+
assert.equal(yield* Ref.get(fixture.pulls), 1);
599+
assert.equal(yield* Ref.get(fixture.attempts), 1);
600+
}).pipe(
601+
Effect.scoped,
602+
Effect.provide(layerServerConfig.pipe(Layer.provideMerge(NodeServices.layer))),
603+
),
604+
);
605+
606+
it.effect("does not start a status fetch while pulling", () =>
607+
Effect.gen(function* () {
608+
const fixture = yield* controlledStatusFetch;
609+
610+
assert.equal((yield* fixture.driver.pullCurrentBranch(fixture.cwd)).status, "pulled");
611+
assert.equal(yield* Ref.get(fixture.pulls), 1);
612+
assert.equal(yield* Ref.get(fixture.attempts), 0);
613+
}).pipe(
614+
Effect.scoped,
615+
Effect.provide(layerServerConfig.pipe(Layer.provideMerge(NodeServices.layer))),
616+
),
617+
);
618+
619+
it.effect("backs off failures after the status waiter has returned", () =>
620+
Effect.gen(function* () {
621+
const fixture = yield* controlledStatusFetch;
622+
const firstStatus = yield* fixture.status().pipe(Effect.forkScoped);
623+
const first = yield* Queue.take(fixture.started);
624+
yield* TestClock.adjust("5 seconds");
625+
yield* Fiber.join(firstStatus);
626+
yield* Deferred.succeed(first.release, ChildProcessSpawner.ExitCode(128));
627+
yield* Queue.take(fixture.finalized);
628+
yield* Queue.take(fixture.warnings);
629+
630+
yield* TestClock.adjust("15 seconds");
631+
yield* fixture.status();
632+
assert.equal(yield* Ref.get(fixture.attempts), 1);
633+
634+
yield* TestClock.adjust("15 seconds");
635+
const secondStatus = yield* fixture.status().pipe(Effect.forkScoped);
636+
const second = yield* Queue.take(fixture.started);
637+
yield* TestClock.adjust("5 seconds");
638+
yield* Fiber.join(secondStatus);
639+
yield* Deferred.succeed(second.release, ChildProcessSpawner.ExitCode(128));
640+
yield* Queue.take(fixture.finalized);
641+
yield* Queue.take(fixture.warnings);
642+
643+
for (let i = 0; i < 3; i++) {
644+
yield* TestClock.adjust("15 seconds");
645+
yield* fixture.status();
646+
assert.equal(yield* Ref.get(fixture.attempts), 2);
647+
}
648+
yield* TestClock.adjust("15 seconds");
649+
const recoveredStatus = yield* fixture.status().pipe(Effect.forkScoped);
650+
const recovered = yield* Queue.take(fixture.started);
651+
yield* Deferred.succeed(recovered.release, ChildProcessSpawner.ExitCode(0));
652+
yield* Fiber.join(recoveredStatus);
653+
assert.equal(yield* Ref.get(fixture.attempts), 3);
654+
655+
yield* TestClock.adjust("15 seconds");
656+
const failedAgainStatus = yield* fixture.status().pipe(Effect.forkScoped);
657+
const failedAgain = yield* Queue.take(fixture.started);
658+
yield* Deferred.succeed(failedAgain.release, ChildProcessSpawner.ExitCode(128));
659+
yield* Fiber.join(failedAgainStatus);
660+
yield* Queue.take(fixture.warnings);
661+
yield* TestClock.adjust("15 seconds");
662+
yield* fixture.status();
663+
assert.equal(yield* Ref.get(fixture.attempts), 4);
664+
665+
yield* TestClock.adjust("15 seconds");
666+
const resetStatus = yield* fixture.status().pipe(Effect.forkScoped);
667+
yield* TestClock.adjust("5 seconds");
668+
yield* Fiber.join(resetStatus);
669+
assert.equal(yield* Ref.get(fixture.attempts), 5);
670+
const reset = yield* Queue.take(fixture.started);
671+
yield* Deferred.succeed(reset.release, ChildProcessSpawner.ExitCode(0));
672+
}).pipe(
673+
Effect.scoped,
674+
Effect.provide(layerServerConfig.pipe(Layer.provideMerge(NodeServices.layer))),
675+
),
676+
);
677+
678+
it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")(
679+
"bounds cleanup when a Git subprocess ignores SIGTERM",
680+
() =>
681+
Effect.gen(function* () {
682+
const driver = yield* GitVcsDriver.GitVcsDriver;
683+
const directory = yield* makeTmpDir();
684+
const probe = yield* makeProcessProbe;
685+
writeFakeCli({
686+
directory,
687+
name: "git",
688+
source: `process.on("SIGTERM", () => {}); ${processProbeSource(probe.port)}`,
689+
});
690+
const run = yield* driver
691+
.execute({
692+
operation: "test.stalledStatusFetch",
693+
cwd: directory,
694+
args: ["fetch"],
695+
env: { PATH: `${directory}:${process.env.PATH ?? ""}` },
696+
timeoutMs: 1_000,
697+
forceKillAfterMs: 100,
698+
})
699+
.pipe(Effect.flip, Effect.forkScoped);
700+
yield* probe.started;
701+
702+
yield* TestClock.adjust("1 second");
703+
704+
yield* probe.exited;
705+
assert.equal((yield* Fiber.join(run)).detail, "Git command timed out.");
706+
}).pipe(Effect.scoped, Effect.provide(layerTest)),
707+
);
708+
709+
it.effect("releases stalled background worker slots at their lifetime limit", () =>
710+
Effect.gen(function* () {
711+
const fs = yield* FileSystem.FileSystem;
712+
const fixture = yield* controlledStatusFetch;
713+
yield* fixture.driver.listRefs({ cwd: fixture.cwd });
714+
const statuses = [];
715+
for (let i = 0; i < 4; i++) {
716+
const cwd = `${fixture.cwd}/repo-${i}`;
717+
yield* fs.makeDirectory(`${cwd}/.git`, { recursive: true });
718+
statuses.push(yield* fixture.status(cwd).pipe(Effect.forkScoped));
719+
yield* Queue.take(fixture.started);
720+
}
721+
yield* TestClock.adjust("5 seconds");
722+
yield* Effect.forEach(statuses, Fiber.join);
723+
const waiting = yield* fixture.status().pipe(Effect.forkScoped);
724+
yield* TestClock.adjust("5 seconds");
725+
yield* Fiber.join(waiting);
726+
assert.equal(yield* Ref.get(fixture.attempts), 4);
727+
728+
yield* TestClock.adjust(290_000);
729+
assert.equal(yield* Ref.get(fixture.attempts), 5);
730+
for (let i = 0; i < 4; i++) yield* Queue.take(fixture.finalized);
731+
const healthy = yield* Queue.take(fixture.started);
732+
assert.equal(
733+
NodeFS.realpathSync.native(healthy.cwd),
734+
NodeFS.realpathSync.native(fixture.cwd),
735+
);
736+
yield* Deferred.succeed(healthy.release, ChildProcessSpawner.ExitCode(0));
737+
yield* Queue.take(fixture.finalized);
738+
}).pipe(
739+
Effect.scoped,
740+
Effect.provide(layerServerConfig.pipe(Layer.provideMerge(NodeServices.layer))),
741+
),
742+
);
743+
496744
// A fast-forward that prints over 1 MB takes thousands of files, so the real pull's output is
497745
// padded past the cap instead.
498746
const withPaddedPull = (

0 commit comments

Comments
 (0)