Skip to content

Commit a37b852

Browse files
authored
perf(server): speed up worktree fetch and checkout (pingdotgg#11633)
1 parent 5b377e2 commit a37b852

6 files changed

Lines changed: 194 additions & 26 deletions

File tree

‎apps/server/src/git/GitWorkflowService.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ export class GitWorkflowService extends Context.Service<
7474
readonly fetchRemote: (input: {
7575
readonly cwd: string;
7676
readonly remoteName: string;
77+
readonly refName?: string;
7778
}) => Effect.Effect<void, GitCommandError>;
7879
readonly remoteExists: (input: {
7980
readonly cwd: string;

‎apps/server/src/server.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10873,6 +10873,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
1087310873
assert.deepEqual(fetchRemote.mock.calls[0]?.[0], {
1087410874
cwd: "/tmp/project",
1087510875
remoteName: "origin",
10876+
refName: "main",
1087610877
});
1087710878
assert.deepEqual(remoteBranchExists.mock.calls[0]?.[0], {
1087810879
cwd: "/tmp/project",

‎apps/server/src/vcs/GitVcsDriver.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -237,6 +237,7 @@ export interface GitFetchRemoteTrackingBranchInput {
237237
export interface GitFetchRemoteInput {
238238
cwd: string;
239239
remoteName: string;
240+
refName?: string;
240241
}
241242

242243
export interface GitRemoteExistsInput {

‎apps/server/src/vcs/GitVcsDriverCore.test.ts‎

Lines changed: 130 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1698,6 +1698,64 @@ it.layer(TestLayer)("GitVcsDriver core integration", (it) => {
16981698
});
16991699

17001700
describe("worktree operations", () => {
1701+
it.effect("uses parallel checkout without skipping filters or hooks", () =>
1702+
Effect.gen(function* () {
1703+
const cwd = yield* makeTmpDir();
1704+
const { initialBranch } = yield* initRepoWithCommit(cwd);
1705+
const fs = yield* FileSystem.FileSystem;
1706+
const path = yield* Path.Path;
1707+
const driver = yield* GitVcsDriver.GitVcsDriver;
1708+
yield* git(cwd, ["config", "filter.test.smudge", "sed s/original/filtered/g"]);
1709+
yield* writeTextFile(cwd, ".gitattributes", "asset.txt filter=test\n");
1710+
yield* writeTextFile(cwd, "asset.txt", "original\n");
1711+
yield* git(cwd, ["add", "."]);
1712+
yield* git(cwd, ["commit", "-m", "filtered asset"]);
1713+
yield* writeTextFile(
1714+
cwd,
1715+
".git/hooks/post-checkout",
1716+
"#!/bin/sh\ngit config checkout.workers > checkout-workers\nexit 0\n",
1717+
);
1718+
yield* fs.chmod(path.join(cwd, ".git/hooks/post-checkout"), 0o755);
1719+
const worktreePath = path.join(yield* makeTmpDir("git-worktrees-"), "parallel");
1720+
1721+
yield* driver.createWorktree({
1722+
cwd,
1723+
path: worktreePath,
1724+
refName: initialBranch,
1725+
newRefName: "feature/parallel",
1726+
baseRefName: initialBranch,
1727+
});
1728+
1729+
assert.notInclude(yield* git(cwd, ["worktree", "list", "--porcelain"]), "locked");
1730+
assert.equal(yield* fs.readFileString(path.join(worktreePath, "checkout-workers")), "0\n");
1731+
assert.equal(yield* fs.readFileString(path.join(worktreePath, "asset.txt")), "filtered\n");
1732+
assert.equal(
1733+
yield* git(worktreePath, ["rev-parse", "HEAD"]),
1734+
yield* git(cwd, ["rev-parse", "HEAD"]),
1735+
);
1736+
assert.equal(
1737+
yield* git(cwd, ["config", "branch.feature/parallel.gh-merge-base"]),
1738+
initialBranch,
1739+
);
1740+
for (const [configured, expected] of [
1741+
["1", "1"],
1742+
["", "0"],
1743+
] as const) {
1744+
yield* git(cwd, ["config", "checkout.workers", configured]);
1745+
const configuredPath = path.join(yield* makeTmpDir("git-worktrees-"), "configured");
1746+
yield* driver.createWorktree({
1747+
cwd,
1748+
path: configuredPath,
1749+
refName: initialBranch,
1750+
newRefName: `feature/configured-${expected}`,
1751+
});
1752+
assert.equal(
1753+
yield* fs.readFileString(path.join(configuredPath, "checkout-workers")),
1754+
`${expected}\n`,
1755+
);
1756+
}
1757+
}),
1758+
);
17011759
it("parses checkout progress lines from git's stderr", () => {
17021760
assert.deepStrictEqual(parseGitCheckoutProgressLine("Updating files: 78% (2104/2700)"), {
17031761
percent: 78,
@@ -1827,11 +1885,11 @@ it.layer(TestLayer)("GitVcsDriver core integration", (it) => {
18271885
}),
18281886
);
18291887

1830-
it.effect("reports checkout progress while creating a worktree", () =>
1888+
it.effect("reports checkout progress during parallel worktree creation", () =>
18311889
Effect.gen(function* () {
18321890
const cwd = yield* makeTmpDir();
18331891
const { initialBranch } = yield* initRepoWithCommit(cwd);
1834-
for (let index = 0; index < 5; index += 1) {
1892+
for (let index = 0; index < 200; index += 1) {
18351893
yield* writeTextFile(cwd, `file-${index}.txt`, `${index}\n`);
18361894
}
18371895
yield* git(cwd, ["add", "."]);
@@ -1865,7 +1923,7 @@ it.layer(TestLayer)("GitVcsDriver core integration", (it) => {
18651923
const updates = yield* Ref.get(seen);
18661924
assert.isAbove(updates.length, 1);
18671925
assert.equal(updates.at(-1)?.percent, 100);
1868-
assert.equal(updates.at(-1)?.total, 6);
1926+
assert.equal(updates.at(-1)?.total, 201);
18691927
const completed = updates.map((update) => update.completed);
18701928
assert.deepEqual(
18711929
completed,
@@ -2098,6 +2156,61 @@ it.layer(TestLayer)("GitVcsDriver core integration", (it) => {
20982156
});
20992157

21002158
describe("remote operations", () => {
2159+
for (const failure of ["offline", "auth", "timeout"] as const) {
2160+
it.effect(`does not retry a scoped fetch after ${failure}`, () =>
2161+
Effect.gen(function* () {
2162+
const cwd = yield* makeTmpDir();
2163+
const delegate = yield* ChildProcessSpawner.ChildProcessSpawner;
2164+
const started = yield* Deferred.make<void>();
2165+
const attempts: Array<ReadonlyArray<string>> = [];
2166+
const spawner = ChildProcessSpawner.make((command) =>
2167+
Effect.gen(function* () {
2168+
if (!ChildProcess.isStandardCommand(command))
2169+
return yield* Effect.die("unexpected command");
2170+
if (command.args[0] !== "fetch") return yield* delegate.spawn(command);
2171+
attempts.push(command.args);
2172+
yield* Deferred.succeed(started, undefined);
2173+
return ChildProcessSpawner.makeHandle({
2174+
...makeNonRepositoryHandle(),
2175+
exitCode:
2176+
failure === "timeout"
2177+
? Effect.never
2178+
: Effect.succeed(ChildProcessSpawner.ExitCode(128)),
2179+
stderr: Stream.encodeText(
2180+
Stream.make(
2181+
failure === "auth"
2182+
? "fatal: Authentication failed"
2183+
: "fatal: Could not resolve host",
2184+
),
2185+
),
2186+
});
2187+
}),
2188+
);
2189+
const driver = yield* makeGitVcsDriverCore().pipe(
2190+
Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner),
2191+
Effect.provide(ServerConfigLayer),
2192+
);
2193+
const fetching = yield* driver
2194+
.fetchRemote({ cwd, remoteName: "origin", refName: "main" })
2195+
.pipe(Effect.result, Effect.forkChild({ startImmediately: true }));
2196+
yield* Deferred.await(started);
2197+
if (failure === "timeout") {
2198+
yield* TestClock.adjust("31 seconds");
2199+
yield* TestClock.adjust("31 seconds");
2200+
}
2201+
const result = yield* Fiber.join(fetching);
2202+
assert.isTrue(Result.isFailure(result));
2203+
assert.equal(attempts.length, 1);
2204+
if (Result.isFailure(result)) {
2205+
assert.equal(
2206+
result.failure.detail,
2207+
failure === "timeout" ? "Git command timed out." : "git fetch origin failed",
2208+
);
2209+
}
2210+
}),
2211+
);
2212+
}
2213+
21012214
it.effect("creates a worktree from the latest fetched remote commit", () =>
21022215
Effect.gen(function* () {
21032216
const cwd = yield* makeTmpDir();
@@ -2120,8 +2233,16 @@ it.layer(TestLayer)("GitVcsDriver core integration", (it) => {
21202233
const remoteHead = yield* git(peer, ["rev-parse", "HEAD"]);
21212234
assert.notEqual(beforeFetch, remoteHead);
21222235

2236+
yield* git(peer, ["push", "origin", "HEAD:refs/heads/unrelated"]);
21232237
const driver = yield* GitVcsDriver.GitVcsDriver;
2124-
yield* driver.fetchRemote({ cwd, remoteName: "origin" });
2238+
yield* driver.fetchRemote({
2239+
cwd,
2240+
remoteName: "origin",
2241+
refName: `origin/${initialBranch}`,
2242+
});
2243+
assert.isFalse(
2244+
yield* driver.remoteBranchExists({ cwd, remoteName: "origin", refName: "unrelated" }),
2245+
);
21252246

21262247
assert.equal(
21272248
yield* driver.remoteBranchExists({
@@ -2183,6 +2304,11 @@ it.layer(TestLayer)("GitVcsDriver core integration", (it) => {
21832304
const status = yield* driver.statusDetails(worktreePath);
21842305
assert.equal(status.aheadCount, 0);
21852306
assert.equal(status.aheadOfDefaultCount, 0);
2307+
2308+
yield* driver.fetchRemote({ cwd, remoteName: "origin", refName: "local-only" });
2309+
assert.isTrue(
2310+
yield* driver.remoteBranchExists({ cwd, remoteName: "origin", refName: "unrelated" }),
2311+
);
21862312
}),
21872313
);
21882314

‎apps/server/src/vcs/GitVcsDriverCore.ts‎

Lines changed: 60 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -3062,23 +3062,29 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
30623062
const progress = options?.progress;
30633063
const onCheckoutProgress = progress?.onCheckoutProgress;
30643064

3065-
yield* executeGit("GitVcsDriver.createWorktree", input.cwd, args, {
3066-
fallbackErrorDetail: "git worktree add failed",
3067-
timeoutMs: WORKTREE_ADD_TIMEOUT_MS,
3068-
...(onCheckoutProgress
3069-
? {
3070-
// Git only prints checkout progress when stderr is a tty or the
3071-
// delay elapsed. GIT_PROGRESS_DELAY=0 forces it through the pipe.
3072-
env: { GIT_PROGRESS_DELAY: "0", LC_ALL: "C" },
3073-
progress: {
3074-
onStderrLine: (line) => {
3075-
const parsed = parseGitCheckoutProgressLine(line);
3076-
return parsed ? onCheckoutProgress(parsed) : Effect.void;
3065+
const checkoutWorkers = (yield* readConfigValue(input.cwd, "checkout.workers")) ?? "0";
3066+
yield* executeGit(
3067+
"GitVcsDriver.createWorktree",
3068+
input.cwd,
3069+
["-c", `checkout.workers=${checkoutWorkers}`, ...args],
3070+
{
3071+
fallbackErrorDetail: "git worktree add failed",
3072+
timeoutMs: WORKTREE_ADD_TIMEOUT_MS,
3073+
...(onCheckoutProgress
3074+
? {
3075+
// Git only prints checkout progress when stderr is a tty or the
3076+
// delay elapsed. GIT_PROGRESS_DELAY=0 forces it through the pipe.
3077+
env: { GIT_PROGRESS_DELAY: "0", LC_ALL: "C" },
3078+
progress: {
3079+
onStderrLine: (line) => {
3080+
const parsed = parseGitCheckoutProgressLine(line);
3081+
return parsed ? onCheckoutProgress(parsed) : Effect.void;
3082+
},
30773083
},
3078-
},
3079-
}
3080-
: {}),
3081-
});
3084+
}
3085+
: {}),
3086+
},
3087+
);
30823088

30833089
if (progress?.onWorktreeClaimed) {
30843090
yield* progress.onWorktreeClaimed(worktreePath);
@@ -3260,15 +3266,47 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
32603266

32613267
const fetchRemote: GitVcsDriver.GitVcsDriver["Service"]["fetchRemote"] = Effect.fn("fetchRemote")(
32623268
function* (input) {
3263-
yield* executeGit(
3269+
const args = ["fetch", "--quiet", input.remoteName];
3270+
const options = {
3271+
env: STATUS_UPSTREAM_REFRESH_ENV,
3272+
fallbackErrorDetail: `git fetch ${input.remoteName} failed`,
3273+
};
3274+
const fetchAll = executeGit("GitVcsDriver.fetchRemote", input.cwd, args, options);
3275+
if (input.refName === undefined) {
3276+
return yield* fetchAll.pipe(Effect.asVoid);
3277+
}
3278+
const branch =
3279+
parseRemoteRefWithRemoteNames(input.refName, [input.remoteName])?.branchName ??
3280+
input.refName;
3281+
const scopedArgs = [
3282+
...args,
3283+
`+refs/heads/${branch}:refs/remotes/${input.remoteName}/${branch}`,
3284+
];
3285+
const result = yield* executeGitWithStableDiagnostics(
32643286
"GitVcsDriver.fetchRemote",
32653287
input.cwd,
3266-
["fetch", "--quiet", input.remoteName],
3267-
{
3268-
env: STATUS_UPSTREAM_REFRESH_ENV,
3269-
fallbackErrorDetail: `git fetch ${input.remoteName} failed`,
3270-
},
3288+
scopedArgs,
3289+
{ ...options, allowNonZeroExit: true },
32713290
);
3291+
if (result.exitCode === 0) return;
3292+
if (
3293+
result.stderr
3294+
.split(/\r?\n/)
3295+
.includes(`fatal: couldn't find remote ref refs/heads/${branch}`)
3296+
) {
3297+
return yield* fetchAll.pipe(Effect.asVoid);
3298+
}
3299+
return yield* new GitCommandError({
3300+
...gitCommandContext({
3301+
operation: "GitVcsDriver.fetchRemote",
3302+
cwd: input.cwd,
3303+
args: scopedArgs,
3304+
}),
3305+
detail: options.fallbackErrorDetail,
3306+
exitCode: result.exitCode,
3307+
stdoutLength: result.stdout.length,
3308+
stderrLength: result.stderr.length,
3309+
});
32723310
},
32733311
);
32743312

‎apps/server/src/ws.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1318,6 +1318,7 @@ const makeWsRpcLayer = (
13181318
yield* gitWorkflow.fetchRemote({
13191319
cwd: prepareWorktree.projectCwd,
13201320
remoteName: "origin",
1321+
refName: prepareWorktree.baseBranch,
13211322
});
13221323
const remoteBaseExists = yield* gitWorkflow.remoteBranchExists({
13231324
cwd: prepareWorktree.projectCwd,

0 commit comments

Comments
 (0)