Skip to content

Commit 8753c74

Browse files
maria-rcksnateEcjuliusmarminge
authored
fix(server): keep worktrees containing nested repositories (pingdotgg#25241)
Co-authored-by: Yukun Shan <[email protected]> Co-authored-by: Julius Marminge <[email protected]>
1 parent aa58dc6 commit 8753c74

2 files changed

Lines changed: 221 additions & 20 deletions

File tree

‎apps/server/src/storageCleanup.test.ts‎

Lines changed: 135 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -612,6 +612,52 @@ const runCleanupTest = <A, E>(
612612
>,
613613
) => effect.pipe(Effect.provide(cleanupTestLayer), Effect.scoped);
614614

615+
it.live.each(["repository", "uppercase", "bare"] as const)(
616+
"keeps a worktree holding an ignored nested %s",
617+
(kind) =>
618+
runCleanupTest(
619+
Effect.gen(function* () {
620+
const { service, fs, repo, worktree, command, setPolicy } = yield* cleanupFixture();
621+
setPolicy("tracked-changes");
622+
const nested = `${worktree}/dist/nested`;
623+
if (kind === "uppercase") {
624+
yield* command(repo, ["worktree", "add", "-b", "nested", nested]);
625+
} else {
626+
yield* command(repo, [
627+
"clone",
628+
"--no-hardlinks",
629+
...(kind === "bare" ? ["--bare"] : []),
630+
repo,
631+
nested,
632+
]);
633+
}
634+
if (kind === "uppercase") yield* fs.rename(`${nested}/.git`, `${nested}/.GIT`);
635+
yield* fs.writeFileString(`${nested}/uncommitted.txt`, "irreplaceable work\n");
636+
expect((yield* command(worktree, ["status", "--porcelain"])).stdout).toBe("");
637+
const report = yield* service.runNow;
638+
expect(report.entries.find((entry) => entry.path === worktree)).toMatchObject({
639+
outcome: "kept",
640+
reason: "Contains a nested Git repository",
641+
});
642+
expect(yield* fs.readFileString(`${nested}/uncommitted.txt`)).toBe("irreplaceable work\n");
643+
expect(
644+
(yield* command(nested, [
645+
...(kind === "uppercase"
646+
? [
647+
`--git-dir=${(yield* fs.readFileString(`${nested}/.GIT`)).trim().slice(8)}`,
648+
"--work-tree=.",
649+
]
650+
: []),
651+
"-c",
652+
"safe.bareRepository=all",
653+
"rev-parse",
654+
"--is-bare-repository",
655+
])).stdout.trim(),
656+
).toBe(String(kind === "bare"));
657+
}),
658+
),
659+
);
660+
615661
const submoduleCleanupFixture = () =>
616662
Effect.gen(function* () {
617663
const fixture = yield* cleanupFixture();
@@ -636,6 +682,48 @@ const submoduleCleanupFixture = () =>
636682
return { ...fixture, submodule, child, recordSubmodules };
637683
});
638684

685+
it.live("inspects a pnpm store excluded from reclaimed bytes for nested repositories", () =>
686+
runCleanupTest(
687+
Effect.gen(function* () {
688+
const { service, fs, repo, worktree, command, setPolicy } = yield* cleanupFixture();
689+
setPolicy("tracked-changes");
690+
const storeDir = yield* fs.makeTempDirectoryScoped({ prefix: "cleanup-store-" });
691+
const nested = `${worktree}/node_modules/.pnpm/nested`;
692+
yield* command(repo, ["clone", "--no-hardlinks", repo, nested]);
693+
yield* fs.writeFileString(`${nested}/uncommitted.txt`, "keep me");
694+
yield* fs.writeFileString(
695+
`${worktree}/node_modules/.modules.yaml`,
696+
JSON.stringify({ storeDir, virtualStoreDir: ".pnpm" }),
697+
);
698+
const report = yield* service.runNow;
699+
expect(report.entries.find((entry) => entry.path === worktree)).toMatchObject({
700+
outcome: "kept",
701+
reason: "Contains a nested Git repository",
702+
});
703+
expect(yield* fs.readFileString(`${nested}/uncommitted.txt`)).toBe("keep me");
704+
}),
705+
),
706+
);
707+
708+
it.live("keeps an ignored repository inside a registered submodule", () =>
709+
runCleanupTest(
710+
Effect.gen(function* () {
711+
const { service, fs, repo, worktree, submodule, command, setPolicy } =
712+
yield* submoduleCleanupFixture();
713+
setPolicy("tracked-changes");
714+
const nested = `${submodule}/dist/nested`;
715+
yield* command(repo, ["clone", "--no-hardlinks", repo, nested]);
716+
yield* fs.writeFileString(`${nested}/uncommitted.txt`, "keep me");
717+
const report = yield* service.runNow;
718+
expect(report.entries.find((entry) => entry.path === worktree)).toMatchObject({
719+
outcome: "kept",
720+
reason: "Contains a nested Git repository",
721+
});
722+
expect(yield* fs.readFileString(`${nested}/uncommitted.txt`)).toBe("keep me");
723+
}),
724+
),
725+
);
726+
639727
it.live("removes clean submodules recursively, including deinitialized repositories", () =>
640728
runCleanupTest(
641729
Effect.gen(function* () {
@@ -649,6 +737,37 @@ it.live("removes clean submodules recursively, including deinitialized repositor
649737
repo,
650738
"stored",
651739
]);
740+
const legacy = `${worktree}/legacy`;
741+
yield* command(repo, ["clone", "--no-hardlinks", repo, legacy]);
742+
yield* command(worktree, ["submodule", "add", repo, "legacy"]);
743+
expect((yield* fs.stat(`${legacy}/.git`)).type).toBe("Directory");
744+
yield* command(legacy, [
745+
"-c",
746+
"protocol.file.allow=always",
747+
"submodule",
748+
"add",
749+
repo,
750+
"child",
751+
]);
752+
yield* command(legacy, [
753+
"-c",
754+
"protocol.file.allow=always",
755+
"submodule",
756+
"add",
757+
repo,
758+
"stored",
759+
]);
760+
yield* command(legacy, [
761+
"-c",
762+
"user.name=Cleanup test",
763+
"-c",
764+
765+
"commit",
766+
"-am",
767+
"record legacy children",
768+
]);
769+
yield* command(legacy, ["push", "origin", "HEAD:legacy"]);
770+
yield* command(legacy, ["submodule", "deinit", "stored"]);
652771
yield* recordSubmodules();
653772
yield* command(submodule, ["submodule", "deinit", "stored"]);
654773
const admin = (yield* command(worktree, ["rev-parse", "--absolute-git-dir"])).stdout.trim();
@@ -2241,9 +2360,14 @@ describe("storage cleanup reports and local file policies", () => {
22412360
Effect.gen(function* () {
22422361
const fs = yield* FileSystem.FileSystem;
22432362
const storeDir = yield* fs.makeTempDirectoryScoped({ prefix: "pnpm-store-" });
2363+
const statted: string[] = [];
22442364
const { service, worktree, setPolicy } = yield* cleanupFixture().pipe(
22452365
Effect.provideService(FileSystem.FileSystem, {
22462366
...fs,
2367+
lstat: (target) => {
2368+
statted.push(target);
2369+
return fs.lstat(target);
2370+
},
22472371
stat: (target) =>
22482372
fs
22492373
.stat(target)
@@ -2284,6 +2408,9 @@ describe("storage cleanup reports and local file policies", () => {
22842408
);
22852409
const report = yield* service.runNow;
22862410
expect(report.entries[0]).toMatchObject({ outcome: "removed", bytes });
2411+
expect(statted.includes(`${nodeModules}/.pnpm/[email protected]/index.js`)).toBe(
2412+
store !== "same-device",
2413+
);
22872414
}),
22882415
),
22892416
);
@@ -2378,7 +2505,7 @@ describe("storage cleanup reports and local file policies", () => {
23782505
}),
23792506
),
23802507
);
2381-
it.live("still removes a worktree when measuring its size fails", () =>
2508+
it.live("keeps a worktree when the inspection cannot read its directories", () =>
23822509
runCleanupTest(
23832510
Effect.gen(function* () {
23842511
const fs = yield* FileSystem.FileSystem;
@@ -2389,9 +2516,14 @@ describe("storage cleanup reports and local file policies", () => {
23892516
}),
23902517
);
23912518
const report = yield* service.runNow;
2392-
expect(report.entries[0]).toMatchObject({ outcome: "removed", bytes: null, files: null });
2519+
expect(report.entries[0]).toMatchObject({
2520+
outcome: "kept",
2521+
reason: "Worktree could not be fully inspected",
2522+
bytes: null,
2523+
files: null,
2524+
});
23932525
expect(report.bytesFreed).toBe(0);
2394-
expect(yield* fs.exists(worktree)).toBe(false);
2526+
expect(yield* fs.exists(worktree)).toBe(true);
23952527
}),
23962528
),
23972529
);

‎apps/server/src/storageCleanup.ts‎

Lines changed: 86 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
// @effect-diagnostics nodeBuiltinImport:off
2+
import * as NodeFSP from "node:fs/promises";
13
import {
24
GitCommandError,
35
OrchestrationV2AppThreadJson,
@@ -79,6 +81,15 @@ class PnpmModulesManifestParseError extends Schema.TaggedError<PnpmModulesManife
7981
}
8082
}
8183

84+
class WorktreeDirectoryReadError extends Schema.TaggedError<WorktreeDirectoryReadError>()(
85+
"WorktreeDirectoryReadError",
86+
{ cause: Schema.Defect() },
87+
) {
88+
override get message(): string {
89+
return "Could not read a worktree directory.";
90+
}
91+
}
92+
8293
const DAY_MS = 86_400_000;
8394
// Give settle scripts time to open terminals that protect the worktree.
8495
const SETTLE_SCRIPT_GRACE_MS = 60_000;
@@ -379,9 +390,47 @@ const make = Effect.gen(function* () {
379390
);
380391

381392
const measureWorktree = Effect.fn("storageCleanup.measureWorktree")(
382-
function* (worktreePath: string) {
393+
function* (worktreePath: string, inspectRepositories = false) {
383394
const root = yield* fs.realPath(worktreePath);
384395
const skipped = yield* sharedPnpmVirtualStore(root);
396+
const incomplete = { bytes: null, reason: "worktree could not be fully inspected" };
397+
const repositories = new Set([pathKey(root)]);
398+
const moduleStores: string[] = [];
399+
if (inspectRepositories && (yield* fs.exists(path.join(root, ".gitmodules")))) {
400+
const submodules = yield* git.execute({
401+
operation: "StorageCleanup.nestedRepositories",
402+
cwd: root,
403+
args: [
404+
"submodule",
405+
"foreach",
406+
"--recursive",
407+
"--quiet",
408+
'printf "%s\\0" "$toplevel/$sm_path"',
409+
],
410+
maxOutputBytes: 64 * 1024,
411+
});
412+
if (submodules.stdoutTruncated || submodules.stderrTruncated) return incomplete;
413+
for (const submodule of submodules.stdout.split("\0").filter(Boolean)) {
414+
repositories.add(pathKey(path.resolve(submodule)));
415+
repositories.add(pathKey(path.resolve(submodule, ".git")));
416+
moduleStores.push(pathKey(path.resolve(submodule, ".git", "modules")));
417+
}
418+
}
419+
// Git status omits ignored repositories, and worktree remove deletes them.
420+
// Registered submodules have their own recursive safety checks.
421+
const isNestedRepository = (target: string, names: ReadonlyArray<string>) => {
422+
const key = pathKey(target);
423+
if (
424+
repositories.has(key) ||
425+
moduleStores.some((store) => key === store || inside(store, key))
426+
)
427+
return false;
428+
const markers = new Set(names.map((name) => name.toLowerCase()));
429+
return (
430+
markers.has(".git") ||
431+
(markers.has("head") && markers.has("objects") && markers.has("refs"))
432+
);
433+
};
385434
const pending = [root];
386435
let entries = 1;
387436
let bytes = 0;
@@ -390,39 +439,57 @@ const make = Effect.gen(function* () {
390439
pending.splice(-8),
391440
// Untraced: this runs once per entry, and a worktree can hold millions.
392441
Effect.fnUntraced(function* (target) {
393-
if (target === skipped) return { target, bytes: 0, names: [] };
442+
const empty = { target, bytes: 0, names: [], children: [] };
443+
const inStore = skipped !== null && (target === skipped || inside(skipped, target));
444+
if (inStore && !inspectRepositories) return empty;
394445
const stat = yield* fs.lstat(target);
395-
if (stat.type === "SymbolicLink") return { target, bytes: 0, names: [] };
446+
if (stat.type === "SymbolicLink") return empty;
396447
if ((yield* fs.realPath(target)) !== target) return null;
397-
if (stat.type === "Directory")
398-
return { target, bytes: 0, names: yield* fs.readDirectory(target) };
448+
if (stat.type === "Directory") {
449+
if (!inStore) {
450+
const names = yield* fs.readDirectory(target);
451+
return { target, bytes: 0, names, children: names };
452+
}
453+
// The store is excluded from reclaimed bytes; only walk its directories.
454+
const dirents = yield* Effect.tryPromise({
455+
try: () => NodeFSP.readdir(target, { withFileTypes: true }),
456+
catch: (cause) => new WorktreeDirectoryReadError({ cause }),
457+
});
458+
return {
459+
target,
460+
bytes: 0,
461+
names: dirents.map((entry) => entry.name),
462+
children: dirents.filter((entry) => entry.isDirectory()).map((entry) => entry.name),
463+
};
464+
}
399465
// Removing a shared hardlink leaves its blocks allocated.
400-
if (stat.type !== "File" || Option.getOrElse(stat.nlink, () => 1n) !== 1n)
401-
return { target, bytes: 0, names: [] };
466+
if (stat.type !== "File" || Option.getOrElse(stat.nlink, () => 1n) !== 1n) return empty;
402467
const bytes = Option.match(stat.blocks, {
403468
onNone: () => Number(stat.size),
404469
onSome: (blocks) => Number(blocks * 512n),
405470
});
406-
return { target, bytes, names: [] };
471+
return { ...empty, bytes };
407472
}),
408473
{ concurrency: "unbounded" },
409474
);
410475
for (const result of inspected) {
411-
if (result === null) return null;
476+
if (result === null) return incomplete;
477+
if (inspectRepositories && isNestedRepository(result.target, result.names))
478+
return { bytes: null, reason: "contains a nested Git repository" };
412479
bytes += result.bytes;
413480
entries += result.names.length;
414-
if (entries > 2_000_000) return null;
415-
for (const name of result.names) {
481+
if (entries > 2_000_000) return incomplete;
482+
for (const name of result.children) {
416483
const child = path.join(result.target, name);
417-
if (!inside(root, child)) return null;
484+
if (!inside(root, child)) return incomplete;
418485
pending.push(child);
419486
}
420487
}
421488
}
422-
return bytes;
489+
return { bytes, reason: null };
423490
},
424491
Effect.timeout("30 seconds"),
425-
Effect.orElseSucceed(() => null),
492+
Effect.orElseSucceed(() => ({ bytes: null, reason: "worktree could not be fully inspected" })),
426493
);
427494

428495
// Large snapshots take seconds; use the reader worker instead of blocking the event loop.
@@ -531,7 +598,7 @@ const make = Effect.gen(function* () {
531598
firstMeasurement ? Effect.asSome : Effect.timeoutOption(remaining),
532599
);
533600
firstMeasurement = false;
534-
if (Option.isSome(measured)) orphanSizes.set(key, measured.value);
601+
if (Option.isSome(measured)) orphanSizes.set(key, measured.value.bytes);
535602
}
536603
const bytes = orphanSizes.get(key) ?? null;
537604
entries.push({
@@ -1294,8 +1361,10 @@ const make = Effect.gen(function* () {
12941361
});
12951362
const blocked = yield* blockedSinceCheck();
12961363
if (blocked !== null) return keep(blocked);
1297-
// Size is an estimate; provider startup must not wait for directory traversal.
1298-
const bytes = yield* measureWorktree(worktreePath);
1364+
// Inspect before the lease; provider startup must not wait for directory traversal.
1365+
const inspection = yield* measureWorktree(worktreePath, true);
1366+
if (inspection.reason !== null) return keep(inspection.reason);
1367+
const bytes = inspection.bytes;
12991368
yield* withWorkspaceLease(
13001369
worktreePath,
13011370
Effect.gen(function* () {

0 commit comments

Comments
 (0)