Skip to content

Commit 4be96e0

Browse files
fix(server): status of a new repository no longer waits on huge untracked files (pingdotgg#34916)
1 parent 080b10c commit 4be96e0

2 files changed

Lines changed: 118 additions & 2 deletions

File tree

‎apps/server/src/vcs/GitVcsDriverCore.test.ts‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices";
77
import * as HostProcess from "@t3tools/shared/HostProcess";
88
import { assert, it, describe } from "@effect/vitest";
99
import * as Cause from "effect/Cause";
10+
import * as Clock from "effect/Clock";
1011
import * as Deferred from "effect/Deferred";
1112
import * as Effect from "effect/Effect";
1213
import * as Exit from "effect/Exit";
@@ -3960,6 +3961,98 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => {
39603961
}),
39613962
);
39623963

3964+
it.effect(
3965+
"reports a new repository without waiting on a huge untracked file's line count",
3966+
() =>
3967+
Effect.gen(function* () {
3968+
const cwd = yield* makeTmpDir();
3969+
yield* git(cwd, ["init", "-b", "main"]);
3970+
yield* writeTextFile(cwd, "small.txt", "line\n");
3971+
yield* writeTextFile(cwd, "large.data", "");
3972+
const fileSystem = yield* FileSystem.FileSystem;
3973+
const pathService = yield* Path.Path;
3974+
yield* fileSystem.truncate(pathService.join(cwd, "large.data"), 8 * 1024 * 1024 * 1024);
3975+
const delegate = yield* ChildProcessSpawner.ChildProcessSpawner;
3976+
const totalsDiffSettled = yield* Deferred.make<void>();
3977+
const largeFileDiffAbandonedAt = yield* Deferred.make<number>();
3978+
// Stands in for `git diff --numstat` reading gigabytes of untracked file content.
3979+
const readsLargeFile = (command: ChildProcess.Command) => {
3980+
const indexFile =
3981+
ChildProcess.isStandardCommand(command) &&
3982+
command.args.includes("--numstat") &&
3983+
command.options.env?.GIT_INDEX_FILE;
3984+
return (
3985+
typeof indexFile === "string" &&
3986+
NodeFS.readFileSync(indexFile).includes(Buffer.from("large.data"))
3987+
);
3988+
};
3989+
const spawner = ChildProcessSpawner.make((command) =>
3990+
readsLargeFile(command)
3991+
? Deferred.succeed(totalsDiffSettled, undefined).pipe(
3992+
Effect.as(
3993+
ChildProcessSpawner.makeHandle({
3994+
...makeSuccessfulHandle(""),
3995+
isRunning: Effect.succeed(true),
3996+
exitCode: Effect.never.pipe(
3997+
Effect.onInterrupt(() =>
3998+
Clock.currentTimeMillis.pipe(
3999+
Effect.flatMap((now) =>
4000+
Deferred.succeed(largeFileDiffAbandonedAt, now),
4001+
),
4002+
),
4003+
),
4004+
),
4005+
}),
4006+
),
4007+
)
4008+
: delegate.spawn(command),
4009+
);
4010+
const driver = yield* makeGitVcsDriverCore().pipe(
4011+
Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner),
4012+
);
4013+
const startedAt = yield* Clock.currentTimeMillis;
4014+
4015+
const status = yield* driver
4016+
.statusDetailsLocal(cwd, { includeDivergence: false, includeBranchChanges: true })
4017+
.pipe(Effect.forkChild);
4018+
yield* Effect.raceFirst(Deferred.await(totalsDiffSettled), Fiber.await(status));
4019+
// Git's default command timeout, which is what releases the status on main.
4020+
yield* TestClock.adjust("30 seconds");
4021+
const reported = yield* Fiber.join(status);
4022+
4023+
const statusHeldForMs = (yield* Deferred.isDone(largeFileDiffAbandonedAt))
4024+
? (yield* Deferred.await(largeFileDiffAbandonedAt)) - startedAt
4025+
: 0;
4026+
assert.isAtMost(statusHeldForMs, 5_000, "status waited on the untracked line count");
4027+
assert.isTrue(reported.isRepo);
4028+
assert.strictEqual(reported.branch, "main");
4029+
}).pipe(Effect.provide(layerServerConfig)),
4030+
);
4031+
4032+
it.effect("counts totals when a large unstaged copy of a removed file stays on disk", () =>
4033+
Effect.gen(function* () {
4034+
const cwd = yield* makeTmpDir();
4035+
yield* initRepoWithCommit(cwd);
4036+
const driver = yield* GitVcsDriver.GitVcsDriver;
4037+
const fileSystem = yield* FileSystem.FileSystem;
4038+
const pathService = yield* Path.Path;
4039+
yield* writeTextFile(cwd, "asset.data", "old\n");
4040+
yield* git(cwd, ["add", "asset.data"]);
4041+
yield* git(cwd, ["commit", "-m", "add asset"]);
4042+
yield* git(cwd, ["rm", "--cached", "asset.data"]);
4043+
yield* fileSystem.truncate(pathService.join(cwd, "asset.data"), 257 * 1024 * 1024);
4044+
yield* writeTextFile(cwd, "ordinary.txt", "new\n");
4045+
4046+
const status = yield* driver.statusDetailsLocal(cwd, { includeBranchChanges: true });
4047+
4048+
assert.deepStrictEqual(status.branchChanges, {
4049+
baseRef: null,
4050+
insertions: 1,
4051+
deletions: 1,
4052+
});
4053+
}),
4054+
);
4055+
39634056
it.effect("reports changes to a file named HEAD", () =>
39644057
Effect.gen(function* () {
39654058
const cwd = yield* makeTmpDir();

‎apps/server/src/vcs/GitVcsDriverCore.ts‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,8 @@ const REVIEW_DIFF_PATCH_MAX_OUTPUT_BYTES = 120_000;
7777
const REVIEW_METADATA_MAX_OUTPUT_BYTES = 16 * 1024 * 1024;
7878
// Temporary-index git add scales poorly (50k files took ~30 s); report incomplete totals above this cap.
7979
const REVIEW_UNTRACKED_MAX_FILES = 5_000;
80+
// git diff --numstat reads every untracked byte (~5 ms/MB), so status totals skip larger untracked input.
81+
const UNTRACKED_TOTALS_MAX_BYTES = 256 * 1024 * 1024;
8082
// The listing walks the whole untracked tree before the cap applies, so a folder whose listing
8183
// failed (too many files, output limit, timeout) is not listed again on every status poll.
8284
const UNTRACKED_TOTALS_RETRY_MIN_MS = 60_000;
@@ -3133,6 +3135,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
31333135
const prepareReviewIndex = Effect.fn("prepareReviewIndex")(function* (
31343136
cwd: string,
31353137
untrackedPaths: ReadonlyArray<string>,
3138+
maxBytes?: number,
31363139
) {
31373140
const [stagedDeletionsStdout, indexValue] = yield* Effect.all(
31383141
[
@@ -3162,6 +3165,19 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
31623165
const stagedDeletions = new Set(stagedDeletionsStdout.split("\0").filter(Boolean));
31633166
const pathsToAdd = untrackedPaths.filter((relativePath) => !stagedDeletions.has(relativePath));
31643167
if (pathsToAdd.length === 0) return undefined;
3168+
if (maxBytes !== undefined) {
3169+
const sizes = yield* Effect.forEach(
3170+
pathsToAdd,
3171+
(relativePath) =>
3172+
fileSystem.lstat(path.resolve(cwd, relativePath)).pipe(
3173+
Effect.map((info) => (info.type === "File" ? Number(info.size) : 0)),
3174+
Effect.orElseSucceed(() => 0),
3175+
),
3176+
{ concurrency: 16 },
3177+
);
3178+
// null: over the byte budget, so the caller reports totals as unavailable.
3179+
if (sizes.reduce((total, size) => total + size, 0) > maxBytes) return null;
3180+
}
31653181

31663182
const indexPath = path.isAbsolute(indexValue.trim())
31673183
? indexValue.trim()
@@ -3231,6 +3247,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
32313247
cwd: string,
32323248
pathArgs: ReadonlyArray<string>,
32333249
onlyPath?: string,
3250+
maxBytes?: number,
32343251
) {
32353252
const untracked = yield* executeGit(
32363253
"GitVcsDriver.review.listUntracked",
@@ -3252,7 +3269,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
32523269
if (paths.length === 0) return { env: undefined };
32533270
if (paths.length > REVIEW_UNTRACKED_MAX_FILES)
32543271
return { unavailable: "Too many untracked files to count." };
3255-
const env = yield* prepareReviewIndex(cwd, paths).pipe(
3272+
const env = yield* prepareReviewIndex(cwd, paths, maxBytes).pipe(
32563273
Effect.catchTags({
32573274
PlatformError: (cause) =>
32583275
Effect.fail(
@@ -3266,6 +3283,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
32663283
),
32673284
}),
32683285
);
3286+
if (env === null) return { unavailable: "Untracked files are too large to count." };
32693287
return { env };
32703288
});
32713289
const prepareUntrackedReviewIndex = (
@@ -3365,7 +3383,12 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
33653383
});
33663384
}
33673385
// Empty status can mean untracked files are hidden; retain the full scan.
3368-
const untracked = yield* prepareUntrackedReviewIndexOrReason(cwd, pathspecs);
3386+
const untracked = yield* prepareUntrackedReviewIndexOrReason(
3387+
cwd,
3388+
pathspecs,
3389+
undefined,
3390+
UNTRACKED_TOTALS_MAX_BYTES,
3391+
);
33693392
if ("unavailable" in untracked) {
33703393
const count = remembered?.roots === rootsKey ? remembered.count + 1 : 1;
33713394
untrackedTotalsFailures.delete(cwd);

0 commit comments

Comments
 (0)