Skip to content

Commit 3123830

Browse files
fix(server): review diffs no longer read oversized untracked files
Review previews put every untracked file into the temporary review index, so git read all of their bytes; only the status totals had a size budget. The preview now uses the same 256 MiB budget. Files that do not fit stay out of the index, smallest files are kept first, and the source is reported the way an unlistable untracked tree already is: no file list, marked truncated, with the bounded patch of everything that did fit. Staged deletions are dropped before sizes are counted, so their worktree copies cannot spend the budget. Co-authored-by: luis-7654 <[email protected]> Co-Authored-By: Claude Fable 5.2 <[email protected]>
1 parent 9efd960 commit 3123830

2 files changed

Lines changed: 72 additions & 34 deletions

File tree

‎apps/server/src/vcs/GitVcsDriverCore.test.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4323,6 +4323,39 @@ it.layer(layerTest)("GitVcsDriver core integration", (it) => {
43234323
insertions: 1,
43244324
deletions: 1,
43254325
});
4326+
const preview = yield* driver.getReviewDiffPreview({ cwd, metadataOnly: true });
4327+
assert.deepStrictEqual(
4328+
preview.sources.find((source) => source.kind === "working-tree")?.files,
4329+
[
4330+
{ path: "asset.data", previousPath: null, additions: 0, deletions: 1 },
4331+
{ path: "ordinary.txt", previousPath: null, additions: 1, deletions: 0 },
4332+
],
4333+
);
4334+
}),
4335+
);
4336+
4337+
it.effect("keeps an oversized untracked file out of the review diff", () =>
4338+
Effect.gen(function* () {
4339+
const cwd = yield* makeTmpDir();
4340+
yield* initRepoWithCommit(cwd);
4341+
const driver = yield* GitVcsDriver.GitVcsDriver;
4342+
const fileSystem = yield* FileSystem.FileSystem;
4343+
const pathService = yield* Path.Path;
4344+
yield* writeTextFile(cwd, "README.md", "# tracked change\n");
4345+
yield* writeTextFile(cwd, "ordinary.txt", "new\n");
4346+
yield* writeTextFile(cwd, "large.data", "");
4347+
// Sparse: the logical size is over the budget without allocating it.
4348+
yield* fileSystem.truncate(pathService.join(cwd, "large.data"), 257 * 1024 * 1024);
4349+
4350+
const preview = yield* driver.getReviewDiffPreview({ cwd, metadataOnly: true });
4351+
4352+
const source = preview.sources.find((source) => source.kind === "working-tree")!;
4353+
assert.isUndefined(source.files);
4354+
assert.isUndefined(source.patchesDeferred);
4355+
assert.isTrue(source.truncated);
4356+
assert.include(source.diff, "+# tracked change");
4357+
assert.include(source.diff, "b/ordinary.txt");
4358+
assert.notInclude(source.diff, "large.data");
43264359
}),
43274360
);
43284361

‎apps/server/src/vcs/GitVcsDriverCore.ts‎

Lines changed: 39 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -81,8 +81,8 @@ const REVIEW_DIFF_PATCH_MAX_OUTPUT_BYTES = 120_000;
8181
const REVIEW_METADATA_MAX_OUTPUT_BYTES = 16 * 1024 * 1024;
8282
// Temporary-index git add scales poorly (50k files took ~30 s); report incomplete totals above this cap.
8383
const REVIEW_UNTRACKED_MAX_FILES = 5_000;
84-
// git diff --numstat reads every untracked byte (~5 ms/MB), so status totals skip larger untracked input.
85-
const UNTRACKED_TOTALS_MAX_BYTES = 256 * 1024 * 1024;
84+
// git diff reads every untracked byte (~5 ms/MB), so larger untracked input stays out of the review index.
85+
const REVIEW_UNTRACKED_MAX_BYTES = 256 * 1024 * 1024;
8686
// The listing walks the whole untracked tree before the cap applies, so a folder whose listing
8787
// failed (too many files, output limit, timeout) is not listed again on every status poll.
8888
const UNTRACKED_TOTALS_RETRY_MIN_MS = 60_000;
@@ -3242,7 +3242,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
32423242
const prepareReviewIndex = Effect.fn("prepareReviewIndex")(function* (
32433243
cwd: string,
32443244
untrackedPaths: ReadonlyArray<string>,
3245-
maxBytes?: number,
3245+
allowPartial: boolean,
32463246
) {
32473247
const [stagedDeletionsStdout, indexValue] = yield* Effect.all(
32483248
[
@@ -3270,21 +3270,28 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
32703270
{ concurrency: 2 },
32713271
);
32723272
const stagedDeletions = new Set(stagedDeletionsStdout.split("\0").filter(Boolean));
3273-
const pathsToAdd = untrackedPaths.filter((relativePath) => !stagedDeletions.has(relativePath));
3274-
if (pathsToAdd.length === 0) return undefined;
3275-
if (maxBytes !== undefined) {
3276-
const sizes = yield* Effect.forEach(
3277-
pathsToAdd,
3278-
(relativePath) =>
3279-
fileSystem.lstat(path.resolve(cwd, relativePath)).pipe(
3280-
Effect.map((info) => (info.type === "File" ? Number(info.size) : 0)),
3281-
Effect.orElseSucceed(() => 0),
3282-
),
3283-
{ concurrency: 16 },
3284-
);
3285-
// null: over the byte budget, so the caller reports totals as unavailable.
3286-
if (sizes.reduce((total, size) => total + size, 0) > maxBytes) return null;
3273+
// Staged deletions never enter the index, so their worktree copies must not spend the budget.
3274+
const candidates = untrackedPaths.filter((relativePath) => !stagedDeletions.has(relativePath));
3275+
const sizes = yield* Effect.forEach(
3276+
candidates,
3277+
(relativePath) =>
3278+
fileSystem.lstat(path.resolve(cwd, relativePath)).pipe(
3279+
Effect.map((info) => (info.type === "File" ? Number(info.size) : 0)),
3280+
Effect.orElseSucceed(() => 0),
3281+
Effect.map((size) => ({ relativePath, size })),
3282+
),
3283+
{ concurrency: 16 },
3284+
);
3285+
// Smallest first, so which files fit does not depend on how their names sort.
3286+
const pathsToAdd: string[] = [];
3287+
let bytes = 0;
3288+
for (const { relativePath, size } of sizes.toSorted((left, right) => left.size - right.size)) {
3289+
bytes += size;
3290+
if (bytes > REVIEW_UNTRACKED_MAX_BYTES) break;
3291+
pathsToAdd.push(relativePath);
32873292
}
3293+
const omitted = pathsToAdd.length < candidates.length;
3294+
if (pathsToAdd.length === 0 || (omitted && !allowPartial)) return { env: undefined, omitted };
32883295

32893296
const indexPath = path.isAbsolute(indexValue.trim())
32903297
? indexValue.trim()
@@ -3337,7 +3344,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
33373344
],
33383345
{ env, stdin: `${pathsToAdd.join("\0")}\0` },
33393346
);
3340-
return env;
3347+
return { env, omitted };
33413348
});
33423349

33433350
const readEmptyTreeHash = Effect.fn("readEmptyTreeHash")(function* (cwd: string) {
@@ -3353,8 +3360,8 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
33533360
const prepareUntrackedReviewIndexOrReason = Effect.fn("prepareUntrackedReviewIndex")(function* (
33543361
cwd: string,
33553362
pathArgs: ReadonlyArray<string>,
3356-
onlyPath?: string,
3357-
maxBytes?: number,
3363+
onlyPath: string | undefined,
3364+
allowPartial: boolean,
33583365
) {
33593366
const untracked = yield* executeGit(
33603367
"GitVcsDriver.review.listUntracked",
@@ -3373,10 +3380,10 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
33733380
(candidate) =>
33743381
onlyPath === undefined || candidate === onlyPath || candidate === `${onlyPath}/`,
33753382
);
3376-
if (paths.length === 0) return { env: undefined };
3383+
if (paths.length === 0) return { env: undefined, omitted: false };
33773384
if (paths.length > REVIEW_UNTRACKED_MAX_FILES)
33783385
return { unavailable: "Too many untracked files to count." };
3379-
const env = yield* prepareReviewIndex(cwd, paths, maxBytes).pipe(
3386+
const index = yield* prepareReviewIndex(cwd, paths, allowPartial).pipe(
33803387
Effect.catchTags({
33813388
PlatformError: (cause) =>
33823389
Effect.fail(
@@ -3390,15 +3397,16 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
33903397
),
33913398
}),
33923399
);
3393-
if (env === null) return { unavailable: "Untracked files are too large to count." };
3394-
return { env };
3400+
if (index.omitted && !allowPartial)
3401+
return { unavailable: "Untracked files are too large to count." };
3402+
return index;
33953403
});
33963404
const prepareUntrackedReviewIndex = (
33973405
cwd: string,
33983406
pathArgs: ReadonlyArray<string>,
33993407
onlyPath?: string,
34003408
) =>
3401-
prepareUntrackedReviewIndexOrReason(cwd, pathArgs, onlyPath).pipe(
3409+
prepareUntrackedReviewIndexOrReason(cwd, pathArgs, onlyPath, true).pipe(
34023410
Effect.map((result) => ("unavailable" in result ? null : result)),
34033411
);
34043412

@@ -3490,12 +3498,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
34903498
});
34913499
}
34923500
// Empty status can mean untracked files are hidden; retain the full scan.
3493-
const untracked = yield* prepareUntrackedReviewIndexOrReason(
3494-
cwd,
3495-
pathspecs,
3496-
undefined,
3497-
UNTRACKED_TOTALS_MAX_BYTES,
3498-
);
3501+
const untracked = yield* prepareUntrackedReviewIndexOrReason(cwd, pathspecs, undefined, false);
34993502
if ("unavailable" in untracked) {
35003503
const count = remembered?.roots === rootsKey ? remembered.count + 1 : 1;
35013504
untrackedTotalsFailures.delete(cwd);
@@ -3846,9 +3849,11 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
38463849
});
38473850
const [dirtyTrackedResult, baseResult] = yield* Effect.gen(function* () {
38483851
const untracked = input.trackedOnly
3849-
? { env: undefined }
3852+
? { env: undefined, omitted: false }
38503853
: yield* prepareUntrackedReviewIndex(cwd, pathArgs, input.file?.path);
3851-
const metadataOnly = input.metadataOnly === true && !input.file && untracked !== null;
3854+
// Untracked files left out make the file list partial, so the bounded patch is sent instead.
3855+
const complete = untracked !== null && !untracked.omitted;
3856+
const metadataOnly = input.metadataOnly === true && !input.file && complete;
38523857
const [dirty, base] =
38533858
review.mergeBase === dirtyRef
38543859
? yield* readTrackedDiff(dirtyRef, untracked?.env, metadataOnly).pipe(
@@ -3861,7 +3866,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
38613866
],
38623867
{ concurrency: 2 },
38633868
);
3864-
if (untracked !== null) return [dirty, base] as const;
3869+
if (complete) return [dirty, base] as const;
38653870
const incomplete = (ref: string | null, result: typeof dirty) =>
38663871
ref === null ? result : { ...result, files: undefined, stdoutTruncated: true };
38673872
return [incomplete(dirtyRef, dirty), incomplete(review.mergeBase, base)] as const;

0 commit comments

Comments
 (0)