Skip to content

Commit 1601077

Browse files
yaacovcorcosclaude
andcommitted
fix(server): branch totals stop relisting untracked files after a failed listing
Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 7e565c5 commit 1601077

2 files changed

Lines changed: 169 additions & 7 deletions

File tree

‎apps/server/src/vcs/GitVcsDriverCore.test.ts‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6945,3 +6945,105 @@ describe("Windows long path configuration", () => {
69456945
}
69466946
});
69476947
});
6948+
6949+
// The listing walks the whole untracked tree before the 5,000-path cap applies, so a folder
6950+
// that failed once costs the same on every status poll. Branch totals remember the failure and
6951+
// back off until the untracked roots change.
6952+
it.layer(layerTest)("branch change totals after a failed untracked listing", (it) => {
6953+
// A listing that fails (here: output past the limit) backs off like one over the cap.
6954+
it.effect("do not repeat an untracked listing that failed", () =>
6955+
Effect.gen(function* () {
6956+
const fs = yield* FileSystem.FileSystem;
6957+
const path = yield* Path.Path;
6958+
const cwd = yield* makeTmpDir("git-untracked-error-");
6959+
yield* initRepoWithCommit(cwd);
6960+
yield* fs.writeFileString(path.join(cwd, "untracked.txt"), "new\n");
6961+
let listings = 0;
6962+
const delegate = yield* ChildProcessSpawner.ChildProcessSpawner;
6963+
// Stands in for a listing that floods its output past the 16 MiB limit.
6964+
const flood = ChildProcess.make(process.execPath, [
6965+
"-e",
6966+
"const b = Buffer.alloc(1 << 20, 120); (function w() { process.stdout.write(b, w); })();",
6967+
]);
6968+
const spawner = ChildProcessSpawner.make((command) =>
6969+
Effect.suspend(() => {
6970+
if (
6971+
ChildProcess.isStandardCommand(command) &&
6972+
command.args.includes("ls-files") &&
6973+
command.args.includes("--others")
6974+
) {
6975+
listings++;
6976+
return delegate.spawn(flood);
6977+
}
6978+
return delegate.spawn(command);
6979+
}),
6980+
);
6981+
const driver = yield* makeGitVcsDriverCore().pipe(
6982+
Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner),
6983+
Effect.provide(layerServerConfig),
6984+
);
6985+
const read = driver.statusDetailsLocal(cwd, { includeBranchChanges: true });
6986+
assert.strictEqual((yield* read).branchChanges, undefined);
6987+
yield* read;
6988+
assert.strictEqual(listings, 1, "the failed listing is not repeated within the backoff");
6989+
yield* TestClock.adjust("1 minute");
6990+
yield* read;
6991+
assert.strictEqual(listings, 2);
6992+
}),
6993+
);
6994+
6995+
it.effect("do not list untracked files again on every status read", () =>
6996+
Effect.gen(function* () {
6997+
const fs = yield* FileSystem.FileSystem;
6998+
const path = yield* Path.Path;
6999+
const cwd = yield* makeTmpDir("git-untracked-totals-");
7000+
yield* initRepoWithCommit(cwd);
7001+
const many = path.join(cwd, "many");
7002+
yield* fs.makeDirectory(many);
7003+
yield* Effect.forEach(
7004+
Array.from({ length: 5_001 }, (_, index) => index),
7005+
(index) => fs.writeFileString(path.join(many, `file-${index}`), ""),
7006+
{ concurrency: 64, discard: true },
7007+
);
7008+
7009+
let listings = 0;
7010+
const delegate = yield* ChildProcessSpawner.ChildProcessSpawner;
7011+
const spawner = ChildProcessSpawner.make((command) =>
7012+
Effect.suspend(() => {
7013+
if (
7014+
ChildProcess.isStandardCommand(command) &&
7015+
command.args.includes("ls-files") &&
7016+
command.args.includes("--others")
7017+
)
7018+
listings++;
7019+
return delegate.spawn(command);
7020+
}),
7021+
);
7022+
const driver = yield* makeGitVcsDriverCore().pipe(
7023+
Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner),
7024+
Effect.provide(layerServerConfig),
7025+
);
7026+
const read = driver.statusDetailsLocal(cwd, { includeBranchChanges: true });
7027+
7028+
assert.strictEqual((yield* read).branchChanges, undefined, "too many files to count");
7029+
assert.strictEqual(listings, 1);
7030+
yield* read;
7031+
yield* read;
7032+
assert.strictEqual(listings, 1, "the failed listing is not repeated within the backoff");
7033+
7034+
yield* TestClock.adjust("1 minute");
7035+
yield* read;
7036+
assert.strictEqual(listings, 2, "it is retried once the backoff passes");
7037+
yield* TestClock.adjust("1 minute");
7038+
yield* read;
7039+
assert.strictEqual(listings, 2, "and the next backoff is longer");
7040+
7041+
// A change at the untracked-roots level scans again at once and counts again.
7042+
yield* fs.remove(many, { recursive: true });
7043+
yield* writeTextFile(cwd, "notes.txt", "one\ntwo\n");
7044+
const status = yield* read;
7045+
assert.strictEqual(listings, 3);
7046+
assert.deepStrictEqual(status.branchChanges, { baseRef: null, insertions: 2, deletions: 0 });
7047+
}),
7048+
);
7049+
});

‎apps/server/src/vcs/GitVcsDriverCore.ts‎

Lines changed: 67 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,10 @@ const REVIEW_DIFF_PATCH_MAX_OUTPUT_BYTES = 120_000;
7777
const REVIEW_METADATA_MAX_OUTPUT_BYTES = 16 * 1024 * 1024;
7878
// Temporary-index git add scales poorly (50k files took ~30 s); report incomplete totals above this cap.
7979
const REVIEW_UNTRACKED_MAX_FILES = 5_000;
80+
// The listing walks the whole untracked tree before the cap applies, so a folder whose listing
81+
// failed (too many files, output limit, timeout) is not listed again on every status poll.
82+
const UNTRACKED_TOTALS_RETRY_MIN_MS = 60_000;
83+
const UNTRACKED_TOTALS_RETRY_MAX_MS = 30 * 60_000;
8084
const REVIEW_DIFF_FILE_MAX_OUTPUT_BYTES = 1024 * 1024;
8185
const REVIEW_BRANCH_COMMIT_LIMIT = 100;
8286
const REVIEW_BRANCH_SUBJECT_LIMIT = 300;
@@ -1623,6 +1627,15 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
16231627
});
16241628

16251629
const statusRemoteRefreshFailures = new Map<string, { count: number; retryAt: number }>();
1630+
const untrackedTotalsFailures = new Map<
1631+
string,
1632+
{
1633+
readonly roots: string;
1634+
readonly count: number;
1635+
readonly retryAt: number;
1636+
readonly detail: string;
1637+
}
1638+
>();
16261639
const statusRemoteRefreshFailureKey = (cacheKey: StatusRemoteRefreshCacheKey) =>
16271640
`${cacheKey.gitCommonDir}\0${cacheKey.remoteName}`;
16281641
const recordStatusRemoteRefreshFailure = Effect.fn(
@@ -3149,7 +3162,8 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
31493162
return stdout.trim();
31503163
});
31513164

3152-
const prepareUntrackedReviewIndex = Effect.fn("prepareUntrackedReviewIndex")(function* (
3165+
// `unavailable` says why the untracked files could not be indexed.
3166+
const prepareUntrackedReviewIndexOrReason = Effect.fn("prepareUntrackedReviewIndex")(function* (
31533167
cwd: string,
31543168
pathArgs: ReadonlyArray<string>,
31553169
onlyPath?: string,
@@ -3159,15 +3173,21 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
31593173
cwd,
31603174
["ls-files", "--others", "--exclude-standard", "-z", "--", ...pathArgs],
31613175
{ maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES },
3162-
).pipe(Effect.orElseSucceed(() => null));
3163-
if (untracked === null) return null;
3176+
).pipe(
3177+
Effect.catchTags({
3178+
GitCommandError: (error) =>
3179+
Effect.succeed({ unavailable: `Could not list untracked files: ${error.detail}` }),
3180+
}),
3181+
);
3182+
if ("unavailable" in untracked) return untracked;
31643183
// ls-files lists a nested repository with a trailing slash; its diff path has none.
31653184
const paths = splitNullSeparatedGitStdoutPaths(untracked).filter(
31663185
(candidate) =>
31673186
onlyPath === undefined || candidate === onlyPath || candidate === `${onlyPath}/`,
31683187
);
31693188
if (paths.length === 0) return { env: undefined };
3170-
if (paths.length > REVIEW_UNTRACKED_MAX_FILES) return null;
3189+
if (paths.length > REVIEW_UNTRACKED_MAX_FILES)
3190+
return { unavailable: "Too many untracked files to count." };
31713191
const env = yield* prepareReviewIndex(cwd, paths).pipe(
31723192
Effect.catchTags({
31733193
PlatformError: (cause) =>
@@ -3184,6 +3204,14 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
31843204
);
31853205
return { env };
31863206
});
3207+
const prepareUntrackedReviewIndex = (
3208+
cwd: string,
3209+
pathArgs: ReadonlyArray<string>,
3210+
onlyPath?: string,
3211+
) =>
3212+
prepareUntrackedReviewIndexOrReason(cwd, pathArgs, onlyPath).pipe(
3213+
Effect.map((result) => ("unavailable" in result ? null : result)),
3214+
);
31873215

31883216
const resolveReviewMergeBase = Effect.fn("resolveReviewMergeBase")(function* (
31893217
cwd: string,
@@ -3257,16 +3285,48 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
32573285
roots.length <= 128 && roots.reduce((length, root) => length + root.length + 16, 0) < 8_000
32583286
? roots.map((root) => `:(top,literal)${root}`)
32593287
: [];
3288+
// Until the retry time, the same untracked roots get the remembered failure without a scan;
3289+
// different roots (files added or removed at status level) scan again at once.
3290+
const rootsKey = roots.join("\0");
3291+
const remembered = untrackedTotalsFailures.get(cwd);
3292+
if (
3293+
remembered !== undefined &&
3294+
remembered.roots === rootsKey &&
3295+
(yield* Clock.currentTimeMillis) < remembered.retryAt
3296+
) {
3297+
return yield* new GitCommandError({
3298+
operation: "GitVcsDriver.readBranchChangeTotals",
3299+
command: "git ls-files",
3300+
cwd,
3301+
detail: remembered.detail,
3302+
});
3303+
}
32603304
// Empty status can mean untracked files are hidden; retain the full scan.
3261-
const untracked = yield* prepareUntrackedReviewIndex(cwd, pathspecs);
3262-
if (untracked === null) {
3305+
const untracked = yield* prepareUntrackedReviewIndexOrReason(cwd, pathspecs);
3306+
if ("unavailable" in untracked) {
3307+
const count = remembered?.roots === rootsKey ? remembered.count + 1 : 1;
3308+
// Bounded like the remote refresh failures: the oldest folder is forgotten first.
3309+
untrackedTotalsFailures.delete(cwd);
3310+
untrackedTotalsFailures.set(cwd, {
3311+
roots: rootsKey,
3312+
count,
3313+
retryAt:
3314+
(yield* Clock.currentTimeMillis) +
3315+
Math.min(UNTRACKED_TOTALS_RETRY_MIN_MS * 2 ** (count - 1), UNTRACKED_TOTALS_RETRY_MAX_MS),
3316+
detail: untracked.unavailable,
3317+
});
3318+
if (untrackedTotalsFailures.size > STATUS_UPSTREAM_REFRESH_CACHE_CAPACITY) {
3319+
const oldestKey = untrackedTotalsFailures.keys().next().value;
3320+
if (oldestKey !== undefined) untrackedTotalsFailures.delete(oldestKey);
3321+
}
32633322
return yield* new GitCommandError({
32643323
operation: "GitVcsDriver.readBranchChangeTotals",
32653324
command: "git ls-files",
32663325
cwd,
3267-
detail: "Too many untracked files to count.",
3326+
detail: untracked.unavailable,
32683327
});
32693328
}
3329+
untrackedTotalsFailures.delete(cwd);
32703330
const readNumstat = (ref: string) =>
32713331
executeGit(
32723332
"GitVcsDriver.readBranchChangeTotals",

0 commit comments

Comments
 (0)