Skip to content

Commit 9994396

Browse files
fix(server): branch totals stop relisting untracked files after a failed listing (pingdotgg#34460)
Co-authored-by: Julius Marminge <[email protected]>
1 parent a8ccf14 commit 9994396

2 files changed

Lines changed: 121 additions & 7 deletions

File tree

‎apps/server/src/vcs/GitVcsDriverCore.test.ts‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6993,3 +6993,60 @@ describe("Windows long path configuration", () => {
69936993
}
69946994
});
69956995
});
6996+
6997+
it.layer(layerTest)("branch change totals after a failed untracked listing", (it) => {
6998+
it.effect("do not list untracked files again on every status read", () =>
6999+
Effect.gen(function* () {
7000+
const fs = yield* FileSystem.FileSystem;
7001+
const path = yield* Path.Path;
7002+
const cwd = yield* makeTmpDir("git-untracked-totals-");
7003+
yield* initRepoWithCommit(cwd);
7004+
const many = path.join(cwd, "many");
7005+
yield* fs.makeDirectory(many);
7006+
yield* Effect.forEach(
7007+
Array.from({ length: 5_001 }, (_, index) => index),
7008+
(index) => fs.writeFileString(path.join(many, `file-${index}`), ""),
7009+
{ concurrency: 64, discard: true },
7010+
);
7011+
7012+
let listings = 0;
7013+
const delegate = yield* ChildProcessSpawner.ChildProcessSpawner;
7014+
const spawner = ChildProcessSpawner.make((command) =>
7015+
Effect.suspend(() => {
7016+
if (
7017+
ChildProcess.isStandardCommand(command) &&
7018+
command.args.includes("ls-files") &&
7019+
command.args.includes("--others")
7020+
)
7021+
listings++;
7022+
return delegate.spawn(command);
7023+
}),
7024+
);
7025+
const driver = yield* makeGitVcsDriverCore().pipe(
7026+
Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner),
7027+
Effect.provide(layerServerConfig),
7028+
);
7029+
const read = driver.statusDetailsLocal(cwd, { includeBranchChanges: true });
7030+
7031+
assert.strictEqual((yield* read).branchChanges, undefined, "too many files to count");
7032+
assert.strictEqual(listings, 1);
7033+
yield* read;
7034+
yield* read;
7035+
assert.strictEqual(listings, 1, "the failed listing is not repeated within the backoff");
7036+
7037+
yield* TestClock.adjust("1 minute");
7038+
yield* read;
7039+
assert.strictEqual(listings, 2, "it is retried once the backoff passes");
7040+
yield* TestClock.adjust("1 minute");
7041+
yield* read;
7042+
assert.strictEqual(listings, 2, "and the next backoff is longer");
7043+
7044+
// A change at the untracked-roots level scans again at once and counts again.
7045+
yield* fs.remove(many, { recursive: true });
7046+
yield* writeTextFile(cwd, "notes.txt", "one\ntwo\n");
7047+
const status = yield* read;
7048+
assert.strictEqual(listings, 3);
7049+
assert.deepStrictEqual(status.branchChanges, { baseRef: null, insertions: 2, deletions: 0 });
7050+
}),
7051+
);
7052+
});

‎apps/server/src/vcs/GitVcsDriverCore.ts‎

Lines changed: 64 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,10 @@ const REVIEW_DIFF_PATCH_MAX_OUTPUT_BYTES = 120_000;
7777
const REVIEW_METADATA_MAX_OUTPUT_BYTES = 16 * 1024 * 1024;
7878
// Temporary-index git add scales poorly (50k files took ~30 s); report incomplete totals above this cap.
7979
const REVIEW_UNTRACKED_MAX_FILES = 5_000;
80+
// The listing walks the whole untracked tree before the cap applies, so a folder whose listing
81+
// failed (too many files, output limit, timeout) is not listed again on every status poll.
82+
const UNTRACKED_TOTALS_RETRY_MIN_MS = 60_000;
83+
const UNTRACKED_TOTALS_RETRY_MAX_MS = 30 * 60_000;
8084
const REVIEW_DIFF_FILE_MAX_OUTPUT_BYTES = 1024 * 1024;
8185
const REVIEW_BRANCH_COMMIT_LIMIT = 100;
8286
const REVIEW_BRANCH_SUBJECT_LIMIT = 300;
@@ -1623,6 +1627,15 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
16231627
});
16241628

16251629
const statusRemoteRefreshFailures = new Map<string, { count: number; retryAt: number }>();
1630+
const untrackedTotalsFailures = new Map<
1631+
string,
1632+
{
1633+
readonly roots: string;
1634+
readonly count: number;
1635+
readonly retryAt: number;
1636+
readonly detail: string;
1637+
}
1638+
>();
16261639
const statusRemoteRefreshFailureKey = (cacheKey: StatusRemoteRefreshCacheKey) =>
16271640
`${cacheKey.gitCommonDir}\0${cacheKey.remoteName}`;
16281641
const recordStatusRemoteRefreshFailure = Effect.fn(
@@ -3149,7 +3162,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
31493162
return stdout.trim();
31503163
});
31513164

3152-
const prepareUntrackedReviewIndex = Effect.fn("prepareUntrackedReviewIndex")(function* (
3165+
const prepareUntrackedReviewIndexOrReason = Effect.fn("prepareUntrackedReviewIndex")(function* (
31533166
cwd: string,
31543167
pathArgs: ReadonlyArray<string>,
31553168
onlyPath?: string,
@@ -3159,15 +3172,21 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
31593172
cwd,
31603173
["ls-files", "--others", "--exclude-standard", "-z", "--", ...pathArgs],
31613174
{ maxOutputBytes: REVIEW_METADATA_MAX_OUTPUT_BYTES },
3162-
).pipe(Effect.orElseSucceed(() => null));
3163-
if (untracked === null) return null;
3175+
).pipe(
3176+
Effect.catchTags({
3177+
GitCommandError: (error) =>
3178+
Effect.succeed({ unavailable: `Could not list untracked files: ${error.detail}` }),
3179+
}),
3180+
);
3181+
if ("unavailable" in untracked) return untracked;
31643182
// ls-files lists a nested repository with a trailing slash; its diff path has none.
31653183
const paths = splitNullSeparatedGitStdoutPaths(untracked).filter(
31663184
(candidate) =>
31673185
onlyPath === undefined || candidate === onlyPath || candidate === `${onlyPath}/`,
31683186
);
31693187
if (paths.length === 0) return { env: undefined };
3170-
if (paths.length > REVIEW_UNTRACKED_MAX_FILES) return null;
3188+
if (paths.length > REVIEW_UNTRACKED_MAX_FILES)
3189+
return { unavailable: "Too many untracked files to count." };
31713190
const env = yield* prepareReviewIndex(cwd, paths).pipe(
31723191
Effect.catchTags({
31733192
PlatformError: (cause) =>
@@ -3184,6 +3203,14 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
31843203
);
31853204
return { env };
31863205
});
3206+
const prepareUntrackedReviewIndex = (
3207+
cwd: string,
3208+
pathArgs: ReadonlyArray<string>,
3209+
onlyPath?: string,
3210+
) =>
3211+
prepareUntrackedReviewIndexOrReason(cwd, pathArgs, onlyPath).pipe(
3212+
Effect.map((result) => ("unavailable" in result ? null : result)),
3213+
);
31873214

31883215
const resolveReviewMergeBase = Effect.fn("resolveReviewMergeBase")(function* (
31893216
cwd: string,
@@ -3257,16 +3284,46 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
32573284
roots.length <= 128 && roots.reduce((length, root) => length + root.length + 16, 0) < 8_000
32583285
? roots.map((root) => `:(top,literal)${root}`)
32593286
: [];
3287+
// Changed untracked roots skip the backoff and scan again.
3288+
const rootsKey = roots.join("\0");
3289+
const remembered = untrackedTotalsFailures.get(cwd);
3290+
if (
3291+
remembered !== undefined &&
3292+
remembered.roots === rootsKey &&
3293+
(yield* Clock.currentTimeMillis) < remembered.retryAt
3294+
) {
3295+
return yield* new GitCommandError({
3296+
operation: "GitVcsDriver.readBranchChangeTotals",
3297+
command: "git ls-files",
3298+
cwd,
3299+
detail: remembered.detail,
3300+
});
3301+
}
32603302
// Empty status can mean untracked files are hidden; retain the full scan.
3261-
const untracked = yield* prepareUntrackedReviewIndex(cwd, pathspecs);
3262-
if (untracked === null) {
3303+
const untracked = yield* prepareUntrackedReviewIndexOrReason(cwd, pathspecs);
3304+
if ("unavailable" in untracked) {
3305+
const count = remembered?.roots === rootsKey ? remembered.count + 1 : 1;
3306+
untrackedTotalsFailures.delete(cwd);
3307+
untrackedTotalsFailures.set(cwd, {
3308+
roots: rootsKey,
3309+
count,
3310+
retryAt:
3311+
(yield* Clock.currentTimeMillis) +
3312+
Math.min(UNTRACKED_TOTALS_RETRY_MIN_MS * 2 ** (count - 1), UNTRACKED_TOTALS_RETRY_MAX_MS),
3313+
detail: untracked.unavailable,
3314+
});
3315+
if (untrackedTotalsFailures.size > STATUS_UPSTREAM_REFRESH_CACHE_CAPACITY) {
3316+
const oldestKey = untrackedTotalsFailures.keys().next().value;
3317+
if (oldestKey !== undefined) untrackedTotalsFailures.delete(oldestKey);
3318+
}
32633319
return yield* new GitCommandError({
32643320
operation: "GitVcsDriver.readBranchChangeTotals",
32653321
command: "git ls-files",
32663322
cwd,
3267-
detail: "Too many untracked files to count.",
3323+
detail: untracked.unavailable,
32683324
});
32693325
}
3326+
untrackedTotalsFailures.delete(cwd);
32703327
const readNumstat = (ref: string) =>
32713328
executeGit(
32723329
"GitVcsDriver.readBranchChangeTotals",

0 commit comments

Comments
 (0)