Skip to content

Commit 07a1e79

Browse files
fix(server): checkpoints no longer rehash every file when one entry is skip-worktree or assume-unchanged (#23624)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 42ca7c0 commit 07a1e79

2 files changed

Lines changed: 112 additions & 38 deletions

File tree

‎apps/server/src/vcs/GitVcsDriver.test.ts‎

Lines changed: 38 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -597,7 +597,17 @@ it.effect.each(
597597
Effect.gen(function* () {
598598
const fs = yield* FileSystem.FileSystem;
599599
const path = yield* Path.Path;
600-
const driver = yield* GitVcsDriver.makeVcsDriverShape();
600+
const liveProcess = yield* VcsProcess.VcsProcess;
601+
let freshReadTrees = 0;
602+
const driver = yield* GitVcsDriver.makeVcsDriverShape().pipe(
603+
Effect.provideService(VcsProcess.VcsProcess, {
604+
run: (input) => {
605+
if (input.args.includes("read-tree") && input.args.includes("index.sparse=true"))
606+
freshReadTrees++;
607+
return liveProcess.run(input);
608+
},
609+
}),
610+
);
601611
const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-checkpoint-sparse-" });
602612
const { git } = yield* makeCheckpointFixture(driver, cwd);
603613
const write = Effect.fn(function* (name: string, contents: string) {
@@ -680,11 +690,12 @@ it.effect.each(
680690
"working outside\n",
681691
);
682692
}
693+
assert.strictEqual(freshReadTrees, indexState === "missing" ? 2 : 0);
683694
}).pipe(Effect.scoped, Effect.provide(layerGitContract)),
684695
);
685696

686697
it.effect.each(
687-
(["normal", "flags", "sparse"] as const).flatMap((indexMode) =>
698+
(["normal", "skip-worktree", "assume-unchanged", "sparse"] as const).flatMap((indexMode) =>
688699
[1, 3, 4096].map((chunkSize) => ({ indexMode, chunkSize })),
689700
),
690701
)(
@@ -700,26 +711,39 @@ it.effect.each(
700711
yield* fs.writeFileString(path.join(cwd, ".gitattributes"), "stable filter=probe\n");
701712
yield* fs.writeFileString(path.join(cwd, "stable"), "unchanged\n");
702713
yield* fs.writeFileString(path.join(cwd, "z-skipped"), "original\n");
714+
yield* fs.writeFileString(path.join(cwd, "z-deleted"), "original\n");
703715
yield* fs.makeDirectory(path.join(cwd, "excluded"));
704716
yield* fs.writeFileString(path.join(cwd, "excluded/file"), "absent\n");
705717
yield* fs.writeFileString(
706718
path.join(cwd, ".git/filter.cjs"),
707719
'require("node:fs").appendFileSync(".git/reads", "read\\n"); process.stdin.pipe(process.stdout);',
708720
);
709721
yield* git(["config", "filter.probe.clean", "node .git/filter.cjs"]);
722+
yield* git(["config", "core.trustctime", "false"]);
710723
yield* fs.utimes(path.join(cwd, "stable"), 1_700_000_000, 1_700_000_000);
724+
yield* fs.utimes(path.join(cwd, "z-skipped"), 1_700_000_000, 1_700_000_000);
711725
yield* git(["add", "."]);
712726
yield* git(["commit", "-m", "inspection fixture"]);
713-
if (indexMode === "flags") yield* git(["update-index", "--skip-worktree", "z-skipped"]);
727+
const flagged = indexMode === "skip-worktree" || indexMode === "assume-unchanged";
728+
if (flagged) {
729+
yield* git(["update-index", `--${indexMode}`, "z-skipped", "z-deleted"]);
730+
yield* git(["config", "core.ignorestat", "true"]);
731+
yield* fs.remove(path.join(cwd, "z-deleted"));
732+
}
714733
if (indexMode === "sparse")
715734
yield* git(["sparse-checkout", "set", "--cone", "--sparse-index", "included"]);
716735
yield* fs.writeFileString(path.join(cwd, "z-skipped"), "modified\n");
736+
// A flagged edit with the indexed size and mtime is invisible to Git's stat check.
737+
if (flagged) yield* fs.utimes(path.join(cwd, "z-skipped"), 1_700_000_000, 1_700_000_000);
717738
yield* fs.writeFileString(path.join(cwd, ".git/reads"), "");
718739
const originalIndex = yield* fs.readFile(path.join(cwd, ".git/index"));
740+
let freshReadTrees = 0;
719741
const captureDriver = yield* GitVcsDriver.makeVcsDriverShape().pipe(
720742
Effect.provideService(VcsProcess.VcsProcess, {
721-
run: (input) =>
722-
liveProcess.run(
743+
run: (input) => {
744+
if (input.args.includes("read-tree") && !input.args.includes("--reset"))
745+
freshReadTrees++;
746+
return liveProcess.run(
723747
input.args.includes("ls-files")
724748
? {
725749
...input,
@@ -730,13 +754,19 @@ it.effect.each(
730754
},
731755
}
732756
: input,
733-
),
757+
);
758+
},
734759
}),
735760
);
736761
yield* captureDriver.checkpoints.captureCheckpoint({ cwd, checkpointRef });
737762
assert.strictEqual((yield* git(["show", `${checkpointRef}:z-skipped`])).stdout, "modified\n");
738-
if (indexMode !== "flags")
739-
assert.strictEqual(yield* fs.readFileString(path.join(cwd, ".git/reads")), "");
763+
const files = (yield* git(["ls-tree", "-z", "--name-only", checkpointRef])).stdout.split(
764+
"\0",
765+
);
766+
if (flagged) assert.notInclude(files, "z-deleted");
767+
else assert.include(files, "z-deleted");
768+
assert.strictEqual(freshReadTrees, 0);
769+
assert.strictEqual(yield* fs.readFileString(path.join(cwd, ".git/reads")), "");
740770
assert.deepEqual(yield* fs.readFile(path.join(cwd, ".git/index")), originalIndex);
741771
}).pipe(Effect.scoped, Effect.provide(layerGitContract)),
742772
);

‎apps/server/src/vcs/GitVcsDriver.ts‎

Lines changed: 74 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -444,6 +444,7 @@ export class GitVcsDriver extends Context.Service<
444444
>()("t3/vcs/GitVcsDriver") {}
445445

446446
const WORKSPACE_FILES_MAX_OUTPUT_BYTES = 16 * 1024 * 1024;
447+
const FLAGGED_INDEX_ENTRIES_MAX_BYTES = 1024 * 1024;
447448
const CHECKPOINT_RECOVERY_MAX_CANDIDATES = 64;
448449
const CHECKPOINT_RECOVERY_TIMEOUT = "5 seconds";
449450
const GIT_CHECK_IGNORE_MAX_STDIN_BYTES = 256 * 1024;
@@ -967,43 +968,60 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
967968
});
968969
// read-tree can rewrite the index, so restore its racy timestamp afterward.
969970
yield* fileSystem.utimes(tempIndexPath, indexTime, indexTime);
970-
let specialFlags = false;
971+
// Manually flagged entries hide edits from add -A, so they are re-added below
972+
// with no stat cache. Every other entry keeps its stat data and is not rehashed.
973+
let fallback = false;
974+
let flaggedBytes = 0;
971975
let recordStart = true;
972-
let skipped = false;
973-
let skippedRecord: Uint8Array[] = [];
974-
const skippedPaths: string[] = [];
976+
let collect: "flagged" | "skipped" | undefined;
977+
let record: Uint8Array[] = [];
978+
const flagged: string[] = [];
979+
const skipped: string[] = [];
975980
yield* vcsProcess.run({
976981
operation,
977982
command: "git",
978983
cwd: input.cwd,
979-
args: [...indexConfig, "ls-files", "--full-name", "--sparse", "-v", "-z"],
984+
args: [...indexConfig, "ls-files", "--full-name", "--sparse", "--stage", "-v", "-z"],
980985
env: commitEnv,
981986
maxOutputBytes: 4_096,
982987
outputMode: "truncate",
983-
// Only tags and sparse skip-worktree paths matter; scan NUL-delimited records.
988+
// Only tags and flagged or skip-worktree records matter; scan NUL-delimited records.
984989
onStdoutChunk: (chunk) => {
985990
let start = 0;
986-
while (start < chunk.length) {
991+
while (start < chunk.length && !fallback) {
987992
const end = chunk.indexOf(0, start);
988993
if (recordStart) {
989994
const tag = chunk[start]!;
990-
skipped = tag === 83;
991-
if ((tag >= 97 && tag <= 122) || (!sparseCheckout && tag === 83)) {
992-
specialFlags = true;
993-
}
995+
record = [];
996+
// Sparse skip-worktree bits are expected; only in-cone ones are manual.
997+
collect =
998+
tag === 83 && sparseCheckout
999+
? "skipped"
1000+
: (tag >= 97 && tag <= 122) || tag === 83
1001+
? "flagged"
1002+
: undefined;
1003+
// index-info cannot keep skip-worktree while dropping assume-unchanged.
1004+
if (tag === 115 && sparseCheckout) fallback = true;
9941005
}
995-
if (skipped && sparseCheckout) {
996-
skippedRecord.push(
997-
Buffer.from(chunk.subarray(start, end === -1 ? undefined : end)),
998-
);
999-
if (end !== -1) {
1000-
const record = Buffer.concat(skippedRecord);
1001-
if (record.at(-1) !== 47) {
1002-
const name = record.subarray(2);
1003-
if (!NodeBuffer.isUtf8(name)) specialFlags = true;
1004-
else skippedPaths.push(name.toString("utf8"));
1006+
if (collect !== undefined && !fallback) {
1007+
const piece = chunk.subarray(start, end === -1 ? undefined : end);
1008+
record.push(Buffer.from(piece));
1009+
if (
1010+
collect === "flagged" &&
1011+
(flaggedBytes += piece.length) > FLAGGED_INDEX_ENTRIES_MAX_BYTES
1012+
)
1013+
fallback = true;
1014+
if (end !== -1 && !fallback) {
1015+
const full = Buffer.concat(record);
1016+
record = [];
1017+
// Sparse directory entries stay collapsed.
1018+
if (collect === "flagged" || full.at(-1) !== 47) {
1019+
// Drop the "<tag> " prefix, leaving index-info's "<mode> <oid> <stage>\t<path>".
1020+
const entry = full.subarray(2);
1021+
if (!NodeBuffer.isUtf8(entry)) fallback = true;
1022+
else
1023+
(collect === "flagged" ? flagged : skipped).push(entry.toString("utf8"));
10051024
}
1006-
skippedRecord = [];
10071025
}
10081026
}
10091027
recordStart = end !== -1;
@@ -1012,20 +1030,46 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
10121030
}
10131031
},
10141032
});
1015-
if (skippedPaths.length > 0 && !specialFlags) {
1016-
const selected = yield* execute({
1033+
if (fallback) return false;
1034+
if (skipped.length > 0) {
1035+
const paths = skipped.map((entry) => entry.slice(entry.indexOf("\t") + 1));
1036+
const stdin = paths.join("\0") + "\0";
1037+
// check-rules echoes a subset of its input, so the input size bounds its output.
1038+
const rules = yield* execute({
10171039
operation,
10181040
cwd: input.cwd,
10191041
args: [...indexConfig, "sparse-checkout", "check-rules", "-z"],
1020-
stdin: skippedPaths.join("\0") + "\0",
1042+
stdin,
1043+
env: commitEnv,
1044+
maxOutputBytes: Buffer.byteLength(stdin),
1045+
outputMode: "error",
1046+
});
1047+
const selected = new Set(rules.stdout.split("\0"));
1048+
skipped.forEach((entry, index) => {
1049+
if (selected.has(paths[index]!)) flagged.push(entry);
1050+
});
1051+
}
1052+
if (flagged.length > 0) {
1053+
// Flagged gitlinks keep the fresh-index behaviour rather than a guessed one.
1054+
if (flagged.some((entry) => entry.startsWith("160000 "))) return false;
1055+
yield* execute({
1056+
operation,
1057+
cwd: input.cwd,
1058+
// core.ignorestat would mark these assume-unchanged, hiding edits from add -A.
1059+
args: [
1060+
...indexConfig,
1061+
"-c",
1062+
"core.ignorestat=false",
1063+
"update-index",
1064+
"-z",
1065+
"--index-info",
1066+
],
1067+
stdin: flagged.join("\0") + "\0",
10211068
env: commitEnv,
1022-
maxOutputBytes: 1,
1023-
outputMode: "truncate",
10241069
});
1025-
specialFlags = selected.stdout.length > 0 || selected.stdoutTruncated;
1070+
yield* fileSystem.utimes(tempIndexPath, indexTime, indexTime);
10261071
}
1027-
// Sparse Git clears skip-worktree for present files. Manual flags still need a reset.
1028-
return !specialFlags;
1072+
return true;
10291073
}).pipe(Effect.orElseSucceed(() => false));
10301074
if (!reusedIndex) {
10311075
if (sparseCheckout) {

0 commit comments

Comments
 (0)