Skip to content

Commit f3a69c7

Browse files
t3dotggdylan1020claude
authored
fix(checkpoint): pulls and rebases no longer flood a turn's changed files (pingdotgg#17161)
Co-authored-by: Dylan Zahn <[email protected]> Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent e858ddb commit f3a69c7

12 files changed

Lines changed: 390 additions & 37 deletions

‎apps/server/src/checkpointing/CheckpointDiffQuery.test.ts‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ function makeProjection(): ProjectionCheckpointContext {
4848
function layerFor(input: {
4949
readonly projection: Effect.Effect<ProjectionCheckpointContext, OrchestratorProjectionError>;
5050
readonly diffCheckpoints?: CheckpointStore.CheckpointStore["Service"]["diffCheckpoints"];
51+
readonly authoredPaths?: ReadonlySet<string>;
5152
}) {
5253
return CheckpointDiffQuery.layer.pipe(
5354
Layer.provide(
@@ -57,6 +58,7 @@ function layerFor(input: {
5758
}),
5859
Layer.mock(CheckpointStore.CheckpointStore)({
5960
diffCheckpoints: input.diffCheckpoints ?? (() => Effect.succeed("diff")),
61+
listAuthoredPaths: () => Effect.succeed(input.authoredPaths ?? null),
6062
}),
6163
),
6264
),
@@ -93,6 +95,34 @@ it.effect("computes V2 run diffs from projected checkpoint scopes", () => {
9395
}).pipe(Effect.provide(layer));
9496
});
9597

98+
it.effect("leaves Git imports out of large turn diffs", () => {
99+
const ownPaths = Array.from({ length: 1_000 }, (_, index) => `src/module-${index}/file.ts`);
100+
const diffCheckpoints = vi.fn((input: CheckpointStore.DiffCheckpointsInput) =>
101+
Effect.succeed(
102+
input.format === "numstat"
103+
? [...ownPaths, "upstream.ts"].map((path) => `1\t0\t${path}\0`).join("")
104+
: (input.filePaths ?? ["all"]).join("\n"),
105+
),
106+
);
107+
const layer = layerFor({
108+
projection: Effect.succeed(makeProjection()),
109+
diffCheckpoints,
110+
authoredPaths: new Set(ownPaths),
111+
});
112+
113+
return Effect.gen(function* () {
114+
const query = yield* CheckpointDiffQuery.CheckpointDiffQuery;
115+
yield* query.getFullThreadDiff({ threadId, toTurnCount: 2 });
116+
117+
const patchPaths = diffCheckpoints.mock.calls
118+
.map(([call]) => call)
119+
.filter((call) => call.format !== "numstat")
120+
.map((call) => call.filePaths ?? ["all"]);
121+
assert.isAbove(patchPaths.length, 1);
122+
assert.sameMembers(patchPaths.flat(), ownPaths);
123+
}).pipe(Effect.provide(layer));
124+
});
125+
96126
it.effect("preserves the typed missing-thread error contract", () => {
97127
const layer = layerFor({
98128
projection: Effect.fail(new OrchestratorProjectionError({ threadId })),

‎apps/server/src/checkpointing/CheckpointDiffQuery.ts‎

Lines changed: 59 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,31 @@ import {
3131
type CheckpointServiceError,
3232
} from "./Errors.ts";
3333
import * as CheckpointStore from "./CheckpointStore.ts";
34+
import { isGitImport, parseTurnDiffFilesFromNumstat, type TurnDiffFileSummary } from "./Diffs.ts";
35+
36+
// Windows limits a command line to 32,767 characters. Leave room for the rest of git's arguments.
37+
const MAX_PATHSPEC_CHARS = 24_000;
38+
39+
/** Splits files into pathspec lists that each fit on one git command line. Renames keep both paths together. */
40+
function batchFilePaths(files: ReadonlyArray<TurnDiffFileSummary>) {
41+
const batches: string[][] = [];
42+
let batch: string[] = [];
43+
let length = 0;
44+
for (const file of files) {
45+
const paths = file.previousPath === undefined ? [file.path] : [file.previousPath, file.path];
46+
// Each pathspec also carries a `:(top,literal)` prefix and a separator.
47+
const size = paths.reduce((total, path) => total + path.length + 16, 0);
48+
if (batch.length > 0 && length + size > MAX_PATHSPEC_CHARS) {
49+
batches.push(batch);
50+
batch = [];
51+
length = 0;
52+
}
53+
batch.push(...paths);
54+
length += size;
55+
}
56+
if (batch.length > 0) batches.push(batch);
57+
return batches;
58+
}
3459

3560
/** Service tag for checkpoint diff queries. */
3661
export class CheckpointDiffQuery extends Context.Service<
@@ -182,15 +207,40 @@ export const make = Effect.gen(function* () {
182207
});
183208
}
184209

185-
const diff = yield* checkpointStore
186-
.diffCheckpoints({
187-
cwd: toScope.cwd,
188-
fromCheckpointRef,
189-
toCheckpointRef: toCheckpoint.ref,
190-
fallbackFromToHead: false,
191-
ignoreWhitespace,
192-
})
193-
.pipe(Effect.withSpan("checkpoint.turnDiff.diffCheckpoints"));
210+
const comparison = {
211+
cwd: toScope.cwd,
212+
fromCheckpointRef,
213+
toCheckpointRef: toCheckpoint.ref,
214+
fallbackFromToHead: false,
215+
ignoreWhitespace,
216+
};
217+
// Leave out files that a pull, merge, or rebase brought in, matching the turn's file summary.
218+
// Filtering is optional: when it cannot run, the complete diff still loads.
219+
const authoredPaths = yield* checkpointStore
220+
.listAuthoredPaths(comparison)
221+
.pipe(Effect.orElseSucceed(() => null));
222+
const files =
223+
authoredPaths === null
224+
? []
225+
: yield* checkpointStore.diffCheckpoints({ ...comparison, format: "numstat" }).pipe(
226+
Effect.map(parseTurnDiffFilesFromNumstat),
227+
Effect.orElseSucceed(() => []),
228+
);
229+
const retainedFiles = files.filter((file) => !isGitImport(file, authoredPaths));
230+
// Select retained paths before generating a patch, so imported bulk cannot exhaust its output limit.
231+
const diff =
232+
retainedFiles.length === files.length
233+
? yield* checkpointStore
234+
.diffCheckpoints(comparison)
235+
.pipe(Effect.withSpan("checkpoint.turnDiff.diffCheckpoints"))
236+
: (yield* Effect.forEach(
237+
batchFilePaths(retainedFiles),
238+
(filePaths) =>
239+
checkpointStore
240+
.diffCheckpoints({ ...comparison, filePaths })
241+
.pipe(Effect.withSpan("checkpoint.turnDiff.diffCheckpoints")),
242+
{ concurrency: 4 },
243+
)).join("");
194244

195245
const turnDiff = buildTurnDiffResult(input, diff);
196246
if (!isTurnDiffResult(turnDiff)) {

‎apps/server/src/checkpointing/CheckpointStore.test.ts‎

Lines changed: 128 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -57,15 +57,22 @@ function writeTextFile(
5757
function git(
5858
cwd: string,
5959
args: ReadonlyArray<string>,
60+
options: { readonly committedAt?: string; readonly allowNonZeroExit?: boolean } = {},
6061
): Effect.Effect<string, VcsError, VcsProcess.VcsProcess> {
6162
return Effect.gen(function* () {
62-
const process = yield* VcsProcess.VcsProcess;
63-
const result = yield* process.run({
63+
const vcsProcess = yield* VcsProcess.VcsProcess;
64+
const result = yield* vcsProcess.run({
6465
operation: "CheckpointStore.test.git",
6566
command: "git",
6667
cwd,
6768
args,
6869
timeoutMs: 10_000,
70+
...(options.committedAt === undefined
71+
? {}
72+
: { env: { ...process.env, GIT_COMMITTER_DATE: options.committedAt } }),
73+
...(options.allowNonZeroExit === undefined
74+
? {}
75+
: { allowNonZeroExit: options.allowNonZeroExit }),
6976
});
7077
return result.stdout.trim();
7178
});
@@ -371,12 +378,12 @@ it.layer(layerTest)("CheckpointStore.layer", (it) => {
371378
);
372379
const expectedFiles = [
373380
{ path: "binary.bin", additions: 0, deletions: 0 },
374-
{ path: "copied.txt", additions: 0, deletions: 0 },
381+
{ path: "copied.txt", previousPath: "copy-source.txt", additions: 0, deletions: 0 },
375382
{ path: "copy-source.txt", additions: 1, deletions: 0 },
376383
{ path: "deleted.txt", additions: 0, deletions: 1 },
377384
{ path: "empty.txt", additions: 0, deletions: 0 },
378385
{ path: addedPath, additions: 2, deletions: 0 },
379-
{ path: renamedPath, additions: 1, deletions: 1 },
386+
{ path: renamedPath, previousPath: "rename-old.txt", additions: 1, deletions: 1 },
380387
].toSorted((left, right) => left.path.localeCompare(right.path));
381388
expect(firstSummary).toEqual(expectedFiles);
382389

@@ -441,5 +448,122 @@ it.layer(layerTest)("CheckpointStore.layer", (it) => {
441448
]);
442449
}),
443450
);
451+
it.effect("limits a diff to repository paths from a subdirectory workspace", () =>
452+
Effect.gen(function* () {
453+
const tmp = yield* makeTmpDir();
454+
yield* initRepoWithCommit(tmp);
455+
const fileSystem = yield* FileSystem.FileSystem;
456+
const cwd = NodePath.join(tmp, "app");
457+
yield* fileSystem.makeDirectory(cwd);
458+
const checkpointStore = yield* CheckpointStore.CheckpointStore;
459+
const threadId = ThreadId.make("checkpoint-file-paths-subdirectory");
460+
const refs = {
461+
cwd,
462+
fromCheckpointRef: checkpointRefForThreadTurn(threadId, 0),
463+
toCheckpointRef: checkpointRefForThreadTurn(threadId, 1),
464+
};
465+
yield* checkpointStore.captureCheckpoint({ cwd, checkpointRef: refs.fromCheckpointRef });
466+
yield* writeTextFile(NodePath.join(cwd, "kept.txt"), "kept\n");
467+
yield* writeTextFile(NodePath.join(tmp, "README.md"), "# hidden\n");
468+
yield* checkpointStore.captureCheckpoint({ cwd, checkpointRef: refs.toCheckpointRef });
469+
470+
const patch = yield* checkpointStore.diffCheckpoints({
471+
...refs,
472+
ignoreWhitespace: false,
473+
filePaths: ["app/kept.txt"],
474+
});
475+
expect(patch).toContain("+kept");
476+
expect(patch).not.toContain("hidden");
477+
}),
478+
);
479+
});
480+
481+
describe("listAuthoredPaths", () => {
482+
it.effect("leaves out paths that only a merged upstream changed", () =>
483+
Effect.gen(function* () {
484+
const tmp = yield* makeTmpDir();
485+
yield* initRepoWithCommit(tmp);
486+
const checkpointStore = yield* CheckpointStore.CheckpointStore;
487+
const threadId = ThreadId.make("checkpoint-authored-paths");
488+
const refs = {
489+
cwd: tmp,
490+
fromCheckpointRef: checkpointRefForThreadTurn(threadId, 0),
491+
toCheckpointRef: checkpointRefForThreadTurn(threadId, 1),
492+
};
493+
const commitBeforeTurn = (file: string, contents: string) =>
494+
Effect.gen(function* () {
495+
yield* writeTextFile(NodePath.join(tmp, file), contents);
496+
yield* git(tmp, ["add", "."]);
497+
yield* git(tmp, ["commit", "-m", file], { committedAt: "2020-01-01T00:00:00Z" });
498+
});
499+
500+
yield* git(tmp, ["checkout", "-b", "upstream"]);
501+
yield* commitBeforeTurn("upstream.txt", "upstream\n");
502+
yield* commitBeforeTurn("shared.txt", "upstream\n");
503+
yield* git(tmp, ["checkout", "-"]);
504+
yield* commitBeforeTurn("shared.txt", "ours\n");
505+
yield* checkpointStore.captureCheckpoint({
506+
cwd: tmp,
507+
checkpointRef: refs.fromCheckpointRef,
508+
});
509+
510+
yield* checkpointStore.captureCheckpoint({ cwd: tmp, checkpointRef: refs.toCheckpointRef });
511+
expect(yield* checkpointStore.listAuthoredPaths(refs)).toBeNull();
512+
513+
// The turn merges upstream, fixes the conflict by taking upstream, and edits one more file.
514+
yield* git(tmp, ["merge", "upstream", "--no-edit"], { allowNonZeroExit: true });
515+
yield* writeTextFile(NodePath.join(tmp, "shared.txt"), "upstream\n");
516+
yield* git(tmp, ["commit", "-am", "merge upstream"]);
517+
yield* writeTextFile(NodePath.join(tmp, "README.md"), "# edited\n");
518+
yield* checkpointStore.captureCheckpoint({ cwd: tmp, checkpointRef: refs.toCheckpointRef });
519+
520+
const numstat = yield* checkpointStore.diffCheckpoints({
521+
...refs,
522+
ignoreWhitespace: false,
523+
format: "numstat",
524+
});
525+
expect(parseTurnDiffFilesFromNumstat(numstat).map((file) => file.path)).toEqual([
526+
"README.md",
527+
"shared.txt",
528+
"upstream.txt",
529+
]);
530+
expect(yield* checkpointStore.listAuthoredPaths(refs)).toEqual(
531+
new Set(["README.md", "shared.txt"]),
532+
);
533+
}),
534+
);
535+
536+
it.effect("leaves out paths that only a rebase onto upstream brought in", () =>
537+
Effect.gen(function* () {
538+
const tmp = yield* makeTmpDir();
539+
yield* initRepoWithCommit(tmp);
540+
const checkpointStore = yield* CheckpointStore.CheckpointStore;
541+
const threadId = ThreadId.make("checkpoint-authored-paths-rebase");
542+
const refs = {
543+
cwd: tmp,
544+
fromCheckpointRef: checkpointRefForThreadTurn(threadId, 0),
545+
toCheckpointRef: checkpointRefForThreadTurn(threadId, 1),
546+
};
547+
const commitBeforeTurn = (file: string) =>
548+
Effect.gen(function* () {
549+
yield* writeTextFile(NodePath.join(tmp, file), `${file}\n`);
550+
yield* git(tmp, ["add", "."]);
551+
yield* git(tmp, ["commit", "-m", file], { committedAt: "2020-01-01T00:00:00Z" });
552+
});
553+
554+
yield* git(tmp, ["checkout", "-b", "upstream"]);
555+
yield* commitBeforeTurn("upstream.txt");
556+
yield* git(tmp, ["checkout", "-"]);
557+
yield* commitBeforeTurn("feature.txt");
558+
yield* checkpointStore.captureCheckpoint({
559+
cwd: tmp,
560+
checkpointRef: refs.fromCheckpointRef,
561+
});
562+
yield* git(tmp, ["rebase", "upstream"]);
563+
yield* checkpointStore.captureCheckpoint({ cwd: tmp, checkpointRef: refs.toCheckpointRef });
564+
565+
expect(yield* checkpointStore.listAuthoredPaths(refs)).toEqual(new Set(["feature.txt"]));
566+
}),
567+
);
444568
});
445569
});

‎apps/server/src/checkpointing/CheckpointStore.ts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,14 @@ export interface DiffCheckpointsInput {
4040
readonly fallbackFromToHead?: boolean;
4141
readonly ignoreWhitespace: boolean;
4242
readonly format?: "patch" | "numstat";
43+
/** Limits the diff to these exact paths. An empty list yields an empty diff. */
44+
readonly filePaths?: ReadonlyArray<string>;
45+
}
46+
47+
export interface ListAuthoredPathsInput {
48+
readonly cwd: string;
49+
readonly fromCheckpointRef: CheckpointRef;
50+
readonly toCheckpointRef: CheckpointRef;
4351
}
4452

4553
export interface DeleteCheckpointRefsInput {
@@ -87,6 +95,19 @@ export class CheckpointStore extends Context.Service<
8795
input: DiffCheckpointsInput,
8896
) => Effect.Effect<string, CheckpointStoreError>;
8997

98+
/**
99+
* List paths changed by work done after the "from" checkpoint: uncommitted
100+
* edits at either checkpoint, commits made after "from", and commits that
101+
* left HEAD. Commits a pull, merge, or rebase brought in are older than
102+
* "from", so their paths are not listed.
103+
*
104+
* Returns null when HEAD did not move or a checkpoint does not record HEAD.
105+
* Then every changed path belongs to the turn.
106+
*/
107+
readonly listAuthoredPaths: (
108+
input: ListAuthoredPathsInput,
109+
) => Effect.Effect<ReadonlySet<string> | null, CheckpointStoreError>;
110+
90111
/**
91112
* Delete the provided checkpoint refs.
92113
*
@@ -150,6 +171,13 @@ export const make = Effect.gen(function* () {
150171
return yield* checkpoints.diffCheckpoints(input);
151172
});
152173

174+
const listAuthoredPaths: CheckpointStore["Service"]["listAuthoredPaths"] = Effect.fn(
175+
"listAuthoredPaths",
176+
)(function* (input) {
177+
const checkpoints = yield* resolveCheckpoints("CheckpointStore.listAuthoredPaths", input.cwd);
178+
return yield* checkpoints.listAuthoredPaths(input);
179+
});
180+
153181
const deleteCheckpointRefs: CheckpointStore["Service"]["deleteCheckpointRefs"] = Effect.fn(
154182
"deleteCheckpointRefs",
155183
)(function* (input) {
@@ -166,6 +194,7 @@ export const make = Effect.gen(function* () {
166194
hasCheckpointRef,
167195
restoreCheckpoint,
168196
diffCheckpoints,
197+
listAuthoredPaths,
169198
deleteCheckpointRefs,
170199
});
171200
});

‎apps/server/src/checkpointing/Diffs.test.ts‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ describe("parseTurnDiffFilesFromNumstat", () => {
1515
]);
1616
});
1717

18-
it("uses destination paths for renames and copies", () => {
18+
it("preserves both paths for renames and copies", () => {
1919
const numstat = [
2020
"0\t0\t",
2121
"src/old.ts",
@@ -29,8 +29,8 @@ describe("parseTurnDiffFilesFromNumstat", () => {
2929

3030
expect(parseTurnDiffFilesFromNumstat(numstat)).toEqual([
3131
{ path: "other.ts", additions: 1, deletions: 0 },
32-
{ path: "src/copied.ts", additions: 2, deletions: 1 },
33-
{ path: "src/new.ts", additions: 0, deletions: 0 },
32+
{ path: "src/copied.ts", previousPath: "src/source.ts", additions: 2, deletions: 1 },
33+
{ path: "src/new.ts", previousPath: "src/old.ts", additions: 0, deletions: 0 },
3434
]);
3535
});
3636

@@ -46,7 +46,9 @@ describe("parseTurnDiffFilesFromNumstat", () => {
4646
const path = " café\tline\r\nname.txt ";
4747
const numstat = `3\t2\t\0old\tname\n.txt\0${path}\0`;
4848

49-
expect(parseTurnDiffFilesFromNumstat(numstat)).toEqual([{ path, additions: 3, deletions: 2 }]);
49+
expect(parseTurnDiffFilesFromNumstat(numstat)).toEqual([
50+
{ path, previousPath: "old\tname\n.txt", additions: 3, deletions: 2 },
51+
]);
5052
expect(parseTurnDiffFilesFromNumstat(`1\t0\t${path}\0`)).toEqual([
5153
{ path, additions: 1, deletions: 0 },
5254
]);

0 commit comments

Comments
 (0)