Skip to content

Commit 55ba1ba

Browse files
perf(server): batch checkpoint object flushes on macOS and Windows (pingdotgg#21291)
Co-authored-by: Julius Marminge <[email protected]>
1 parent 826ef55 commit 55ba1ba

4 files changed

Lines changed: 108 additions & 65 deletions

File tree

‎apps/server/src/vcs/GitVcsDriver.test.ts‎

Lines changed: 73 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -1367,69 +1367,83 @@ it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")(
13671367
}).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
13681368
);
13691369

1370-
it.effect("GitVcsDriver flushes checkpoint objects and refs to disk before publishing them", () => {
1371-
const observedArgs: ReadonlyArray<string>[] = [];
1372-
1373-
return Effect.gen(function* () {
1374-
const driver = yield* GitVcsDriver.makeVcsDriverShape();
1370+
it.effect.each([
1371+
{ platform: "linux", fsyncMethods: ["fsync"] },
1372+
{ platform: "darwin", fsyncMethods: ["fsync", "batch"] },
1373+
{ platform: "win32", fsyncMethods: ["fsync", "batch"] },
1374+
] as const)(
1375+
"GitVcsDriver flushes checkpoint objects and refs to disk before publishing them on $platform",
1376+
({ platform, fsyncMethods }) => {
1377+
const observedArgs: ReadonlyArray<string>[] = [];
1378+
1379+
return Effect.gen(function* () {
1380+
const driver = yield* GitVcsDriver.makeVcsDriverShape();
13751381

1376-
yield* driver.checkpoints.captureCheckpoint({
1377-
cwd: "/repo",
1378-
checkpointRef: CheckpointRef.make("refs/t3/checkpoints/thread/turn/1"),
1379-
});
1382+
yield* driver.checkpoints.captureCheckpoint({
1383+
cwd: "/repo",
1384+
checkpointRef: CheckpointRef.make("refs/t3/checkpoints/thread/turn/1"),
1385+
});
13801386

1381-
const writeCommands = ["add", "write-tree", "commit-tree", "update-ref"];
1382-
const writes = observedArgs.filter((args) =>
1383-
writeCommands.some((command) => args.includes(command)),
1384-
);
1385-
assert.strictEqual(writes.length, 4);
1386-
for (const args of writes) {
1387-
const command = args.findIndex((arg) => writeCommands.includes(arg));
1388-
for (const setting of ["core.fsync=objects,reference", "core.fsyncMethod=fsync"]) {
1389-
const index = args.indexOf(setting);
1390-
assert.strictEqual(args[index - 1], "-c", args.join(" "));
1391-
assert.isBelow(index, command);
1387+
const writeCommands = ["add", "write-tree", "commit-tree", "update-ref"];
1388+
const writes = observedArgs.filter((args) =>
1389+
writeCommands.some((command) => args.includes(command)),
1390+
);
1391+
assert.strictEqual(writes.length, 4);
1392+
for (const args of writes) {
1393+
const command = args.findIndex((arg) => writeCommands.includes(arg));
1394+
assert.deepStrictEqual(
1395+
args.filter((arg) => arg.startsWith("core.fsyncMethod=")),
1396+
fsyncMethods.map((method) => `core.fsyncMethod=${method}`),
1397+
);
1398+
for (const setting of [
1399+
"core.fsync=objects,reference",
1400+
...fsyncMethods.map((method) => `core.fsyncMethod=${method}`),
1401+
]) {
1402+
const index = args.indexOf(setting);
1403+
assert.strictEqual(args[index - 1], "-c", args.join(" "));
1404+
assert.isBelow(index, command);
1405+
}
13921406
}
1393-
}
1394-
assert.deepStrictEqual(observedArgs.at(-1), [
1395-
"-C",
1396-
"/repo",
1397-
"-c",
1398-
"core.fsync=objects,reference",
1399-
"-c",
1400-
"core.fsyncMethod=fsync",
1401-
"update-ref",
1402-
"refs/t3/checkpoints/thread/turn/1",
1403-
"commit0000",
1404-
]);
1405-
}).pipe(
1406-
Effect.provide(
1407-
Layer.mergeAll(
1408-
NodeServices.layer,
1409-
Layer.mock(VcsProcess.VcsProcess)({
1410-
run: (input) =>
1411-
Effect.sync(() => {
1412-
observedArgs.push(input.args);
1413-
const stdout = input.args.includes("write-tree")
1414-
? "tree0000\n"
1415-
: input.args.includes("commit-tree")
1416-
? "commit0000\n"
1417-
: input.args.includes("--git-common-dir")
1418-
? ".git\n"
1419-
: "";
1420-
return {
1421-
exitCode: ChildProcessSpawner.ExitCode(0),
1422-
stdout,
1423-
stderr: "",
1424-
stdoutTruncated: false,
1425-
stderrTruncated: false,
1426-
};
1427-
}),
1428-
}),
1407+
assert.deepStrictEqual(observedArgs.at(-1), [
1408+
"-C",
1409+
"/repo",
1410+
"-c",
1411+
"core.fsync=objects,reference",
1412+
...fsyncMethods.flatMap((method) => ["-c", `core.fsyncMethod=${method}`]),
1413+
"update-ref",
1414+
"refs/t3/checkpoints/thread/turn/1",
1415+
"commit0000",
1416+
]);
1417+
}).pipe(
1418+
Effect.provideService(HostProcess.Platform, platform),
1419+
Effect.provide(
1420+
Layer.mergeAll(
1421+
NodeServices.layer,
1422+
Layer.mock(VcsProcess.VcsProcess)({
1423+
run: (input) =>
1424+
Effect.sync(() => {
1425+
observedArgs.push(input.args);
1426+
const stdout = input.args.includes("write-tree")
1427+
? "tree0000\n"
1428+
: input.args.includes("commit-tree")
1429+
? "commit0000\n"
1430+
: input.args.includes("--git-common-dir")
1431+
? ".git\n"
1432+
: "";
1433+
return {
1434+
exitCode: ChildProcessSpawner.ExitCode(0),
1435+
stdout,
1436+
stderr: "",
1437+
stdoutTruncated: false,
1438+
stderrTruncated: false,
1439+
};
1440+
}),
1441+
}),
1442+
),
14291443
),
1430-
),
1431-
);
1432-
});
1444+
);
1445+
},
1446+
);
14331447

14341448
it.effect.each(["win32", "linux"] as const)(
14351449
"checkpoint working-tree commands enable Git long paths only on Windows (%s)",

‎apps/server/src/vcs/GitVcsDriver.ts‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -542,6 +542,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
542542
const path = yield* Path.Path;
543543
const vcsProcess = yield* VcsProcess.VcsProcess;
544544
const crypto = yield* Crypto.Crypto;
545+
const hostPlatform = yield* HostProcess.Platform;
545546
const capabilities = {
546547
kind: "git" as const,
547548
supportsWorktrees: true,
@@ -807,17 +808,23 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
807808
// an unclean restart can leave 0-byte files under refs/t3/** that break every
808809
// later fetch and push. Checkpoint writes flush before they are published;
809810
// macOS defaults to writeout-only, which does not reach the disk either.
811+
// Set fsync first: Git 2.36 ignores unknown batch without resetting the method.
812+
// Git documents batch/fsync equivalence on macOS HFS+/APFS and Windows NTFS/ReFS;
813+
// keep per-object fsync on other platforms. Batch still fully flushes refs.
810814
// On macOS, VcsProcess retries unsupported full flushes with plain fsync.
811815
const durableWrite = [
812816
"-c",
813817
"core.fsync=objects,reference",
814818
"-c",
815819
"core.fsyncMethod=fsync",
820+
...(hostPlatform === "darwin" || hostPlatform === "win32"
821+
? ["-c", "core.fsyncMethod=batch"]
822+
: []),
816823
] as const;
817824

818825
// Checkpoints must handle existing paths beyond Windows MAX_PATH, even when
819826
// core.longpaths is disabled in the user's Git configuration.
820-
const longPaths = (yield* HostProcess.Platform) === "win32" ? ["-c", "core.longpaths=true"] : [];
827+
const longPaths = hostPlatform === "win32" ? ["-c", "core.longpaths=true"] : [];
821828

822829
const checkpoints: VcsDriver.VcsCheckpointOps = {
823830
captureCheckpoint: Effect.fn("GitVcsDriver.checkpoints.captureCheckpoint")(function* (input) {

‎apps/server/src/vcs/VcsProcess.test.ts‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -184,8 +184,17 @@ describe("VcsProcess.run", () => {
184184
stderr: "fatal: fsync error on '.git/objects/a5/tmp_obj_Pseg66': Operation not supported",
185185
fsyncMethods: ["fsync"],
186186
},
187+
{
188+
stderr: "fatal: fsync error on '.git/objects/bulk_fsync_Pseg66': Operation not supported",
189+
fsyncMethods: ["batch", "writeout-only"],
190+
},
191+
{
192+
stderr:
193+
"fatal: update_ref failed for ref 'refs/t3/checkpoints/x': cannot update ref 'refs/t3/checkpoints/x': couldn't write '/Volumes/share/.git/refs/t3/checkpoints/x.lock'",
194+
fsyncMethods: ["batch", "writeout-only"],
195+
},
187196
] as const)(
188-
"falls back to writeout-only only for unsupported fsync on macOS: $stderr",
197+
"falls back to writeout-only only for unsupported flushes on macOS ($fsyncMethods): $stderr",
189198
({ stderr, fsyncMethods, ...scenario }) =>
190199
Effect.gen(function* () {
191200
const seen: Array<ReadonlyArray<string>> = [];
@@ -199,7 +208,7 @@ describe("VcsProcess.run", () => {
199208
Effect.sync(() => {
200209
seen.push(input.args);
201210
const failed =
202-
input.args.includes("core.fsyncMethod=fsync") ||
211+
input.args.includes(`core.fsyncMethod=${fsyncMethods[0]}`) ||
203212
("fallbackFails" in scenario && scenario.fallbackFails);
204213
return {
205214
stdout: "",
@@ -221,14 +230,24 @@ describe("VcsProcess.run", () => {
221230
"operation" in scenario
222231
? scenario.operation
223232
: VcsProcess.CHECKPOINT_CAPTURE_OPERATION,
224-
args: ["-c", "core.fsync=objects,reference", "-c", "core.fsyncMethod=fsync", "add"],
233+
args: [
234+
"-c",
235+
"core.fsync=objects,reference",
236+
"-c",
237+
...(fsyncMethods[0] === "batch" ? ["core.fsyncMethod=fsync", "-c"] : []),
238+
`core.fsyncMethod=${fsyncMethods[0]}`,
239+
"add",
240+
],
225241
})
226242
.pipe(Effect.exit);
227243
expect(seen).toEqual(
228244
fsyncMethods.map((method) => [
229245
"-c",
230246
"core.fsync=objects,reference",
231247
"-c",
248+
...(fsyncMethods[0] === "batch"
249+
? [`core.fsyncMethod=${method === "batch" ? "fsync" : method}`, "-c"]
250+
: []),
232251
`core.fsyncMethod=${method}`,
233252
"add",
234253
]),

‎apps/server/src/vcs/VcsProcess.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,7 @@ const isTransientGitExit = (stderr: string) =>
172172
// bare write error too; explicit unrelated errors remain failures.
173173
// Linux writeout-only only starts writeback, so it must not use this fallback.
174174
const FULL_FSYNC_ARG = "core.fsyncMethod=fsync";
175+
const BATCH_FSYNC_ARG = "core.fsyncMethod=batch";
175176
const WRITEOUT_ONLY_FSYNC_ARG = "core.fsyncMethod=writeout-only";
176177
const canRetryFullFsync = (stderr: string) =>
177178
/fsync error on [^\r\n]+: operation not supported\r?$/im.test(stderr) ||
@@ -249,13 +250,15 @@ export const make = Effect.gen(function* () {
249250
result.code !== null &&
250251
result.code !== 0 &&
251252
input.command === "git" &&
252-
input.args.includes(FULL_FSYNC_ARG) &&
253+
(input.args.includes(FULL_FSYNC_ARG) || input.args.includes(BATCH_FSYNC_ARG)) &&
253254
canRetryFullFsync(result.stderr)
254255
) {
255256
result = yield* spawn(
256257
{
257258
...input,
258-
args: input.args.map((arg) => (arg === FULL_FSYNC_ARG ? WRITEOUT_ONLY_FSYNC_ARG : arg)),
259+
args: input.args.map((arg) =>
260+
arg === FULL_FSYNC_ARG || arg === BATCH_FSYNC_ARG ? WRITEOUT_ONLY_FSYNC_ARG : arg,
261+
),
259262
},
260263
baseError,
261264
);

0 commit comments

Comments
 (0)