Skip to content

Commit e340aae

Browse files
fix(server): meter checkpoint git subprocess attempts (pingdotgg#24218)
Signed-off-by: Yordis Prieto <[email protected]> Co-authored-by: Yordis Prieto <[email protected]>
1 parent 5bb54a0 commit e340aae

3 files changed

Lines changed: 307 additions & 35 deletions

File tree

‎apps/server/src/vcs/GitVcsDriver.test.ts‎

Lines changed: 224 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,15 @@ import * as Layer from "effect/Layer";
1212
import * as Option from "effect/Option";
1313
import * as Path from "effect/Path";
1414
import * as PlatformError from "effect/PlatformError";
15+
import { ChildProcessSpawner } from "effect/process";
16+
import * as Metric from "effect/Metric";
1517
import { assert, it } from "@effect/vitest";
1618

1719
import { CheckpointRef, GitCommandError, VcsProcessExitError } from "@t3tools/contracts";
1820
import * as HostProcess from "@t3tools/shared/HostProcess";
1921
import { symlinksSupported } from "@t3tools/shared/testing/symlinks";
2022
import * as ServerConfig from "../config.ts";
23+
import { gitCommandDuration, gitCommandsTotal } from "../observability/Metrics.ts";
2124
import * as CheckpointStore from "../checkpointing/CheckpointStore.ts";
2225
import * as ProcessRunner from "../processRunner.ts";
2326
import * as GitVcsDriver from "./GitVcsDriver.ts";
@@ -1017,3 +1020,224 @@ it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")(
10171020
assert.strictEqual(detectionError._tag, "VcsProcessSpawnError");
10181021
}).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
10191022
);
1023+
1024+
const gitCommandCount = (operation: string, outcome: string) =>
1025+
Metric.value(
1026+
Metric.withAttributes(gitCommandsTotal, [
1027+
["operation", operation],
1028+
["outcome", outcome],
1029+
]),
1030+
).pipe(Effect.map((state) => state.count));
1031+
1032+
const gitCommandTiming = (operation: string) =>
1033+
Metric.value(Metric.withAttributes(gitCommandDuration, [["operation", operation]]));
1034+
1035+
// Keep VcsProcess's exit classification; replace only the subprocess and its clock.
1036+
const makeMeteredDriver = Effect.fn("test.makeMeteredDriver")(function* (exitCode: number) {
1037+
const process = yield* VcsProcess.make.pipe(
1038+
Effect.provideService(ProcessRunner.ProcessRunner, {
1039+
run: () =>
1040+
TestClock.adjust("250 millis").pipe(
1041+
Effect.as({
1042+
code: ChildProcessSpawner.ExitCode(exitCode),
1043+
stdout: "git output",
1044+
stderr: exitCode === 0 ? "" : "fatal: test failure",
1045+
timedOut: false,
1046+
stdoutTruncated: false,
1047+
stderrTruncated: false,
1048+
stdoutInvalidUtf8: false,
1049+
stderrInvalidUtf8: false,
1050+
}),
1051+
),
1052+
}),
1053+
);
1054+
return yield* GitVcsDriver.makeVcsDriverShape().pipe(
1055+
Effect.provideService(VcsProcess.VcsProcess, process),
1056+
);
1057+
});
1058+
1059+
it.effect("GitVcsDriver meters each git command once under its operation", () =>
1060+
Effect.gen(function* () {
1061+
const operation = "GitVcsDriver.checkpoints.resolveHeadCommit";
1062+
const driver = yield* makeMeteredDriver(0);
1063+
1064+
const result = yield* driver.execute({ operation, cwd: "/repo", args: ["rev-parse", "HEAD"] });
1065+
yield* driver.execute({ operation, cwd: "/another-repo", args: ["rev-parse", "other-ref"] });
1066+
1067+
assert.strictEqual(result.stdout, "git output");
1068+
assert.strictEqual(yield* gitCommandCount(operation, "success"), 2);
1069+
assert.strictEqual(yield* gitCommandCount(operation, "failure"), 0);
1070+
const timing = yield* gitCommandTiming(operation);
1071+
assert.strictEqual(timing.count, 2);
1072+
assert.strictEqual(timing.sum, 500);
1073+
}).pipe(
1074+
Effect.provide(NodeServices.layer),
1075+
Effect.provideService(Metric.MetricRegistry, new Map()),
1076+
),
1077+
);
1078+
1079+
it.effect("GitVcsDriver counts an allowed non-zero exit as a success", () =>
1080+
Effect.gen(function* () {
1081+
const operation = "GitVcsDriver.checkpoints.hasHeadCommit";
1082+
const driver = yield* makeMeteredDriver(1);
1083+
1084+
const result = yield* driver.execute({
1085+
operation,
1086+
cwd: "/repo",
1087+
args: ["rev-parse", "--verify", "HEAD"],
1088+
allowNonZeroExit: true,
1089+
});
1090+
1091+
assert.strictEqual(result.exitCode, 1);
1092+
assert.strictEqual(yield* gitCommandCount(operation, "success"), 1);
1093+
assert.strictEqual(yield* gitCommandCount(operation, "failure"), 0);
1094+
const timing = yield* gitCommandTiming(operation);
1095+
assert.strictEqual(timing.count, 1);
1096+
assert.strictEqual(timing.sum, 250);
1097+
}).pipe(
1098+
Effect.provide(NodeServices.layer),
1099+
Effect.provideService(Metric.MetricRegistry, new Map()),
1100+
),
1101+
);
1102+
1103+
it.effect("GitVcsDriver counts a failed git command as a failure", () =>
1104+
Effect.gen(function* () {
1105+
const operation = "GitVcsDriver.checkpoints.resolveHeadCommit";
1106+
const driver = yield* makeMeteredDriver(128);
1107+
1108+
const error = yield* driver
1109+
.execute({ operation, cwd: "/repo", args: ["rev-parse", "HEAD"] })
1110+
.pipe(Effect.flip);
1111+
1112+
assert.strictEqual(error._tag, "VcsProcessExitError");
1113+
assert.strictEqual(yield* gitCommandCount(operation, "failure"), 1);
1114+
assert.strictEqual(yield* gitCommandCount(operation, "success"), 0);
1115+
const timing = yield* gitCommandTiming(operation);
1116+
assert.strictEqual(timing.count, 1);
1117+
assert.strictEqual(timing.sum, 250);
1118+
}).pipe(
1119+
Effect.provide(NodeServices.layer),
1120+
Effect.provideService(Metric.MetricRegistry, new Map()),
1121+
),
1122+
);
1123+
1124+
it.effect("checkpoint capture meters every git process it starts, including streamed ones", () =>
1125+
Effect.gen(function* () {
1126+
const fileSystem = yield* FileSystem.FileSystem;
1127+
const liveProcess = yield* VcsProcess.VcsProcess;
1128+
const operation = VcsProcess.CHECKPOINT_CAPTURE_OPERATION;
1129+
let started = 0;
1130+
let streamed = 0;
1131+
const driver = yield* GitVcsDriver.makeVcsDriverShape();
1132+
const captureDriver = yield* GitVcsDriver.makeVcsDriverShape().pipe(
1133+
Effect.provideService(VcsProcess.VcsProcess, {
1134+
run: (input) => {
1135+
if (input.operation === operation) {
1136+
started++;
1137+
if (input.onStdoutChunk !== undefined) streamed++;
1138+
}
1139+
return liveProcess.run(input);
1140+
},
1141+
}),
1142+
);
1143+
const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-checkpoint-metrics-" });
1144+
const { checkpointRef } = yield* makeCheckpointFixture(driver, cwd);
1145+
1146+
yield* captureDriver.checkpoints.captureCheckpoint({ cwd, checkpointRef });
1147+
1148+
assert.isAbove(streamed, 0);
1149+
assert.strictEqual(yield* gitCommandCount(operation, "success"), started);
1150+
assert.strictEqual(yield* gitCommandCount(operation, "failure"), 0);
1151+
assert.strictEqual((yield* gitCommandTiming(operation)).count, started);
1152+
}).pipe(
1153+
Effect.scoped,
1154+
Effect.provide(layerGitContract),
1155+
Effect.provideService(Metric.MetricRegistry, new Map()),
1156+
),
1157+
);
1158+
1159+
it.effect("git metrics finalize cancellation without exposing arbitrary operation labels", () =>
1160+
Effect.gen(function* () {
1161+
const started = yield* Deferred.make<void>();
1162+
const process = yield* VcsProcess.make.pipe(
1163+
Effect.provideService(ProcessRunner.ProcessRunner, {
1164+
run: () => Deferred.succeed(started, undefined).pipe(Effect.andThen(Effect.never)),
1165+
}),
1166+
);
1167+
const driver = yield* GitVcsDriver.makeVcsDriverShape().pipe(
1168+
Effect.provideService(VcsProcess.VcsProcess, process),
1169+
);
1170+
const operation = "arbitrary operation /private/path";
1171+
const fiber = yield* driver
1172+
.execute({ operation, cwd: "/repo", args: ["rev-parse", "HEAD"] })
1173+
.pipe(Effect.forkChild);
1174+
yield* Deferred.await(started);
1175+
yield* TestClock.adjust("250 millis");
1176+
yield* Fiber.interrupt(fiber);
1177+
1178+
assert.strictEqual(yield* gitCommandCount("other", "interrupt"), 1);
1179+
assert.strictEqual(yield* gitCommandCount(operation, "interrupt"), 0);
1180+
const timing = yield* gitCommandTiming("other");
1181+
assert.strictEqual(timing.count, 1);
1182+
assert.strictEqual(timing.sum, 250);
1183+
}).pipe(
1184+
Effect.provide(NodeServices.layer),
1185+
Effect.provideService(Metric.MetricRegistry, new Map()),
1186+
),
1187+
);
1188+
1189+
it.effect("git metrics count fsync fallback and transient retry subprocesses separately", () =>
1190+
Effect.gen(function* () {
1191+
const retryReady = yield* Deferred.make<void>();
1192+
const operation = VcsProcess.CHECKPOINT_CAPTURE_OPERATION;
1193+
let attempts = 0;
1194+
const process = yield* VcsProcess.make.pipe(
1195+
Effect.provideService(HostProcess.Platform, "darwin"),
1196+
Effect.provideService(ProcessRunner.ProcessRunner, {
1197+
run: (input) =>
1198+
Effect.gen(function* () {
1199+
attempts++;
1200+
yield* TestClock.adjust("250 millis");
1201+
if (attempts === 2) {
1202+
assert.include(input.args, "core.fsyncMethod=writeout-only");
1203+
yield* Deferred.succeed(retryReady, undefined);
1204+
}
1205+
return {
1206+
code: ChildProcessSpawner.ExitCode(attempts < 3 ? 128 : 0),
1207+
stdout: "",
1208+
stderr:
1209+
attempts === 1
1210+
? "fatal: fsync error on '.git/objects/test': Operation not supported"
1211+
: attempts === 2
1212+
? "fatal: Unable to create '/repo/.git/index.lock': File exists."
1213+
: "",
1214+
timedOut: false,
1215+
stdoutTruncated: false,
1216+
stderrTruncated: false,
1217+
stdoutInvalidUtf8: false,
1218+
stderrInvalidUtf8: false,
1219+
};
1220+
}),
1221+
}),
1222+
);
1223+
const driver = yield* GitVcsDriver.makeVcsDriverShape().pipe(
1224+
Effect.provideService(VcsProcess.VcsProcess, process),
1225+
);
1226+
const fiber = yield* driver
1227+
.execute({ operation, cwd: "/repo", args: ["-c", "core.fsyncMethod=batch", "add"] })
1228+
.pipe(Effect.forkChild);
1229+
yield* Deferred.await(retryReady);
1230+
yield* TestClock.adjust("75 millis");
1231+
yield* Fiber.join(fiber);
1232+
1233+
assert.strictEqual(attempts, 3);
1234+
assert.strictEqual(yield* gitCommandCount(operation, "failure"), 2);
1235+
assert.strictEqual(yield* gitCommandCount(operation, "success"), 1);
1236+
const timing = yield* gitCommandTiming(operation);
1237+
assert.strictEqual(timing.count, 3);
1238+
assert.strictEqual(timing.sum, 750);
1239+
}).pipe(
1240+
Effect.provide(NodeServices.layer),
1241+
Effect.provideService(Metric.MetricRegistry, new Map()),
1242+
),
1243+
);

‎apps/server/src/vcs/GitVcsDriver.ts‎

Lines changed: 35 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -503,39 +503,41 @@ function parseGitRemoteVerboseOutput(
503503
return remotes;
504504
}
505505

506-
const gitCommand = (
507-
process: VcsProcess.VcsProcess["Service"],
508-
operation: string,
509-
cwd: string,
510-
args: ReadonlyArray<string>,
511-
options?: {
512-
readonly stdin?: string;
513-
readonly env?: NodeJS.ProcessEnv;
514-
readonly allowNonZeroExit?: boolean;
515-
readonly timeoutMs?: number;
516-
readonly maxOutputBytes?: number;
517-
readonly outputMode?: VcsProcess.VcsProcessInput["outputMode"];
518-
readonly appendTruncationMarker?: boolean;
519-
},
520-
) =>
521-
process.run({
522-
operation,
523-
command: "git",
524-
args: ["-C", cwd, ...args],
525-
cwd,
526-
spawnCwd: globalThis.process.cwd(),
527-
...(options?.stdin !== undefined ? { stdin: options.stdin } : {}),
528-
...(options?.env !== undefined ? { env: options.env } : {}),
529-
...(options?.allowNonZeroExit !== undefined
530-
? { allowNonZeroExit: options.allowNonZeroExit }
531-
: {}),
532-
...(options?.timeoutMs !== undefined ? { timeoutMs: options.timeoutMs } : {}),
533-
...(options?.maxOutputBytes !== undefined ? { maxOutputBytes: options.maxOutputBytes } : {}),
534-
...(options?.outputMode !== undefined ? { outputMode: options.outputMode } : {}),
535-
...(options?.appendTruncationMarker !== undefined
536-
? { appendTruncationMarker: options.appendTruncationMarker }
537-
: {}),
538-
});
506+
const gitCommand = Effect.fn("GitVcsDriver.gitCommand")(
507+
(
508+
process: VcsProcess.VcsProcess["Service"],
509+
operation: string,
510+
cwd: string,
511+
args: ReadonlyArray<string>,
512+
options?: {
513+
readonly stdin?: string;
514+
readonly env?: NodeJS.ProcessEnv;
515+
readonly allowNonZeroExit?: boolean;
516+
readonly timeoutMs?: number;
517+
readonly maxOutputBytes?: number;
518+
readonly outputMode?: VcsProcess.VcsProcessInput["outputMode"];
519+
readonly appendTruncationMarker?: boolean;
520+
},
521+
) =>
522+
process.run({
523+
operation,
524+
command: "git",
525+
args: ["-C", cwd, ...args],
526+
cwd,
527+
spawnCwd: globalThis.process.cwd(),
528+
...(options?.stdin !== undefined ? { stdin: options.stdin } : {}),
529+
...(options?.env !== undefined ? { env: options.env } : {}),
530+
...(options?.allowNonZeroExit !== undefined
531+
? { allowNonZeroExit: options.allowNonZeroExit }
532+
: {}),
533+
...(options?.timeoutMs !== undefined ? { timeoutMs: options.timeoutMs } : {}),
534+
...(options?.maxOutputBytes !== undefined ? { maxOutputBytes: options.maxOutputBytes } : {}),
535+
...(options?.outputMode !== undefined ? { outputMode: options.outputMode } : {}),
536+
...(options?.appendTruncationMarker !== undefined
537+
? { appendTruncationMarker: options.appendTruncationMarker }
538+
: {}),
539+
}),
540+
);
539541

540542
export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* () {
541543
const fileSystem = yield* FileSystem.FileSystem;

0 commit comments

Comments
 (0)