Skip to content

Commit ef8b584

Browse files
fix(server): meter each checkpoint git subprocess attempt
1 parent 20cf7e2 commit ef8b584

3 files changed

Lines changed: 169 additions & 48 deletions

File tree

‎apps/server/src/vcs/GitVcsDriver.test.ts‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1155,3 +1155,89 @@ it.effect("checkpoint capture meters every git process it starts, including stre
11551155
Effect.provideService(Metric.MetricRegistry, new Map()),
11561156
),
11571157
);
1158+
1159+
it.effect("git metrics finalize cancellation without exposing arbitrary operation labels", () =>
1160+
Effect.gen(function* () {
1161+
const started = yield* Deferred.make<void>();
1162+
const process = yield* VcsProcess.make.pipe(
1163+
Effect.provideService(ProcessRunner.ProcessRunner, {
1164+
run: () => Deferred.succeed(started, undefined).pipe(Effect.andThen(Effect.never)),
1165+
}),
1166+
);
1167+
const driver = yield* GitVcsDriver.makeVcsDriverShape().pipe(
1168+
Effect.provideService(VcsProcess.VcsProcess, process),
1169+
);
1170+
const operation = "arbitrary operation /private/path";
1171+
const fiber = yield* driver
1172+
.execute({ operation, cwd: "/repo", args: ["rev-parse", "HEAD"] })
1173+
.pipe(Effect.forkChild);
1174+
yield* Deferred.await(started);
1175+
yield* TestClock.adjust("250 millis");
1176+
yield* Fiber.interrupt(fiber);
1177+
1178+
assert.strictEqual(yield* gitCommandCount("other", "interrupt"), 1);
1179+
assert.strictEqual(yield* gitCommandCount(operation, "interrupt"), 0);
1180+
const timing = yield* gitCommandTiming("other");
1181+
assert.strictEqual(timing.count, 1);
1182+
assert.strictEqual(timing.sum, 250);
1183+
}).pipe(
1184+
Effect.provide(NodeServices.layer),
1185+
Effect.provideService(Metric.MetricRegistry, new Map()),
1186+
),
1187+
);
1188+
1189+
it.effect("git metrics count fsync fallback and transient retry subprocesses separately", () =>
1190+
Effect.gen(function* () {
1191+
const retryReady = yield* Deferred.make<void>();
1192+
const operation = VcsProcess.CHECKPOINT_CAPTURE_OPERATION;
1193+
let attempts = 0;
1194+
const process = yield* VcsProcess.make.pipe(
1195+
Effect.provideService(HostProcess.Platform, "darwin"),
1196+
Effect.provideService(ProcessRunner.ProcessRunner, {
1197+
run: (input) =>
1198+
Effect.gen(function* () {
1199+
attempts++;
1200+
yield* TestClock.adjust("250 millis");
1201+
if (attempts === 2) {
1202+
assert.include(input.args, "core.fsyncMethod=writeout-only");
1203+
yield* Deferred.succeed(retryReady, undefined);
1204+
}
1205+
return {
1206+
code: ChildProcessSpawner.ExitCode(attempts < 3 ? 128 : 0),
1207+
stdout: "",
1208+
stderr:
1209+
attempts === 1
1210+
? "fatal: fsync error on '.git/objects/test': Operation not supported"
1211+
: attempts === 2
1212+
? "fatal: Unable to create '/repo/.git/index.lock': File exists."
1213+
: "",
1214+
timedOut: false,
1215+
stdoutTruncated: false,
1216+
stderrTruncated: false,
1217+
stdoutInvalidUtf8: false,
1218+
stderrInvalidUtf8: false,
1219+
};
1220+
}),
1221+
}),
1222+
);
1223+
const driver = yield* GitVcsDriver.makeVcsDriverShape().pipe(
1224+
Effect.provideService(VcsProcess.VcsProcess, process),
1225+
);
1226+
const fiber = yield* driver
1227+
.execute({ operation, cwd: "/repo", args: ["-c", "core.fsyncMethod=batch", "add"] })
1228+
.pipe(Effect.forkChild);
1229+
yield* Deferred.await(retryReady);
1230+
yield* TestClock.adjust("75 millis");
1231+
yield* Fiber.join(fiber);
1232+
1233+
assert.strictEqual(attempts, 3);
1234+
assert.strictEqual(yield* gitCommandCount(operation, "failure"), 2);
1235+
assert.strictEqual(yield* gitCommandCount(operation, "success"), 1);
1236+
const timing = yield* gitCommandTiming(operation);
1237+
assert.strictEqual(timing.count, 3);
1238+
assert.strictEqual(timing.sum, 750);
1239+
}).pipe(
1240+
Effect.provide(NodeServices.layer),
1241+
Effect.provideService(Metric.MetricRegistry, new Map()),
1242+
),
1243+
);

‎apps/server/src/vcs/GitVcsDriver.ts‎

Lines changed: 35 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@ import {
3535
} from "@t3tools/contracts";
3636
import { isMissingExecutable } from "@t3tools/source-control-core/server/discovery";
3737
import * as HostProcess from "@t3tools/shared/HostProcess";
38-
import { gitCommandDuration, gitCommandsTotal, withMetrics } from "../observability/Metrics.ts";
3938
import {
4039
makeGitVcsDriverCore,
4140
PATCH_RENDER_PREFIX_ARGS,
@@ -504,51 +503,41 @@ function parseGitRemoteVerboseOutput(
504503
return remotes;
505504
}
506505

507-
const runGitProcess = (
508-
process: VcsProcess.VcsProcess["Service"],
509-
input: VcsProcess.VcsProcessInput,
510-
) =>
511-
process.run(input).pipe(
512-
withMetrics({
513-
counter: gitCommandsTotal,
514-
timer: gitCommandDuration,
515-
attributes: { operation: input.operation },
506+
const gitCommand = Effect.fn("GitVcsDriver.gitCommand")(
507+
(
508+
process: VcsProcess.VcsProcess["Service"],
509+
operation: string,
510+
cwd: string,
511+
args: ReadonlyArray<string>,
512+
options?: {
513+
readonly stdin?: string;
514+
readonly env?: NodeJS.ProcessEnv;
515+
readonly allowNonZeroExit?: boolean;
516+
readonly timeoutMs?: number;
517+
readonly maxOutputBytes?: number;
518+
readonly outputMode?: VcsProcess.VcsProcessInput["outputMode"];
519+
readonly appendTruncationMarker?: boolean;
520+
},
521+
) =>
522+
process.run({
523+
operation,
524+
command: "git",
525+
args: ["-C", cwd, ...args],
526+
cwd,
527+
spawnCwd: globalThis.process.cwd(),
528+
...(options?.stdin !== undefined ? { stdin: options.stdin } : {}),
529+
...(options?.env !== undefined ? { env: options.env } : {}),
530+
...(options?.allowNonZeroExit !== undefined
531+
? { allowNonZeroExit: options.allowNonZeroExit }
532+
: {}),
533+
...(options?.timeoutMs !== undefined ? { timeoutMs: options.timeoutMs } : {}),
534+
...(options?.maxOutputBytes !== undefined ? { maxOutputBytes: options.maxOutputBytes } : {}),
535+
...(options?.outputMode !== undefined ? { outputMode: options.outputMode } : {}),
536+
...(options?.appendTruncationMarker !== undefined
537+
? { appendTruncationMarker: options.appendTruncationMarker }
538+
: {}),
516539
}),
517-
);
518-
519-
const gitCommand = (
520-
process: VcsProcess.VcsProcess["Service"],
521-
operation: string,
522-
cwd: string,
523-
args: ReadonlyArray<string>,
524-
options?: {
525-
readonly stdin?: string;
526-
readonly env?: NodeJS.ProcessEnv;
527-
readonly allowNonZeroExit?: boolean;
528-
readonly timeoutMs?: number;
529-
readonly maxOutputBytes?: number;
530-
readonly outputMode?: VcsProcess.VcsProcessInput["outputMode"];
531-
readonly appendTruncationMarker?: boolean;
532-
},
533-
) =>
534-
runGitProcess(process, {
535-
operation,
536-
command: "git",
537-
args: ["-C", cwd, ...args],
538-
cwd,
539-
spawnCwd: globalThis.process.cwd(),
540-
...(options?.stdin !== undefined ? { stdin: options.stdin } : {}),
541-
...(options?.env !== undefined ? { env: options.env } : {}),
542-
...(options?.allowNonZeroExit !== undefined
543-
? { allowNonZeroExit: options.allowNonZeroExit }
544-
: {}),
545-
...(options?.timeoutMs !== undefined ? { timeoutMs: options.timeoutMs } : {}),
546-
...(options?.maxOutputBytes !== undefined ? { maxOutputBytes: options.maxOutputBytes } : {}),
547-
...(options?.outputMode !== undefined ? { outputMode: options.outputMode } : {}),
548-
...(options?.appendTruncationMarker !== undefined
549-
? { appendTruncationMarker: options.appendTruncationMarker }
550-
: {}),
551-
});
540+
);
552541

553542
export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* () {
554543
const fileSystem = yield* FileSystem.FileSystem;
@@ -916,7 +905,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
916905
let skipped = false;
917906
let skippedRecord: number[] = [];
918907
const skippedPaths: string[] = [];
919-
yield* runGitProcess(vcsProcess, {
908+
yield* vcsProcess.run({
920909
operation,
921910
command: "git",
922911
cwd: input.cwd,

‎apps/server/src/vcs/VcsProcess.ts‎

Lines changed: 48 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import {
2121
} from "@t3tools/contracts";
2222
import * as HostProcess from "@t3tools/shared/HostProcess";
2323

24+
import { gitCommandDuration, gitCommandsTotal, withMetrics } from "../observability/Metrics.ts";
2425
import * as ProcessRunner from "../processRunner.ts";
2526

2627
export interface VcsProcessInput {
@@ -68,6 +69,27 @@ export const GIT_INDEX_LOCKED_DETAIL =
6869

6970
export const CHECKPOINT_CAPTURE_OPERATION = "GitVcsDriver.checkpoints.captureCheckpoint";
7071

72+
// Caller operations are diagnostic strings; only known labels may become metric attributes.
73+
const gitMetricOperations = new Set([
74+
CHECKPOINT_CAPTURE_OPERATION,
75+
"GitVcsDriver.checkpoints.deleteCheckpointRefs",
76+
"GitVcsDriver.checkpoints.diffCheckpoints",
77+
"GitVcsDriver.checkpoints.hasHeadCommit",
78+
"GitVcsDriver.checkpoints.listAuthoredPaths",
79+
"GitVcsDriver.checkpoints.resolveCheckpointCommit",
80+
"GitVcsDriver.checkpoints.resolveGitCommonDir",
81+
"GitVcsDriver.checkpoints.resolveHeadCommit",
82+
"GitVcsDriver.checkpoints.restoreCheckpoint",
83+
"GitVcsDriver.detectRepository.commonDir",
84+
"GitVcsDriver.detectRepository.root",
85+
"GitVcsDriver.filterIgnoredPaths",
86+
"GitVcsDriver.initRepository",
87+
"GitVcsDriver.isInsideWorkTree",
88+
"GitVcsDriver.listRemotes",
89+
"GitVcsDriver.listWorkspaceFiles",
90+
"GitVcsDriver.resolveGitCommonDir",
91+
]);
92+
7193
/**
7294
* Appends `key=value` to the `GIT_CONFIG_*` entries git reads as command-line
7395
* config, so argv stays the same on every platform and nothing is written to
@@ -184,8 +206,12 @@ export const make = Effect.gen(function* () {
184206
const vcsProcesses = yield* Semaphore.make(VCS_PROCESS_CONCURRENCY);
185207
const githubProcesses = yield* Semaphore.make(GITHUB_PROCESS_CONCURRENCY);
186208

187-
const spawn = (input: VcsProcessInput, baseError: VcsProcessErrorContext) =>
188-
processRunner
209+
const spawn = Effect.fn("VcsProcess.spawn")(function* (
210+
input: VcsProcessInput,
211+
baseError: VcsProcessErrorContext,
212+
) {
213+
let failedExit = false;
214+
const execution = processRunner
189215
.run({
190216
command: input.command,
191217
args: input.args,
@@ -204,6 +230,10 @@ export const make = Effect.gen(function* () {
204230
timeoutBehavior: "error",
205231
})
206232
.pipe(
233+
Effect.tap((result) => {
234+
failedExit = result.code === null || (!input.allowNonZeroExit && result.code !== 0);
235+
return Effect.void;
236+
}),
207237
Effect.mapError(
208238
Match.valueTags({
209239
ProcessSpawnError: (error) =>
@@ -233,6 +263,22 @@ export const make = Effect.gen(function* () {
233263
),
234264
);
235265

266+
return yield* input.command === "git"
267+
? execution.pipe(
268+
withMetrics({
269+
counter: gitCommandsTotal,
270+
timer: gitCommandDuration,
271+
attributes: {
272+
operation: gitMetricOperations.has(input.operation) ? input.operation : "other",
273+
},
274+
outcomeAttributes: (outcome) => ({
275+
outcome: outcome === "success" && failedExit ? "failure" : outcome,
276+
}),
277+
}),
278+
)
279+
: execution;
280+
});
281+
236282
const runUnbounded = Effect.fn("VcsProcess.runUnbounded")(function* (input: VcsProcessInput) {
237283
const baseError = {
238284
operation: input.operation,

0 commit comments

Comments
 (0)