Skip to content

Commit 0d71f46

Browse files
committed
fix(server): stop the metadata reader from piling up stuck reads, and time its answers
A file read cannot be cancelled once started and holds a libuv thread until the disk answers. While an attempt that ran out of budget is still pending, the reader now starts no new reads and leaves every command to git, so a stuck disk or share cannot exhaust the threadpool and stall unrelated file work. Answered commands now record their read time in t3_git_command_duration instead of almost zero.
1 parent 35e22bd commit 0d71f46

4 files changed

Lines changed: 70 additions & 13 deletions

File tree

‎apps/server/src/observability/Metrics.ts‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,20 @@ const withMetricsImpl = <A, E, R>(
104104
const exit = yield* Effect.exit(effect);
105105
const endedAt = yield* Clock.currentTimeNanos;
106106
const elapsedNanos = endedAt > startedAt ? endedAt - startedAt : 0n;
107-
const duration = Duration.nanos(elapsedNanos);
107+
yield* recordMetrics(options, exit, Duration.nanos(elapsedNanos));
108+
if (Exit.isSuccess(exit)) {
109+
return exit.value;
110+
}
111+
return yield* Effect.failCause(exit.cause);
112+
});
113+
114+
/** Records what `withMetrics` would for an outcome whose duration was measured elsewhere. */
115+
export const recordMetrics = (
116+
options: WithMetricsOptions,
117+
exit: Exit.Exit<unknown, unknown>,
118+
duration: Duration.Duration,
119+
) =>
120+
Effect.gen(function* () {
108121
const baseAttributes =
109122
typeof options.attributes === "function" ? options.attributes() : (options.attributes ?? {});
110123

@@ -129,11 +142,6 @@ const withMetricsImpl = <A, E, R>(
129142
1,
130143
);
131144
}
132-
133-
if (Exit.isSuccess(exit)) {
134-
return exit.value;
135-
}
136-
return yield* Effect.failCause(exit.cause);
137145
});
138146

139147
export const withMetrics: {

‎apps/server/src/vcs/GitMetadataFastPath.test.ts‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import {
1212
rememberGitAnswer,
1313
resetGitFastPathCaches,
1414
tryAnswerGitCommand,
15+
withinReadBudget,
1516
} from "./GitMetadataFastPath.ts";
1617

1718
const git = (cwd: string, ...args: ReadonlyArray<string>) =>
@@ -509,6 +510,32 @@ describe("GitMetadataFastPath on repositories git treats differently", () => {
509510
await answered("tilde.inc", "tilde");
510511
});
511512

513+
it("starts no reads while an abandoned one is still stuck", async () => {
514+
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
515+
try {
516+
let releaseStuck!: () => void;
517+
const stuck = withinReadBudget(
518+
10,
519+
() =>
520+
new Promise<string>((resolve) => {
521+
releaseStuck = () => resolve("late");
522+
}),
523+
);
524+
await vi.advanceTimersByTimeAsync(10);
525+
expect(await stuck).toBeNull();
526+
527+
const work = vi.fn(async () => "fresh");
528+
expect(await withinReadBudget(10, work)).toBeNull();
529+
expect(work).not.toHaveBeenCalled();
530+
531+
releaseStuck();
532+
await vi.advanceTimersByTimeAsync(0);
533+
expect(await withinReadBudget(10, work)).toBe("fresh");
534+
} finally {
535+
vi.useRealTimers();
536+
}
537+
});
538+
512539
it("stays inside the caller's time and output budget", async () => {
513540
const cwd = repos.plain!;
514541
expect(await tryAnswerGitCommand({ cwd, args: ["remote", "-v"], timeoutMs: 0 })).toBeNull();

‎apps/server/src/vcs/GitMetadataFastPath.ts‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ export const isGitFastPathEnabled = (env: NodeJS.ProcessEnv = process.env) =>
4545
env.T3CODE_GIT_FAST_PATH !== "0";
4646

4747
const ANSWER_TIMEOUT_MS = 2_000;
48+
// Attempts that ran out of budget while their reads were still pending.
49+
let abandonedReads = 0;
4850
// Same default as the process runners that would otherwise spawn git.
4951
const DEFAULT_MAX_OUTPUT_BYTES = 1_000_000;
5052

@@ -1294,24 +1296,38 @@ export async function tryAnswerGitCommand(
12941296
* ANSWER_TIMEOUT_MS runs out, whichever comes first. A rejection of `work`
12951297
* before then is passed on.
12961298
*/
1297-
async function withinReadBudget<T>(
1299+
export async function withinReadBudget<T>(
12981300
timeoutMs: number | null | undefined,
12991301
work: () => Promise<T>,
13001302
): Promise<T | null> {
13011303
// No budget at all: nothing may be answered, however fast the reads turn out.
13021304
if (typeof timeoutMs === "number" && timeoutMs <= 0) return null;
1305+
// A started read cannot be cancelled and holds one of libuv's few threads until
1306+
// the disk answers. Starting more while one is stuck would pile them up and
1307+
// stall every file operation in the server, so git gets every question instead.
1308+
if (abandonedReads > 0) return null;
13031309
let timer: NodeJS.Timeout | undefined;
13041310
// Reads that take this long mean a stuck disk or share; git gets the question instead.
13051311
const timedOut = new Promise<null>((resolve) => {
13061312
timer = setTimeout(resolve, Math.min(ANSWER_TIMEOUT_MS, timeoutMs ?? Infinity), null);
13071313
});
1308-
const working = work();
1314+
let settled = false;
1315+
const working = work().finally(() => {
1316+
settled = true;
1317+
});
13091318
// When the timer wins, the abandoned attempt still settles; its decline is not an error.
13101319
working.catch(() => undefined);
13111320
try {
13121321
return await Promise.race([working, timedOut]);
13131322
} finally {
13141323
clearTimeout(timer);
1324+
if (!settled) {
1325+
abandonedReads++;
1326+
void working.then(
1327+
() => abandonedReads--,
1328+
() => abandonedReads--,
1329+
);
1330+
}
13151331
}
13161332
}
13171333

‎apps/server/src/vcs/GitVcsDriverCore.ts‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,12 @@ import { compactTraceAttributes } from "@t3tools/shared/observability";
3333
import { decodeJsonResult } from "@t3tools/shared/schemaJson";
3434
import { parseT3ProjectFile } from "@t3tools/shared/t3ProjectFile";
3535
import { resolveProjectFileBackedSetting } from "@t3tools/shared/projectSettings";
36-
import { gitCommandDuration, gitCommandsTotal, withMetrics } from "../observability/Metrics.ts";
36+
import {
37+
gitCommandDuration,
38+
gitCommandsTotal,
39+
recordMetrics,
40+
withMetrics,
41+
} from "../observability/Metrics.ts";
3742
import * as GitMetadataFastPath from "./GitMetadataFastPath.ts";
3843
import * as GitVcsDriver from "./GitVcsDriver.ts";
3944
import {
@@ -1008,7 +1013,8 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
10081013
);
10091014

10101015
const execute: GitVcsDriver.GitVcsDriver["Service"]["execute"] = (input) => {
1011-
// Times the command itself, not the wait for a process permit.
1016+
// Times the command itself, not the wait for a process permit. An answered
1017+
// command took as long as its file reads; a miss is timed from its spawn.
10121018
const metrics = {
10131019
counter: gitCommandsTotal,
10141020
timer: gitCommandDuration,
@@ -1017,10 +1023,10 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
10171023
},
10181024
};
10191025
const spawnGit = executeRaw(input).pipe(withMetrics(metrics));
1020-
return answerWithoutGit(input).pipe(
1021-
Effect.flatMap((answer) =>
1026+
return Effect.timed(answerWithoutGit(input)).pipe(
1027+
Effect.flatMap(([readDuration, answer]) =>
10221028
answer !== null
1023-
? Effect.succeed(answer).pipe(withMetrics(metrics))
1029+
? recordMetrics(metrics, Exit.succeed(answer), readDuration).pipe(Effect.as(answer))
10241030
: input.timeoutMs === null || (input.timeoutMs ?? DEFAULT_TIMEOUT_MS) > DEFAULT_TIMEOUT_MS
10251031
? spawnGit
10261032
: gitProcesses.withPermits(1)(spawnGit),

0 commit comments

Comments
 (0)