Skip to content

Commit e82895b

Browse files
fix(server): hung mounts hold at most two threadpool threads during spawn checks
Checks were only serialized per directory, so spawns into four distinct hung directories (worktrees of a project on one NFS mount) took every libuv threadpool thread and stalled WebSocket compression again. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent b38ff31 commit e82895b

2 files changed

Lines changed: 80 additions & 5 deletions

File tree

‎apps/server/src/process/NonBlockingSpawner.test.ts‎

Lines changed: 62 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,25 @@
1-
// @effect-diagnostics-next-line nodeBuiltinImport:off - a skip condition must be known before the test runs.
1+
// @effect-diagnostics nodeBuiltinImport:off - a skip condition must be known before the test runs, and the spawner's checks are node:fs/promises calls.
22
import * as NodeChildProcess from "node:child_process";
3+
import * as NodeFSP from "node:fs/promises";
34

45
import * as NodeServices from "@effect/platform-node/NodeServices";
56
import { describe, expect, it } from "@effect/vitest";
7+
import { vi } from "vite-plus/test";
68
import * as HostProcess from "@t3tools/shared/HostProcess";
9+
import * as Deferred from "effect/Deferred";
710
import * as Effect from "effect/Effect";
11+
import * as Fiber from "effect/Fiber";
812
import * as FileSystem from "effect/FileSystem";
913
import * as Layer from "effect/Layer";
14+
import * as Path from "effect/Path";
1015
import * as PlatformError from "effect/PlatformError";
1116
import * as ChildProcess from "effect/process/ChildProcess";
1217
import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
1318

1419
import * as NonBlockingSpawner from "./NonBlockingSpawner.ts";
1520

21+
vi.mock("node:fs/promises", { spy: true });
22+
1623
const wrapped = NonBlockingSpawner.layer.pipe(Layer.provideMerge(NodeServices.layer));
1724

1825
// A host without /usr/bin/env, such as a minimal container, spawns through /bin/sh.
@@ -186,6 +193,60 @@ describe.skipIf(HostProcess.Platform.defaultValue() !== "linux")("NonBlockingSpa
186193
}).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
187194
);
188195

196+
// A hung check holds a libuv threadpool thread, and those also compress WebSocket frames.
197+
it.live("checks at most two paths at once, however many directories hang", () =>
198+
Effect.gen(function* () {
199+
const allArrived = yield* Deferred.make<void>();
200+
let arrivals = 0;
201+
// Each spawn resolves its cwd, then starts its check within microtasks.
202+
const arrivalPath = Layer.effect(
203+
Path.Path,
204+
Effect.map(Path.Path, (path) => ({
205+
...path,
206+
resolve: (...segments: ReadonlyArray<string>) => {
207+
if (++arrivals === 3) setImmediate(() => Deferred.doneUnsafe(allArrived, Effect.void));
208+
return path.resolve(...segments);
209+
},
210+
})),
211+
);
212+
const hung: Array<() => void> = [];
213+
let releasing = false;
214+
let running = 0;
215+
let mostRunning = 0;
216+
using _access = vi.spyOn(NodeFSP, "access").mockImplementation(
217+
() =>
218+
new Promise<void>((_resolve, reject) => {
219+
mostRunning = Math.max(mostRunning, ++running);
220+
const fail = () => {
221+
running--;
222+
reject(Object.assign(new Error("ENOENT"), { code: "ENOENT" }));
223+
};
224+
if (releasing) fail();
225+
else hung.push(fail);
226+
}),
227+
);
228+
const spawns = yield* Effect.forEach(
229+
["a", "b", "c"],
230+
(name) => spawnError(ChildProcess.make("true", [], { cwd: `/t3code-hung-30183/${name}` })),
231+
{ concurrency: "unbounded" },
232+
).pipe(
233+
Effect.provide(
234+
NonBlockingSpawner.layer.pipe(
235+
Layer.provide(arrivalPath),
236+
Layer.provide(NodeServices.layer),
237+
),
238+
),
239+
Effect.forkChild,
240+
);
241+
yield* Deferred.await(allArrived);
242+
expect(mostRunning).toBe(2);
243+
releasing = true;
244+
for (const fail of hung.splice(0)) fail();
245+
yield* Fiber.join(spawns);
246+
expect(mostRunning).toBe(2);
247+
}),
248+
);
249+
189250
it.live("only ever hands the native spawner a trampoline, without a cwd", () =>
190251
Effect.gen(function* () {
191252
const fileSystem = yield* FileSystem.FileSystem;

‎apps/server/src/process/NonBlockingSpawner.ts‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -114,15 +114,29 @@ const make = Effect.gen(function* () {
114114
Effect.orElseSucceed(() => false),
115115
);
116116

117-
// At most one check runs per directory and checks of one path share it, so a
118-
// hung directory holds one of libuv's four threadpool threads, which also
119-
// compress every WebSocket frame.
117+
// At most one check runs per directory, checks of one path share it, and at
118+
// most two run at once, so hung mounts never hold more than two of libuv's
119+
// four threadpool threads, which also compress every WebSocket frame.
120120
const inFlight = new Map<string, Promise<string>>();
121121
const queues = new Map<string, Promise<unknown>>();
122+
const waiting: Array<() => void> = [];
123+
let probing = 0;
124+
const limitedProbe = async (target: string) => {
125+
if (probing < 2) probing++;
126+
else await new Promise<void>((resume) => waiting.push(resume));
127+
try {
128+
return await probe(target);
129+
} finally {
130+
// A waiter inherits the slot.
131+
const next = waiting.shift();
132+
if (next === undefined) probing--;
133+
else next();
134+
}
135+
};
122136
const check = (directory: string, target: string) => {
123137
const pending = inFlight.get(target);
124138
if (pending !== undefined) return pending;
125-
const result = (queues.get(directory) ?? Promise.resolve()).then(() => probe(target));
139+
const result = (queues.get(directory) ?? Promise.resolve()).then(() => limitedProbe(target));
126140
queues.set(directory, result);
127141
inFlight.set(target, result);
128142
void result.then(() => {

0 commit comments

Comments
 (0)