Skip to content

Commit 65e4272

Browse files
fix(server): a hung network mount no longer freezes the server when it spawns a process (#36327)
1 parent 4115648 commit 65e4272

11 files changed

Lines changed: 567 additions & 6 deletions

File tree

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -374,10 +374,13 @@ const CLAUDE_ROOT_REFUSAL =
374374
"--dangerously-skip-permissions cannot be used with root/sudo privileges for security reasons";
375375

376376
// fd 4 gates the launch until the server has recorded the process it is about to become.
377+
// The shell enters the cwd itself: as a native cwd, a mount that hangs chdir freezes the server.
377378
const CLAUDE_GROUP_WATCH_SCRIPT = [
378379
"read -r _ <&4 || exit 1",
379380
"exec 4<&-",
380381
"{ trap '' TERM; read -r _ <&3; kill -TERM 0; sleep 5; kill -KILL 0; } </dev/null >/dev/null 2>&1 &",
382+
'cd -P -- "$1" || exit 125',
383+
"shift",
381384
'exec "$@" 3<&-',
382385
].join("\n");
383386

@@ -394,11 +397,19 @@ export function spawnClaudeProcessGroup(
394397
onStderrClose: () => void = () => {},
395398
register?: (pid: number) => () => void,
396399
): ClaudeSpawnedProcess & Pick<NodeChildProcess.ChildProcess, "pid"> {
400+
// Relative to the server's cwd, as a native cwd is; the `./` keeps CDPATH out of it.
401+
const cwd = options.cwd ?? ".";
397402
const child = NodeChildProcess.spawn(
398403
"/bin/sh",
399-
["-c", CLAUDE_GROUP_WATCH_SCRIPT, "t3-claude", options.command, ...options.args],
404+
[
405+
"-c",
406+
CLAUDE_GROUP_WATCH_SCRIPT,
407+
"t3-claude",
408+
cwd.startsWith("/") ? cwd : `./${cwd}`,
409+
options.command,
410+
...options.args,
411+
],
400412
{
401-
cwd: options.cwd,
402413
env: options.env,
403414
signal: options.signal,
404415
stdio: ["pipe", "pipe", "pipe", "pipe", "pipe"],

‎apps/server/src/orchestration-v2/Adapters/ClaudeAgentSdkQueryRunner.test.ts‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -748,6 +748,38 @@ describe.skipIf(HostProcess.Platform.defaultValue() === "win32")("Claude CLI pro
748748
}),
749749
);
750750

751+
// A native cwd makes libuv wait for chdir, which a hung mount stalls with the whole server.
752+
it.effect("enters the cwd in the child, so a missing one fails the CLI and not the spawn", () =>
753+
Effect.gen(function* () {
754+
const cli = fakeClaudeCli('pwd -P > "$(dirname "$0")/cwd"');
755+
const dir = NodePath.dirname(cli);
756+
const workspace = NodePath.join(dir, "work space");
757+
NodeFS.mkdirSync(workspace);
758+
yield* Effect.addFinalizer(() =>
759+
Effect.sync(() => NodeFS.rmSync(dir, { recursive: true, force: true })),
760+
);
761+
const exitCode = (cwd: string) =>
762+
Effect.promise(
763+
() =>
764+
new Promise<number | null>((resolve) =>
765+
spawnClaudeProcessGroup({
766+
command: cli,
767+
args: [],
768+
cwd,
769+
env: process.env,
770+
signal: new AbortController().signal,
771+
}).once("exit", resolve),
772+
),
773+
);
774+
assert.equal(yield* exitCode(workspace), 0);
775+
assert.equal(
776+
NodeFS.readFileSync(NodePath.join(dir, "cwd"), "utf8"),
777+
`${NodeFS.realpathSync(workspace)}\n`,
778+
);
779+
assert.equal(yield* exitCode(NodePath.join(dir, "missing")), 125);
780+
}),
781+
);
782+
751783
// A restarted server reaps recorded CLIs, so a CLI must never run before its record exists.
752784
it.effect("records the CLI before it runs and forgets it once it exits", () =>
753785
Effect.gen(function* () {
Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
// @effect-diagnostics nodeBuiltinImport:off - the spawner's checks are node:fs/promises calls, which the test spies on.
2+
import * as NodeFSP from "node:fs/promises";
3+
4+
import * as NodeServices from "@effect/platform-node/NodeServices";
5+
import { describe, expect, it } from "@effect/vitest";
6+
import { vi } from "vite-plus/test";
7+
import * as HostProcess from "@t3tools/shared/HostProcess";
8+
import * as Deferred from "effect/Deferred";
9+
import * as Effect from "effect/Effect";
10+
import * as Fiber from "effect/Fiber";
11+
import * as FileSystem from "effect/FileSystem";
12+
import * as Layer from "effect/Layer";
13+
import * as Path from "effect/Path";
14+
import * as ChildProcess from "effect/process/ChildProcess";
15+
import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
16+
17+
import * as NonBlockingSpawner from "./NonBlockingSpawner.ts";
18+
19+
vi.mock("node:fs/promises", { spy: true });
20+
21+
const hostPath = process.env.PATH;
22+
23+
const wrapped = NonBlockingSpawner.layer.pipe(Layer.provideMerge(NodeServices.layer));
24+
25+
/** What a caller sees of a command: its output, or the error of a spawn that failed. */
26+
const outcome = (command: ChildProcess.Command) =>
27+
ChildProcessSpawner.ChildProcessSpawner.use((spawner) => spawner.string(command)).pipe(
28+
Effect.match({
29+
onSuccess: (output) => ({ output }),
30+
onFailure: (error) => ({ reason: error.reason._tag, message: error.message }),
31+
}),
32+
);
33+
34+
describe.skipIf(HostProcess.Platform.defaultValue() !== "linux")("NonBlockingSpawner", () => {
35+
it.live("gives callers what the native spawner gives them", () =>
36+
Effect.gen(function* () {
37+
const fileSystem = yield* FileSystem.FileSystem;
38+
const root = yield* fileSystem.realPath(yield* fileSystem.makeTempDirectoryScoped());
39+
for (const [file, mode] of [
40+
["stale/tool", 0o644],
41+
["fresh/tool", 0o755],
42+
["target/bin/tool", 0o755],
43+
// GNU env reads a lone `-` as -i and `NAME=value` as an assignment, even after `--`.
44+
["fresh/-", 0o755],
45+
["fresh/launch=stable", 0o755],
46+
] as const) {
47+
yield* fileSystem.makeDirectory(`${root}/${file.slice(0, file.lastIndexOf("/"))}`, {
48+
recursive: true,
49+
});
50+
yield* fileSystem.writeFileString(`${root}/${file}`, `#!/bin/sh\necho ${file} "$@"\n`);
51+
yield* fileSystem.chmod(`${root}/${file}`, mode);
52+
}
53+
yield* fileSystem.makeDirectory(`${root}/target/subdir`);
54+
yield* fileSystem.symlink(`${root}/target/subdir`, `${root}/link`);
55+
const PATH = `${root}/stale:${root}/fresh`;
56+
57+
const commands = {
58+
"enters the cwd": ChildProcess.make("pwd", ["-P"], { cwd: root }),
59+
"keeps argv": ChildProcess.make(
60+
"printf",
61+
["%s\\n", "-n", "--", "a b", `it's "quoted"`, "$HOME", "; exit 3", "", "A=b"],
62+
{ cwd: root },
63+
),
64+
// dash would drop the exported function and add PWD and OLDPWD.
65+
"keeps the environment": ChildProcess.make("env", [], {
66+
cwd: root,
67+
env: { PATH: hostPath, A: "1", "BASH_FUNC_x%%": "() { echo exported\n}" },
68+
}),
69+
"skips a non-executable PATH entry": ChildProcess.make("tool", [], { env: { PATH } }),
70+
"resolves `..` in PATH after a symlink": ChildProcess.make("tool", [], {
71+
env: { PATH: `${root}/link/../bin` },
72+
}),
73+
"searches PATH relative to the cwd": ChildProcess.make("tool", [], {
74+
cwd: root,
75+
env: { PATH: "fresh" },
76+
}),
77+
"a command named -": ChildProcess.make("-", [], { cwd: root, env: { PATH } }),
78+
"a command containing =": ChildProcess.make("launch=stable", [], { env: { PATH } }),
79+
"shell mode": ChildProcess.make('pwd -P; echo "$0" "a b"', [], { cwd: root, shell: true }),
80+
"a pipeline": ChildProcess.make("printf", ["a\\nb\\n"], { cwd: root }).pipe(
81+
ChildProcess.pipeTo(ChildProcess.make("tail", ["-n", "1"])),
82+
),
83+
"a missing command": ChildProcess.make("t3code-missing-command", ["--version"]),
84+
"a missing command in a cwd": ChildProcess.make("t3code-missing-command", [], {
85+
cwd: root,
86+
}),
87+
"a non-executable command": ChildProcess.make("tool", [], {
88+
env: { PATH: `${root}/stale` },
89+
}),
90+
"a missing explicit path": ChildProcess.make("./missing", [], { cwd: root }),
91+
"a non-executable explicit path": ChildProcess.make(`${root}/stale/tool`, []),
92+
"a missing cwd": ChildProcess.make("true", [], { cwd: `${root}/missing` }),
93+
// The native check sees `missing`, although the path resolves to an existing directory.
94+
"a cwd through a missing directory": ChildProcess.make("true", [], {
95+
cwd: `${root}/missing/..`,
96+
}),
97+
};
98+
for (const [name, command] of Object.entries(commands)) {
99+
const native = yield* outcome(command).pipe(Effect.provide(NodeServices.layer));
100+
expect(yield* outcome(command).pipe(Effect.provide(wrapped)), name).toEqual(native);
101+
}
102+
}).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
103+
);
104+
105+
it.live("hands the native spawner only `env`, never a cwd or a command to look up", () =>
106+
Effect.gen(function* () {
107+
const fileSystem = yield* FileSystem.FileSystem;
108+
const directory = yield* fileSystem.makeTempDirectoryScoped();
109+
const received: Array<ChildProcess.StandardCommand> = [];
110+
const record = (command: ChildProcess.Command): void => {
111+
if (command._tag === "StandardCommand") received.push(command);
112+
else [command.left, command.right].forEach(record);
113+
};
114+
const recording = Layer.effect(
115+
ChildProcessSpawner.ChildProcessSpawner,
116+
Effect.map(ChildProcessSpawner.ChildProcessSpawner, (native) =>
117+
ChildProcessSpawner.make((command) => {
118+
record(command);
119+
return native.spawn(command);
120+
}),
121+
),
122+
).pipe(Layer.provideMerge(NodeServices.layer));
123+
const commands = [
124+
ChildProcess.make("true", [], { cwd: directory }),
125+
ChildProcess.make("true", []),
126+
ChildProcess.make("/bin/true", []),
127+
ChildProcess.make("t3code-missing-command", []),
128+
ChildProcess.make("true", [], { cwd: directory, shell: true }),
129+
ChildProcess.make("printf", ["a"], { cwd: directory }).pipe(
130+
ChildProcess.pipeTo(ChildProcess.make("cat", [], { cwd: directory })),
131+
),
132+
];
133+
yield* Effect.forEach(commands, (command) =>
134+
ChildProcessSpawner.ChildProcessSpawner.use((spawner) => spawner.exitCode(command)).pipe(
135+
Effect.exit,
136+
),
137+
).pipe(Effect.provide(NonBlockingSpawner.layer.pipe(Layer.provide(recording))));
138+
139+
expect(received.map((command) => [command.command, command.options.cwd])).toEqual(
140+
Array.from({ length: 6 }, () => ["/usr/bin/env", undefined]),
141+
);
142+
}).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
143+
);
144+
145+
it.live("checks at most two paths at once, however many directories hang", () =>
146+
Effect.gen(function* () {
147+
const allArrived = yield* Deferred.make<void>();
148+
let arrivals = 0;
149+
// Each spawn resolves its cwd, then starts its check within microtasks.
150+
const arrivalPath = Layer.effect(
151+
Path.Path,
152+
Effect.map(Path.Path, (path) => ({
153+
...path,
154+
resolve: (...segments: ReadonlyArray<string>) => {
155+
if (++arrivals === 3) setImmediate(() => Deferred.doneUnsafe(allArrived, Effect.void));
156+
return path.resolve(...segments);
157+
},
158+
})),
159+
);
160+
const hung: Array<() => void> = [];
161+
let releasing = false;
162+
let running = 0;
163+
let mostRunning = 0;
164+
using _access = vi.spyOn(NodeFSP, "access").mockImplementation(
165+
() =>
166+
new Promise<void>((_resolve, reject) => {
167+
mostRunning = Math.max(mostRunning, ++running);
168+
const fail = () => {
169+
running--;
170+
reject(Object.assign(new Error("ENOENT"), { code: "ENOENT" }));
171+
};
172+
if (releasing) fail();
173+
else hung.push(fail);
174+
}),
175+
);
176+
const spawns = yield* Effect.forEach(
177+
["a", "b", "c"],
178+
(name) =>
179+
ChildProcessSpawner.ChildProcessSpawner.use((spawner) =>
180+
spawner.exitCode(ChildProcess.make("true", [], { cwd: `/t3code-hung/${name}` })),
181+
).pipe(Effect.flip),
182+
{ concurrency: "unbounded" },
183+
).pipe(
184+
Effect.provide(
185+
NonBlockingSpawner.layer.pipe(
186+
Layer.provide(arrivalPath),
187+
Layer.provide(NodeServices.layer),
188+
),
189+
),
190+
Effect.forkChild,
191+
);
192+
yield* Deferred.await(allArrived);
193+
expect(mostRunning).toBe(2);
194+
releasing = true;
195+
for (const fail of hung.splice(0)) fail();
196+
yield* Fiber.join(spawns);
197+
expect(mostRunning).toBe(2);
198+
}),
199+
);
200+
});

0 commit comments

Comments
 (0)