Skip to content

Commit 6c540d2

Browse files
committed
fix(server): MCP plugin servers die with the Claude CLI's process group
Closing or deleting a session leaked the CLI's MCP plugin servers (each plugin's bun server.ts): the SDK transport signals only the CLI PID, so the plugin tree survived as orphans logging into a stdout pipe nobody drains, pinning a core per wedged server until kill -9 (#3089). The query runner now hands the SDK a spawner that runs the CLI detached in its own POSIX process group, turns every kill into a group kill, and SIGKILLs whatever is left in the group when the CLI exits for any reason, so the transport's graceful close reaps the whole tree. Caller-supplied spawners (remote runtimes, tests) keep ownership; Windows keeps the SDK's own spawn. This work was done by GLM (a model) in the ZCode agent harness.
1 parent 6524a8d commit 6c540d2

2 files changed

Lines changed: 359 additions & 1 deletion

File tree

Lines changed: 278 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,278 @@
1+
// @effect-diagnostics nodeBuiltinImport:off globalDate:off globalTimers:off -- plain-async polling for process death.
2+
import * as ClaudeSdk from "@anthropic-ai/claude-agent-sdk";
3+
import * as NodeServices from "@effect/platform-node/NodeServices";
4+
import { describe, expect, it } from "@effect/vitest";
5+
import { ProviderSessionId, ThreadId } from "@t3tools/contracts";
6+
import * as Data from "effect/Data";
7+
import * as Effect from "effect/Effect";
8+
import * as Layer from "effect/Layer";
9+
import type * as NodeChildProcess from "node:child_process";
10+
import * as NodeFS from "node:fs";
11+
import * as NodeOS from "node:os";
12+
import * as NodePath from "node:path";
13+
import { vi } from "vite-plus/test";
14+
15+
import { ProviderEventLoggers } from "@t3tools/provider-core/server/ProviderEventLoggers";
16+
17+
import {
18+
ClaudeAgentSdkQueryRunner,
19+
layerQueryRunner,
20+
makeClaudeProcessGroupSpawner,
21+
} from "./ClaudeAdapterV2.ts";
22+
23+
vi.mock("@anthropic-ai/claude-agent-sdk", { spy: true });
24+
25+
const spawnClaudeCodeProcessGroup = makeClaudeProcessGroupSpawner("darwin");
26+
27+
const layer = layerQueryRunner.pipe(
28+
Layer.provide(
29+
Layer.succeed(ProviderEventLoggers, {
30+
native: {
31+
filePath: "/tmp/events.log",
32+
write: () => Effect.void,
33+
close: () => Effect.void,
34+
},
35+
canonical: undefined,
36+
}),
37+
),
38+
Layer.provide(NodeServices.layer),
39+
) as Layer.Layer<ClaudeAgentSdkQueryRunner>;
40+
41+
const openOptions = {
42+
sessionId: "native-session-process-group",
43+
model: "claude-sonnet-4-6",
44+
tools: [],
45+
permissionMode: "default" as const,
46+
};
47+
48+
/** Waits until the predicate holds, sampling every 50 ms, or throws. */
49+
async function until(what: string, predicate: () => boolean, budgetMs = 5000): Promise<void> {
50+
const deadline = Date.now() + budgetMs;
51+
while (Date.now() < deadline) {
52+
if (predicate()) return;
53+
await new Promise((resolve) => setTimeout(resolve, 50));
54+
}
55+
throw new Error(`timed out waiting for ${what}`);
56+
}
57+
58+
const alive = (pid: number): boolean => {
59+
try {
60+
process.kill(pid, 0);
61+
return true;
62+
} catch {
63+
return false;
64+
}
65+
};
66+
67+
/** A stand-in CLI that forks a long-lived grandchild the way the Claude CLI
68+
* forks plugin MCP stdio servers, then waits on it. `trap` controls whether
69+
* the grandchild ignores SIGTERM like a plugin server wedged writing into a
70+
* stdout pipe nobody drains. */
71+
function fakeCli(dir: string, wedged: boolean): string {
72+
const cli = NodePath.join(dir, wedged ? "wedged-cli.sh" : "fake-cli.sh");
73+
const marker = NodePath.join(dir, "grandchild.pid");
74+
NodeFS.writeFileSync(
75+
cli,
76+
[
77+
"#!/bin/sh",
78+
`sh -c 'echo $$ > ${JSON.stringify(marker)}; ${wedged ? `trap "" TERM; ` : ""}exec sleep 300' &`,
79+
"wait",
80+
].join("\n"),
81+
);
82+
NodeFS.chmodSync(cli, 0o755);
83+
return cli;
84+
}
85+
86+
/** A stand-in CLI that forks the same grandchild and then exits on its own,
87+
* the way the transport's graceful close ends: stdin EOF, no kill involved. */
88+
function fakeCliThatExitsItself(dir: string): string {
89+
const cli = NodePath.join(dir, "self-exiting-cli.sh");
90+
const marker = NodePath.join(dir, "grandchild.pid");
91+
NodeFS.writeFileSync(
92+
cli,
93+
[
94+
"#!/bin/sh",
95+
`sh -c 'echo $$ > ${JSON.stringify(marker)}; exec sleep 300' &`,
96+
"sleep 0.3",
97+
"exit 0",
98+
].join("\n"),
99+
);
100+
NodeFS.chmodSync(cli, 0o755);
101+
return cli;
102+
}
103+
104+
async function grandchildPidOf(dir: string): Promise<number> {
105+
const marker = NodePath.join(dir, "grandchild.pid");
106+
await until("grandchild to record its pid", () => NodeFS.existsSync(marker));
107+
return Number(NodeFS.readFileSync(marker, "utf8").trim());
108+
}
109+
110+
const asChild = (child: ClaudeSdk.SpawnedProcess): NodeChildProcess.ChildProcess =>
111+
child as unknown as NodeChildProcess.ChildProcess;
112+
113+
describe("spawnClaudeCodeProcessGroup", () => {
114+
it("spawns the CLI detached, so it leads its own process group", () => {
115+
const controller = new AbortController();
116+
const child = asChild(
117+
spawnClaudeCodeProcessGroup({
118+
command: "/bin/sleep",
119+
args: ["5"],
120+
env: { ...process.env },
121+
signal: controller.signal,
122+
}),
123+
);
124+
try {
125+
expect(child.pid).toBeDefined();
126+
// A detached child changes process group; probing the group with
127+
// kill(-pid, 0) only succeeds when the child leads one.
128+
expect(() => process.kill(-child.pid!, 0)).not.toThrow();
129+
} finally {
130+
child.kill("SIGKILL");
131+
controller.abort();
132+
}
133+
});
134+
135+
it("reaps the whole CLI process group on kill, including plugin-server descendants", async () => {
136+
const dir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-claude-group-"));
137+
const controller = new AbortController();
138+
const child = asChild(
139+
spawnClaudeCodeProcessGroup({
140+
command: fakeCli(dir, false),
141+
args: [],
142+
env: { ...process.env },
143+
signal: controller.signal,
144+
}),
145+
);
146+
try {
147+
const grandchildPid = await grandchildPidOf(dir);
148+
expect(alive(grandchildPid)).toBe(true);
149+
150+
child.kill("SIGTERM");
151+
await until("grandchild to die with its CLI", () => !alive(grandchildPid));
152+
await until("cli to die", () => child.exitCode !== null || child.signalCode !== null);
153+
} finally {
154+
child.kill("SIGKILL");
155+
controller.abort();
156+
}
157+
});
158+
159+
it("reaps the whole CLI process group when the CLI exits by itself", async () => {
160+
const dir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-claude-graceful-"));
161+
const controller = new AbortController();
162+
const child = asChild(
163+
spawnClaudeCodeProcessGroup({
164+
command: fakeCliThatExitsItself(dir),
165+
args: [],
166+
env: { ...process.env },
167+
signal: controller.signal,
168+
}),
169+
);
170+
try {
171+
const grandchildPid = await grandchildPidOf(dir);
172+
expect(alive(grandchildPid)).toBe(true);
173+
174+
// The graceful close shape: the transport ends stdin and the CLI exits
175+
// on its own, no kill involved. The session-scoped plugin server that
176+
// outlives it in production is what wedges a core per closed session.
177+
await until("cli to exit on its own", () => child.exitCode !== null);
178+
await until("grandchild to die with its CLI", () => !alive(grandchildPid));
179+
} finally {
180+
child.kill("SIGKILL");
181+
controller.abort();
182+
}
183+
});
184+
185+
it("reaps a wedged descendant that ignores SIGTERM when the CLI exits", async () => {
186+
const dir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-claude-wedge-"));
187+
const controller = new AbortController();
188+
const child = asChild(
189+
spawnClaudeCodeProcessGroup({
190+
command: fakeCli(dir, true),
191+
args: [],
192+
env: { ...process.env },
193+
signal: controller.signal,
194+
}),
195+
);
196+
try {
197+
const grandchildPid = await grandchildPidOf(dir);
198+
199+
// The graceful close shape: the transport signals the CLI, which exits
200+
// on its own. The wedged plugin server ignores the SIGTERM — in
201+
// production it is stuck writing into a stdout pipe nobody drains, so
202+
// only the SIGKILL that follows its CLI's exit can stop it.
203+
child.kill("SIGTERM");
204+
await until("wedged grandchild to die with its CLI", () => !alive(grandchildPid));
205+
await until("cli to die", () => child.exitCode !== null || child.signalCode !== null);
206+
} finally {
207+
child.kill("SIGKILL");
208+
controller.abort();
209+
}
210+
});
211+
});
212+
213+
class ProcessGroupProbeError extends Data.TaggedError("ProcessGroupProbeError")<{
214+
readonly cause: unknown;
215+
}> {}
216+
217+
describe("query process-group wiring", () => {
218+
const probeSpawner = (
219+
spawn: ((options: ClaudeSdk.SpawnOptions) => ClaudeSdk.SpawnedProcess) | undefined,
220+
) =>
221+
Effect.tryPromise({
222+
try: async () => {
223+
// The spawner is a real ChildProcess factory: stdin/stdout pipes the
224+
// transport writes and reads, and a kill that reaches the whole group.
225+
const controller = new AbortController();
226+
const child = asChild(
227+
spawn!({
228+
command: "/bin/sleep",
229+
args: ["5"],
230+
env: { ...process.env },
231+
signal: controller.signal,
232+
}),
233+
);
234+
try {
235+
expect(child.pid).toBeDefined();
236+
expect(child.stdin!.writable).toBe(true);
237+
expect(child.stdout!.readable).toBe(true);
238+
child.kill("SIGTERM");
239+
await until(
240+
"spawned child to die",
241+
() => child.exitCode !== null || child.signalCode !== null || child.killed,
242+
);
243+
} finally {
244+
child.kill("SIGKILL");
245+
controller.abort();
246+
}
247+
},
248+
catch: (cause) => new ProcessGroupProbeError({ cause }),
249+
});
250+
251+
it.effect("hands the SDK a process-group spawn for the CLI", () =>
252+
Effect.gen(function* () {
253+
const query = vi.spyOn(ClaudeSdk, "query").mockReturnValue({
254+
setModel: async () => {},
255+
setPermissionMode: async () => {},
256+
stopTask: async () => {},
257+
interrupt: async () => undefined,
258+
close: () => {},
259+
} as unknown as ClaudeSdk.Query);
260+
yield* Effect.addFinalizer(() => Effect.sync(() => query.mockRestore()));
261+
const session = yield* Effect.gen(function* () {
262+
const runner = yield* ClaudeAgentSdkQueryRunner;
263+
return yield* runner.open({
264+
threadId: ThreadId.make("thread-process-group"),
265+
providerSessionId: ProviderSessionId.make("session-process-group"),
266+
options: openOptions,
267+
});
268+
}).pipe(Effect.provide(layer));
269+
yield* session.close;
270+
271+
expect(query.mock.calls.length).toBeGreaterThan(0);
272+
const captured = (query.mock.calls[0]?.[0] as { options?: ClaudeSdk.Options } | undefined)
273+
?.options;
274+
expect(typeof captured?.spawnClaudeCodeProcess).toBe("function");
275+
yield* probeSpawner(captured?.spawnClaudeCodeProcess);
276+
}),
277+
);
278+
});

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts‎

Lines changed: 81 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ import {
2828
type PermissionUpdate,
2929
type Query as ClaudeQuery,
3030
type Settings as ClaudeSdkSettings,
31+
type SpawnOptions as ClaudeSpawnOptions,
32+
type SpawnedProcess as ClaudeSpawnedProcess,
3133
type SDKAssistantMessage,
3234
type SDKAPIRetryMessage,
3335
type SDKMessage,
@@ -39,6 +41,8 @@ import type {
3941
AskUserQuestionInput,
4042
WebSearchOutput,
4143
} from "@anthropic-ai/claude-agent-sdk/sdk-tools";
44+
// @effect-diagnostics-next-line nodeBuiltinImport:off -- the CLI's process group is owned with raw POSIX signals.
45+
import * as NodeChildProcess from "node:child_process";
4246
import { parseCliArgs } from "@t3tools/shared/cliArgs";
4347
import { AgentScope } from "@t3tools/shared/AgentScope";
4448
import * as HostProcess from "@t3tools/shared/HostProcess";
@@ -225,6 +229,69 @@ const CLAUDE_SCRIPT_EXTENSIONS = [".js", ".mjs", ".tsx", ".ts", ".jsx"];
225229
// mcp_set_servers once each server connected or hit its own 30 s timeout.
226230
const CLAUDE_MCP_REGISTRATION_TIMEOUT = "90 seconds";
227231

232+
/**
233+
* Spawns the Claude CLI in its own POSIX process group and turns every kill
234+
* into a group kill. The CLI runs MCP plugin servers (each plugin's `bun
235+
* server.ts`) under throwaway `bun run` wrappers, and on close the SDK
236+
* transport signals only the CLI PID: the plugin servers survive the session
237+
* as orphans and keep logging into a stdout pipe nobody drains, burning CPU
238+
* on it (one pinned core per wedged server). A server wedged in that write
239+
* never reaches its SIGTERM handler, so only SIGKILL can stop it. The whole
240+
* tree stays in the CLI's process group, so a group signal reaps it all:
241+
* SIGTERM in the transport's graceful stage, SIGKILL in its escalation.
242+
*/
243+
export function makeClaudeProcessGroupSpawner(
244+
platform: NodeJS.Platform,
245+
): (options: ClaudeSpawnOptions) => ClaudeSpawnedProcess {
246+
// Windows has no POSIX process groups; there the spawn stays as the SDK's
247+
// own and TerminateProcess-style kills keep their native semantics.
248+
const posix = platform !== "win32";
249+
return function spawnClaudeCodeProcessGroup(options: ClaudeSpawnOptions): ClaudeSpawnedProcess {
250+
const child = NodeChildProcess.spawn(options.command, options.args, {
251+
...(options.cwd === undefined ? {} : { cwd: options.cwd }),
252+
env: options.env,
253+
signal: options.signal,
254+
stdio: ["pipe", "pipe", "pipe"],
255+
// Own process group: the precondition for the group kill below. Without
256+
// it, a group signal would hit the server's own group instead.
257+
detached: posix,
258+
windowsHide: true,
259+
});
260+
// The transport only wires stdin and stdout for custom spawns; drain
261+
// stderr so a chatty CLI cannot wedge on a full pipe either.
262+
child.stderr?.on("data", () => {});
263+
const pid = child.pid;
264+
if (posix && pid !== undefined && pid > 0) {
265+
// The CLI can also leave the group without a kill: the transport's
266+
// graceful close ends stdin and the CLI exits by itself. Whatever it
267+
// left running (a plugin server it failed to reap) dies with the group.
268+
child.on("exit", () => {
269+
try {
270+
process.kill(-pid, "SIGKILL");
271+
} catch {}
272+
});
273+
const killChild = child.kill.bind(child);
274+
child.kill = ((signal: NodeJS.Signals | number = "SIGTERM") => {
275+
// Signal the CLI through Node first, so its killed/exit bookkeeping
276+
// stays exactly as the transport expects it.
277+
let notified: boolean;
278+
try {
279+
notified = killChild(signal);
280+
} catch {
281+
return false;
282+
}
283+
// Then take down the process group the CLI leads: the plugin servers
284+
// it spawned. They may already be gone; that is fine.
285+
try {
286+
process.kill(-pid, signal);
287+
} catch {}
288+
return notified;
289+
}) as typeof child.kill;
290+
}
291+
return child;
292+
};
293+
}
294+
228295
export const ClaudeProviderCapabilitiesV2 = {
229296
sessions: {
230297
supportsMultipleProviderThreadsPerSession: false,
@@ -659,6 +726,8 @@ export const layerQueryRunner: Layer.Layer<
659726
ClaudeAgentSdkQueryRunner,
660727
Effect.gen(function* () {
661728
const crypto = yield* Crypto.Crypto;
729+
const platform = yield* HostProcess.Platform;
730+
const claudeProcessGroupSpawner = makeClaudeProcessGroupSpawner(platform);
662731
const { native: nativeEventLogger } = yield* ProviderEventLoggers.ProviderEventLoggers;
663732
const agentScope = yield* AgentScope;
664733

@@ -716,7 +785,18 @@ export const layerQueryRunner: Layer.Layer<
716785
// before the session is handed out fails or is interrupted.
717786
const queryRuntime = yield* Effect.uninterruptibleMask((restore) =>
718787
Effect.try({
719-
try: () => query({ prompt, options }),
788+
try: () =>
789+
query({
790+
prompt,
791+
// A caller-supplied spawner keeps ownership of its process
792+
// (remote runtimes, tests); the default takes the process
793+
// group the close path can then reap as a whole.
794+
options: {
795+
...options,
796+
spawnClaudeCodeProcess:
797+
options.spawnClaudeCodeProcess ?? claudeProcessGroupSpawner,
798+
},
799+
}),
720800
catch: (cause) => queryRunnerError(cause, "query"),
721801
}).pipe(
722802
Effect.tap((queryRuntime) =>

0 commit comments

Comments
 (0)