Skip to content

Commit 1aa7179

Browse files
josephv123juliusmarminget3dotggclaude
authored
fix(server): agent-browser and Chrome no longer outlive the agent that started them (pingdotgg#21299)
Co-authored-by: Julius Marminge <[email protected]> Co-authored-by: Theo Browne <[email protected]> Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 7d6b5be commit 1aa7179

16 files changed

Lines changed: 642 additions & 37 deletions

File tree

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -323,14 +323,18 @@ const CLAUDE_GROUP_WATCH_SCRIPT = [
323323
* When the CLI exits it stops the MCP servers it started, but not their children: a plugin's
324324
* `bun run start` wrapper dies and its `bun server.ts` keeps running, writing to a pipe nobody
325325
* reads (#3089). Those stay in the CLI's group, so the rest of the group gets SIGKILL once the
326-
* CLI exits for any reason.
326+
* CLI exits for any reason. `onExit` then stops the CLI's agent scope, for daemons a tool
327+
* detached from the group with setsid.
327328
*
328329
* That cleanup needs this server alive, and the CLI finishes its turn after its pipes close. A
329330
* server that crashes or is killed would leave the CLI working next to the session it resumes
330331
* (#23370), so a watcher in the group reads a pipe only this server holds and stops the group at
331332
* its end-of-file.
332333
*/
333-
export function spawnClaudeProcessGroup(options: ClaudeSpawnOptions): ClaudeSpawnedProcess {
334+
export function spawnClaudeProcessGroup(
335+
options: ClaudeSpawnOptions,
336+
onExit?: () => void,
337+
): ClaudeSpawnedProcess {
334338
const child = NodeChildProcess.spawn(
335339
"/bin/sh",
336340
["-c", CLAUDE_GROUP_WATCH_SCRIPT, "t3-claude", options.command, ...options.args],
@@ -346,6 +350,7 @@ export function spawnClaudeProcessGroup(options: ClaudeSpawnOptions): ClaudeSpaw
346350
try {
347351
signalProcessGroup(child.pid ?? 0, "SIGKILL");
348352
} catch {}
353+
onExit?.();
349354
});
350355
// Node's typings make stdin and stdout nullable once stdio has a fourth entry; both are pipes.
351356
return child as ClaudeSpawnedProcess;
@@ -892,6 +897,7 @@ export const layerQueryRunner: Layer.Layer<
892897
const { native: nativeEventLogger } = yield* ProviderEventLoggers.ProviderEventLoggers;
893898
const agentScope = yield* AgentScope;
894899
const platform = yield* HostProcess.Platform;
900+
const runFork = Effect.runForkWith(yield* Effect.context<never>());
895901

896902
return ClaudeAgentSdkQueryRunner.of({
897903
allocateSessionId: crypto
@@ -942,10 +948,16 @@ export const layerQueryRunner: Layer.Layer<
942948
pathToClaudeCodeExecutable: launch.command,
943949
executableArgs: [...launch.args, ...(queryOptions.executableArgs ?? [])],
944950
};
951+
const unit = launch?.unit;
952+
const stopScope =
953+
unit === undefined ? undefined : () => void runFork(agentScope.stop(unit));
945954
// Windows has no process groups, so there the SDK spawns the CLI itself.
946955
const spawnClaudeCodeProcess =
947956
options.spawnClaudeCodeProcess ??
948-
(platform === "win32" ? undefined : spawnClaudeProcessGroup);
957+
(platform === "win32"
958+
? undefined
959+
: (spawnOptions: ClaudeSpawnOptions) =>
960+
spawnClaudeProcessGroup(spawnOptions, stopScope));
949961
const queryRuntime = yield* Effect.uninterruptibleMask((restore) =>
950962
Effect.try({
951963
try: () =>

‎apps/server/src/orchestration-v2/Adapters/ClaudeAgentSdkQueryRunner.test.ts‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,14 @@ import * as NodeServices from "@effect/platform-node/NodeServices";
88
import { assert, describe, it } from "@effect/vitest";
99
import { ProviderSessionId, ThreadId } from "@t3tools/contracts";
1010
import * as Effect from "effect/Effect";
11+
import * as Deferred from "effect/Deferred";
1112
import * as FileSystem from "effect/FileSystem";
1213
import * as Layer from "effect/Layer";
1314
import * as Path from "effect/Path";
1415
import { vi } from "vite-plus/test";
1516

1617
import * as HostProcess from "@t3tools/shared/HostProcess";
18+
import { AgentScope } from "@t3tools/shared/AgentScope";
1719
import { ProviderEventLoggers } from "@t3tools/provider-core/server/ProviderEventLoggers";
1820
import { signalProcessGroup } from "@t3tools/provider-core/server/processGroup";
1921
import {
@@ -162,11 +164,26 @@ describe.skipIf(HostProcess.Platform.defaultValue() === "win32")("Claude CLI pro
162164
const leftoverStopped = new Promise<void>((resolve) => alive.once("close", () => resolve()));
163165
alive.resume();
164166

165-
const session = yield* openFakeCli(cli);
167+
const stopped = yield* Deferred.make<string>();
168+
const session = yield* openFakeCli(cli).pipe(
169+
Effect.provide(
170+
realQueryRunner.pipe(
171+
Layer.provide(
172+
Layer.succeed(AgentScope, {
173+
...AgentScope.defaultValue(),
174+
wrap: ({ command, args }) =>
175+
Effect.succeed({ command, args, unit: "t3code-claude-test.scope" }),
176+
stop: (unit) => Deferred.succeed(stopped, unit).pipe(Effect.asVoid),
177+
}),
178+
),
179+
),
180+
),
181+
);
166182
yield* Effect.promise(() => leftoverStarted);
167183
yield* session.close;
168184
yield* Effect.promise(() => leftoverStopped);
169-
}).pipe(Effect.provide(realQueryRunner)),
185+
assert.strictEqual(yield* Deferred.await(stopped), "t3code-claude-test.scope");
186+
}),
170187
);
171188

172189
// The CLI keeps running its turn after its pipes close, so a server that

‎apps/server/src/process/agentScope.test.ts‎

Lines changed: 144 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,9 @@ import { assert, describe, expect, it } from "@effect/vitest";
88
import { AgentScope } from "@t3tools/shared/AgentScope";
99
import * as HostProcess from "@t3tools/shared/HostProcess";
1010
import * as Effect from "effect/Effect";
11+
import * as FileSystem from "effect/FileSystem";
1112
import * as Layer from "effect/Layer";
13+
import * as Logger from "effect/Logger";
1214
import { ChildProcess, ChildProcessSpawner } from "effect/process";
1315

1416
import * as ServerConfig from "../config.ts";
@@ -35,21 +37,33 @@ describe("classifyScope", () => {
3537
});
3638
});
3739

40+
interface FakeUnit {
41+
activeState: string;
42+
result: string;
43+
controlGroup?: string;
44+
}
45+
3846
const withFakeSystemd = <A, E>(
3947
body: (input: {
4048
readonly scope: Effect.Success<typeof make>;
41-
readonly units: Map<string, { activeState: string; result: string }>;
49+
readonly units: Map<string, FakeUnit>;
4250
readonly calls: ReadonlyArray<ReadonlyArray<string>>;
4351
readonly stateDir: string;
52+
readonly cgroupRoot: string;
4453
}) => Effect.Effect<A, E>,
45-
{ oomPolicy = true }: { readonly oomPolicy?: boolean } = {},
54+
// A user manager older than systemd 253 rejects OOMPolicy on scopes.
55+
{
56+
oomPolicy = true,
57+
stopFails = false,
58+
}: { readonly oomPolicy?: boolean; readonly stopFails?: boolean } = {},
4659
) =>
4760
Effect.gen(function* () {
4861
const bin = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-agent-scope-"));
4962
for (const name of ["systemd-run", "systemctl", "agent"]) {
5063
NodeFS.writeFileSync(NodePath.join(bin, name), "", { mode: 0o755 });
5164
}
52-
const units = new Map<string, { activeState: string; result: string }>();
65+
const cgroupRoot = NodePath.join(bin, "cgroup");
66+
const units = new Map<string, FakeUnit>();
5367
const calls: Array<ReadonlyArray<string>> = [];
5468
const output = (stdout: string, code = 0): ProcessRunner.ProcessRunOutput => ({
5569
stdout,
@@ -69,23 +83,37 @@ const withFakeSystemd = <A, E>(
6983
return output("", 1);
7084
}
7185
const [, verb, unit = ""] = args;
86+
if (stopFails && verb === "stop") {
87+
return {
88+
...output(""),
89+
stderr: `Failed to stop ${args.at(-1)}: Unit not loaded.`,
90+
code: ChildProcessSpawner.ExitCode(5),
91+
};
92+
}
7293
if (verb === "reset-failed") units.delete(unit);
7394
if (verb !== "show") return output("");
7495
const state = units.get(unit);
7596
return output(
7697
state === undefined
7798
? "LoadState=not-found\nActiveState=inactive\nResult=success\n"
78-
: `LoadState=loaded\nActiveState=${state.activeState}\nResult=${state.result}\n`,
99+
: `LoadState=loaded\nActiveState=${state.activeState}\nResult=${state.result}\n` +
100+
`ControlGroup=${state.controlGroup ?? ""}\n`,
79101
);
80102
}),
81103
});
104+
const fileSystem = yield* FileSystem.FileSystem;
82105
const scope = yield* make.pipe(
83106
Effect.provideService(ProcessRunner.ProcessRunner, runner),
84107
Effect.provideService(HostProcess.Platform, "linux"),
85108
Effect.provideService(HostProcess.Environment, { PATH: bin, XDG_RUNTIME_DIR: "/run/user/1" }),
109+
Effect.provideService(FileSystem.FileSystem, {
110+
...fileSystem,
111+
readFileString: (path, encoding) =>
112+
fileSystem.readFileString(path.replace("/sys/fs/cgroup", cgroupRoot), encoding),
113+
}),
86114
);
87115
const { stateDir } = yield* ServerConfig.ServerConfig;
88-
return yield* body({ scope, units, calls, stateDir }).pipe(
116+
return yield* body({ scope, units, calls, stateDir, cgroupRoot }).pipe(
89117
Effect.ensuring(Effect.sync(() => NodeFS.rmSync(bin, { recursive: true, force: true }))),
90118
);
91119
}).pipe(
@@ -103,6 +131,31 @@ const unitOf = (launch: { readonly args: ReadonlyArray<string> }) =>
103131
const sliceOf = (launch: { readonly args: ReadonlyArray<string> }) =>
104132
launch.args.find((arg) => arg.startsWith("--slice="))?.slice("--slice=".length) ?? "";
105133

134+
/** An active scope as systemd and the kernel report it, with the agent itself gone. */
135+
const exitedAgent = (
136+
units: Map<string, FakeUnit>,
137+
cgroupRoot: string,
138+
unit: string,
139+
{ populated, oomKills }: { readonly populated: boolean; readonly oomKills: number | undefined },
140+
) => {
141+
const controlGroup = `/${unit}`;
142+
NodeFS.mkdirSync(NodePath.join(cgroupRoot, controlGroup), { recursive: true });
143+
NodeFS.writeFileSync(
144+
NodePath.join(cgroupRoot, controlGroup, "cgroup.events"),
145+
`populated ${populated ? 1 : 0}\nfrozen 0\n`,
146+
);
147+
// An undefined count leaves memory.events unreadable.
148+
if (oomKills !== undefined) {
149+
NodeFS.writeFileSync(
150+
NodePath.join(cgroupRoot, controlGroup, "memory.events"),
151+
`low 0\nhigh 0\nmax 0\noom ${oomKills}\noom_kill ${oomKills}\noom_group_kill 0\n`,
152+
);
153+
}
154+
units.set(unit, { activeState: "active", result: "success", controlGroup });
155+
};
156+
157+
const stopsOf = (calls: ReadonlyArray<ReadonlyArray<string>>) =>
158+
calls.filter(([, verb]) => verb === "stop");
106159
const launchCodex = (scope: Effect.Success<typeof make>, threadId: string) =>
107160
makeCodexAppServerSpawnCommand({ command: "agent", args: ["app-server"], threadId }).pipe(
108161
Effect.provideService(AgentScope, scope),
@@ -194,4 +247,90 @@ describe("AgentScope service", () => {
194247
}),
195248
),
196249
);
250+
251+
it.effect("stops leftover processes only in an active agent scope", () =>
252+
withFakeSystemd(({ scope, units, calls, cgroupRoot }) =>
253+
Effect.gen(function* () {
254+
const agent = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
255+
const unit = unitOf(agent);
256+
expect(agent.unit).toBe(unit);
257+
258+
yield* scope.stop("session-2.scope");
259+
expect(calls.some((args) => args.includes("session-2.scope"))).toBe(false);
260+
yield* scope.stop(unit);
261+
exitedAgent(units, cgroupRoot, unit, { populated: false, oomKills: 0 });
262+
yield* scope.stop(unit);
263+
exitedAgent(units, cgroupRoot, unit, { populated: true, oomKills: 0 });
264+
units.get(unit)!.activeState = "deactivating";
265+
yield* scope.stop(unit);
266+
expect(stopsOf(calls)).toEqual([]);
267+
268+
units.get(unit)!.activeState = "active";
269+
yield* scope.stop(unit);
270+
expect(stopsOf(calls)).toEqual([["--user", "stop", "--no-block", unit]]);
271+
expect(yield* scope.oomKilled("a")).toBe(false);
272+
}),
273+
),
274+
);
275+
276+
it.effect.each([true, false])(
277+
"cleans up an OOM exit without losing its diagnosis (OOMPolicy=%s)",
278+
(oomPolicy) =>
279+
withFakeSystemd(
280+
({ scope, units, calls, cgroupRoot }) =>
281+
Effect.gen(function* () {
282+
const agent = yield* scope.wrap({
283+
command: "agent",
284+
args: [],
285+
name: "t",
286+
threadId: "a",
287+
});
288+
const unit = unitOf(agent);
289+
exitedAgent(units, cgroupRoot, unit, { populated: true, oomKills: undefined });
290+
yield* scope.stop(unit);
291+
expect(stopsOf(calls)).toEqual(
292+
oomPolicy ? [] : [["--user", "stop", "--no-block", unit]],
293+
);
294+
295+
// The kernel records the kill before systemd records the OOM result.
296+
exitedAgent(units, cgroupRoot, unit, { populated: true, oomKills: 1 });
297+
yield* scope.stop(unit);
298+
expect(stopsOf(calls)).toHaveLength(oomPolicy ? 0 : 2);
299+
300+
if (oomPolicy) {
301+
units.set(unit, { activeState: "failed", result: "oom-kill" });
302+
expect(yield* scope.oomKilled("a")).toBe(true);
303+
units.delete(unit);
304+
expect(yield* scope.oomKilled("a")).toBe(true);
305+
}
306+
}),
307+
{ oomPolicy },
308+
),
309+
);
310+
311+
it.effect("reports failed daemon cleanup without failing session teardown", () => {
312+
const warnings: Array<unknown> = [];
313+
const logger = Logger.make(({ message }) => {
314+
warnings.push(message);
315+
});
316+
return withFakeSystemd(
317+
({ scope, units, cgroupRoot }) =>
318+
Effect.gen(function* () {
319+
const agent = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
320+
exitedAgent(units, cgroupRoot, unitOf(agent), { populated: true, oomKills: 0 });
321+
yield* scope.stop(unitOf(agent));
322+
expect(warnings.flat()).toContain("Could not stop an exited agent's scope");
323+
}).pipe(Effect.provide(Logger.layer([logger], { mergeWithExisting: false }))),
324+
{ stopFails: true },
325+
);
326+
});
327+
328+
it.effect("leaves a quiet slice alone", () =>
329+
withFakeSystemd(({ scope, calls }) =>
330+
Effect.gen(function* () {
331+
yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
332+
expect(calls.filter(([, verb]) => verb === "stop")).toEqual([]);
333+
}),
334+
),
335+
);
197336
});

‎apps/server/src/process/agentScope.ts‎

Lines changed: 35 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ export interface ScopeState {
4444
readonly loadState: string;
4545
readonly activeState: string;
4646
readonly result: string;
47-
readonly oomKills: number;
47+
readonly oomKills: number | undefined;
4848
readonly populated: boolean | undefined;
4949
}
5050

@@ -62,7 +62,11 @@ export function classifyScope(state: ScopeState): "oom-killed" | "gone" | "runni
6262
) {
6363
return "gone";
6464
}
65-
if (state.activeState === "deactivating" || state.oomKills > 0 || state.populated === false) {
65+
if (
66+
state.activeState === "deactivating" ||
67+
(state.oomKills ?? 0) > 0 ||
68+
state.populated === false
69+
) {
6670
return "stopping";
6771
}
6872
return "running";
@@ -220,7 +224,7 @@ export const make = Effect.gen(function* () {
220224
loadState: values.get("LoadState") ?? "",
221225
activeState: values.get("ActiveState") ?? "",
222226
result: values.get("Result") ?? "",
223-
oomKills: parseCgroupCounter(memoryEvents, "oom_kill") ?? 0,
227+
oomKills: parseCgroupCounter(memoryEvents, "oom_kill"),
224228
populated: populated === undefined ? undefined : populated === 1,
225229
} satisfies ScopeState;
226230
});
@@ -300,6 +304,32 @@ export const make = Effect.gen(function* () {
300304
},
301305
);
302306

307+
const stop = Effect.fn("AgentScope.stop")(function* (unit: string) {
308+
if (!unit.startsWith("t3code-") || !unit.endsWith(".scope")) return;
309+
const scope = yield* systemd;
310+
if (scope === undefined) return;
311+
const state = yield* readScope(scope.systemctl, unit);
312+
if (
313+
state === undefined ||
314+
state.loadState !== "loaded" ||
315+
state.activeState !== "active" ||
316+
state.result === "oom-kill" ||
317+
state.populated === false
318+
)
319+
return;
320+
// The kernel counts OOM kills before signaling the victim. Let OOMPolicy
321+
// record the result before a manual stop can overwrite it; an unreadable
322+
// counter cannot rule out that race. Older managers need us to stop it.
323+
if (scope.supportsOomPolicy && (state.oomKills === undefined || state.oomKills > 0)) return;
324+
const stopped = yield* run(scope.systemctl, ["--user", "stop", "--no-block", unit]);
325+
if (stopped._tag === "None" || stopped.value.code !== 0) {
326+
yield* Effect.logWarning("Could not stop an exited agent's scope", {
327+
unit,
328+
exitCode: stopped._tag === "Some" ? stopped.value.code : undefined,
329+
});
330+
}
331+
});
332+
303333
const clear: AgentScopeShape["clear"] = Effect.fn("agentScope.clear")(function* (threadId) {
304334
if (!scopeByThread.has(threadId)) return;
305335
const scope = yield* systemd;
@@ -341,9 +371,10 @@ export const make = Effect.gen(function* () {
341371
...(owned ? [] : ["--collect"]),
342372
],
343373
};
344-
return agentScopeCommand({ command: resolved, args, scope: systemdScope });
374+
return { ...agentScopeCommand({ command: resolved, args, scope: systemdScope }), unit };
345375
}),
346376
oomKilled,
377+
stop,
347378
clear,
348379
} satisfies AgentScopeShape;
349380
});

0 commit comments

Comments
 (0)