Skip to content

Commit 77b732f

Browse files
committed
fix(server): agent-browser and Chrome no longer outlive the agent that started them
A systemd scope lives while any process is in it. A daemon a tool detached with setsid (the agent-browser daemon and its Chrome) leaves the agent's process group, so the group kill on exit misses it and the agent's scope keeps running after the agent is gone. Claude, Pi, Muse and ACP agents now stop their scope once the agent has exited. A scope the kernel has already counted an OOM kill in, or that systemd is already stopping, is left to systemd, so the thread keeps its "Killed: out of memory" label.
1 parent e2cc34f commit 77b732f

11 files changed

Lines changed: 250 additions & 21 deletions

File tree

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -277,9 +277,13 @@ const WORKFLOW_MEMBER_TRANSCRIPT_FLUSH_MS = 500;
277277
* When the CLI exits it stops the MCP servers it started, but not their children: a plugin's
278278
* `bun run start` wrapper dies and its `bun server.ts` keeps running, writing to a pipe nobody
279279
* reads (#3089). Those stay in the CLI's group, so the rest of the group gets SIGKILL once the
280-
* CLI exits for any reason.
280+
* CLI exits for any reason. `onExit` then stops the CLI's agent scope, for daemons a tool
281+
* detached from the group.
281282
*/
282-
function spawnClaudeProcessGroup(options: ClaudeSpawnOptions): ClaudeSpawnedProcess {
283+
function spawnClaudeProcessGroup(
284+
options: ClaudeSpawnOptions,
285+
onExit?: () => void,
286+
): ClaudeSpawnedProcess {
283287
const child = NodeChildProcess.spawn(options.command, options.args, {
284288
cwd: options.cwd,
285289
env: options.env,
@@ -291,6 +295,7 @@ function spawnClaudeProcessGroup(options: ClaudeSpawnOptions): ClaudeSpawnedProc
291295
try {
292296
signalProcessGroup(child.pid ?? 0, "SIGKILL");
293297
} catch {}
298+
onExit?.();
294299
});
295300
return child;
296301
}
@@ -751,6 +756,7 @@ export const layerQueryRunner: Layer.Layer<
751756
const { native: nativeEventLogger } = yield* ProviderEventLoggers.ProviderEventLoggers;
752757
const agentScope = yield* AgentScope;
753758
const platform = yield* HostProcess.Platform;
759+
const runFork = Effect.runForkWith(yield* Effect.context<never>());
754760

755761
return ClaudeAgentSdkQueryRunner.of({
756762
allocateSessionId: crypto
@@ -801,10 +807,16 @@ export const layerQueryRunner: Layer.Layer<
801807
pathToClaudeCodeExecutable: launch.command,
802808
executableArgs: [...launch.args, ...(queryOptions.executableArgs ?? [])],
803809
};
810+
const unit = launch?.unit;
811+
const stopScope =
812+
unit === undefined ? undefined : () => void runFork(agentScope.stop(unit));
804813
// Windows has no process groups, so there the SDK spawns the CLI itself.
805814
const spawnClaudeCodeProcess =
806815
options.spawnClaudeCodeProcess ??
807-
(platform === "win32" ? undefined : spawnClaudeProcessGroup);
816+
(platform === "win32"
817+
? undefined
818+
: (spawnOptions: ClaudeSpawnOptions) =>
819+
spawnClaudeProcessGroup(spawnOptions, stopScope));
808820
const queryRuntime = yield* Effect.uninterruptibleMask((restore) =>
809821
Effect.try({
810822
try: () =>

‎apps/server/src/orchestration-v2/Adapters/ClaudeAgentSdkQueryRunner.test.ts‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,14 @@ import * as NodePath from "node:path";
77
import * as NodeServices from "@effect/platform-node/NodeServices";
88
import { assert, describe, it } from "@effect/vitest";
99
import { ProviderSessionId, ThreadId } from "@t3tools/contracts";
10+
import * as Deferred from "effect/Deferred";
1011
import * as Effect from "effect/Effect";
1112
import * as Exit from "effect/Exit";
1213
import * as Layer from "effect/Layer";
1314
import { vi } from "vite-plus/test";
1415

1516
import { ProviderEventLoggers } from "@t3tools/provider-core/server/ProviderEventLoggers";
17+
import { AgentScope } from "@t3tools/shared/AgentScope";
1618
import * as HostProcess from "@t3tools/shared/HostProcess";
1719
import {
1820
CLAUDE_AGENT_SDK_QUERY_PROTOCOL,
@@ -153,4 +155,33 @@ describe.skipIf(HostProcess.Platform.defaultValue() === "win32")("Claude CLI pro
153155
yield* Effect.promise(() => leftoverStopped);
154156
}).pipe(Effect.provide(realQueryRunner)),
155157
);
158+
159+
// A daemon a tool detached with setsid (agent-browser and its Chrome) leaves
160+
// the CLI's process group but stays in its systemd scope.
161+
it.effect("stops the CLI's scope once the CLI exits", () =>
162+
Effect.gen(function* () {
163+
const cli = fakeClaudeCli("exit 0");
164+
yield* Effect.addFinalizer(() =>
165+
Effect.sync(() => NodeFS.rmSync(NodePath.dirname(cli), { recursive: true, force: true })),
166+
);
167+
const stopped = yield* Deferred.make<string>();
168+
const runner = layerQueryRunner.pipe(
169+
Layer.provide(
170+
Layer.succeed(AgentScope, {
171+
...AgentScope.defaultValue(),
172+
wrap: ({ command, args }) =>
173+
Effect.succeed({ command, args, unit: "t3code-claude-test.scope" }),
174+
stop: (unit) => Deferred.succeed(stopped, unit).pipe(Effect.asVoid),
175+
}),
176+
),
177+
Layer.provide(
178+
Layer.succeed(ProviderEventLoggers, { native: undefined, canonical: undefined }),
179+
),
180+
Layer.provide(NodeServices.layer),
181+
);
182+
const session = yield* openFakeCli(cli).pipe(Effect.provide(runner));
183+
assert.strictEqual(yield* Deferred.await(stopped), "t3code-claude-test.scope");
184+
yield* session.close.pipe(Effect.ignore);
185+
}),
186+
);
156187
});

‎apps/server/src/process/agentScope.test.ts‎

Lines changed: 78 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import * as ServerConfig from "../config.ts";
1414
import * as ProcessRunner from "../processRunner.ts";
1515
import {
1616
agentSliceMemoryLimits,
17+
CgroupRoot,
1718
classifyScope,
1819
make,
1920
ownedAgentSlice,
@@ -61,20 +62,28 @@ describe("agentSliceMemoryLimits", () => {
6162
});
6263
});
6364

65+
interface FakeUnit {
66+
activeState: string;
67+
result: string;
68+
controlGroup?: string;
69+
}
70+
6471
const withFakeSystemd = <A, E>(
6572
body: (input: {
6673
readonly scope: Effect.Success<typeof make>;
67-
readonly units: Map<string, { activeState: string; result: string }>;
74+
readonly units: Map<string, FakeUnit>;
6875
readonly calls: ReadonlyArray<ReadonlyArray<string>>;
6976
readonly stateDir: string;
77+
readonly cgroupRoot: string;
7078
}) => Effect.Effect<A, E>,
7179
) =>
7280
Effect.gen(function* () {
7381
const bin = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-agent-scope-"));
7482
for (const name of ["systemd-run", "systemctl", "agent"]) {
7583
NodeFS.writeFileSync(NodePath.join(bin, name), "", { mode: 0o755 });
7684
}
77-
const units = new Map<string, { activeState: string; result: string }>();
85+
const cgroupRoot = NodePath.join(bin, "cgroup");
86+
const units = new Map<string, FakeUnit>();
7887
const calls: Array<ReadonlyArray<string>> = [];
7988
const output = (stdout: string): ProcessRunner.ProcessRunOutput => ({
8089
stdout,
@@ -97,17 +106,19 @@ const withFakeSystemd = <A, E>(
97106
return output(
98107
state === undefined
99108
? "LoadState=not-found\nActiveState=inactive\nResult=success\n"
100-
: `LoadState=loaded\nActiveState=${state.activeState}\nResult=${state.result}\n`,
109+
: `LoadState=loaded\nActiveState=${state.activeState}\nResult=${state.result}\n` +
110+
`ControlGroup=${state.controlGroup ?? ""}\n`,
101111
);
102112
}),
103113
});
104114
const scope = yield* make.pipe(
105115
Effect.provideService(ProcessRunner.ProcessRunner, runner),
106116
Effect.provideService(HostProcess.Platform, "linux"),
107117
Effect.provideService(HostProcess.Environment, { PATH: bin, XDG_RUNTIME_DIR: "/run/user/1" }),
118+
Effect.provideService(CgroupRoot, cgroupRoot),
108119
);
109120
const { stateDir } = yield* ServerConfig.ServerConfig;
110-
return yield* body({ scope, units, calls, stateDir }).pipe(
121+
return yield* body({ scope, units, calls, stateDir, cgroupRoot }).pipe(
111122
Effect.ensuring(Effect.sync(() => NodeFS.rmSync(bin, { recursive: true, force: true }))),
112123
);
113124
}).pipe(
@@ -125,6 +136,29 @@ const unitOf = (launch: { readonly args: ReadonlyArray<string> }) =>
125136
const sliceOf = (launch: { readonly args: ReadonlyArray<string> }) =>
126137
launch.args.find((arg) => arg.startsWith("--slice="))?.slice("--slice=".length) ?? "";
127138

139+
/** An active scope as systemd and the kernel report it, with the agent itself gone. */
140+
const exitedAgent = (
141+
units: Map<string, FakeUnit>,
142+
cgroupRoot: string,
143+
unit: string,
144+
{ populated, oomKills }: { readonly populated: boolean; readonly oomKills: number },
145+
) => {
146+
const controlGroup = `/${unit}`;
147+
NodeFS.mkdirSync(NodePath.join(cgroupRoot, controlGroup), { recursive: true });
148+
NodeFS.writeFileSync(
149+
NodePath.join(cgroupRoot, controlGroup, "cgroup.events"),
150+
`populated ${populated ? 1 : 0}\nfrozen 0\n`,
151+
);
152+
NodeFS.writeFileSync(
153+
NodePath.join(cgroupRoot, controlGroup, "memory.events"),
154+
`low 0\nhigh 0\nmax 0\noom ${oomKills}\noom_kill ${oomKills}\noom_group_kill 0\n`,
155+
);
156+
units.set(unit, { activeState: "active", result: "success", controlGroup });
157+
};
158+
159+
const stopsOf = (calls: ReadonlyArray<ReadonlyArray<string>>) =>
160+
calls.filter(([, verb]) => verb === "stop");
161+
128162
describe("AgentScope service", () => {
129163
it.effect("labels a thread from its newest scope until its next session clears it", () =>
130164
withFakeSystemd(({ scope, units }) =>
@@ -181,6 +215,46 @@ describe("AgentScope service", () => {
181215
),
182216
);
183217

218+
it.effect("stops an exited agent's scope with the daemons it left running", () =>
219+
withFakeSystemd(({ scope, units, calls, cgroupRoot }) =>
220+
Effect.gen(function* () {
221+
const agent = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
222+
expect(agent.unit).toBe(unitOf(agent));
223+
exitedAgent(units, cgroupRoot, unitOf(agent), { populated: true, oomKills: 0 });
224+
yield* scope.stop(unitOf(agent));
225+
expect(stopsOf(calls)).toEqual([["--user", "stop", "--no-block", unitOf(agent)]]);
226+
}),
227+
),
228+
);
229+
230+
it.effect("leaves a scope the kernel OOM-killed to systemd, so the thread keeps its label", () =>
231+
withFakeSystemd(({ scope, units, calls, cgroupRoot }) =>
232+
Effect.gen(function* () {
233+
const agent = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
234+
// The kernel has counted the kill; systemd has not handled it yet.
235+
exitedAgent(units, cgroupRoot, unitOf(agent), { populated: true, oomKills: 1 });
236+
yield* scope.stop(unitOf(agent));
237+
expect(stopsOf(calls)).toEqual([]);
238+
239+
units.set(unitOf(agent), { activeState: "failed", result: "oom-kill" });
240+
expect(yield* scope.oomKilled("a")).toBe(true);
241+
}),
242+
),
243+
);
244+
245+
it.effect("does not stop an empty scope or a unit it did not start", () =>
246+
withFakeSystemd(({ scope, units, calls, cgroupRoot }) =>
247+
Effect.gen(function* () {
248+
const agent = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
249+
exitedAgent(units, cgroupRoot, unitOf(agent), { populated: false, oomKills: 0 });
250+
yield* scope.stop(unitOf(agent));
251+
yield* scope.stop("session-2.scope");
252+
expect(stopsOf(calls)).toEqual([]);
253+
expect(calls.some((args) => args.includes("session-2.scope"))).toBe(false);
254+
}),
255+
),
256+
);
257+
184258
it.effect("leaves a quiet slice alone", () =>
185259
withFakeSystemd(({ scope, calls }) =>
186260
Effect.gen(function* () {

‎apps/server/src/process/agentScope.ts‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import {
1010
} from "@t3tools/shared/AgentScope";
1111
import * as HostProcess from "@t3tools/shared/HostProcess";
1212
import { resolveCommandPath } from "@t3tools/shared/shell";
13+
import * as Context from "effect/Context";
1314
import * as Effect from "effect/Effect";
1415
import * as FileSystem from "effect/FileSystem";
1516
import * as Layer from "effect/Layer";
@@ -20,6 +21,11 @@ import * as ProcessRunner from "../processRunner.ts";
2021

2122
const AGENT_SLICE = "app-t3code-agents.slice";
2223

24+
/** Where the cgroup2 hierarchy is mounted. Tests point it at a temp dir. */
25+
export const CgroupRoot = Context.Reference<string>("t3/process/agentScope/CgroupRoot", {
26+
defaultValue: () => "/sys/fs/cgroup",
27+
});
28+
2329
export function ownedAgentSlice(stateDir: string): string {
2430
const hash = NodeCrypto.createHash("sha256").update(stateDir).digest("hex").slice(0, 12);
2531
return `app-t3code-agents-${hash}.slice`;
@@ -101,6 +107,7 @@ export const make = Effect.gen(function* () {
101107
const hostEnvironment = yield* HostProcess.Environment;
102108
const { stateDir } = yield* ServerConfig.ServerConfig;
103109
const ownedSlice = ownedAgentSlice(stateDir);
110+
const cgroupRoot = yield* CgroupRoot;
104111
const layerScope = yield* Effect.scope;
105112

106113
const resolve = (command: string, env: NodeJS.ProcessEnv) =>
@@ -115,7 +122,7 @@ export const make = Effect.gen(function* () {
115122

116123
const readCgroupFile = (controlGroup: string, file: string) =>
117124
fileSystem
118-
.readFileString(path.join("/sys/fs/cgroup", controlGroup, file))
125+
.readFileString(path.join(cgroupRoot, controlGroup, file))
119126
.pipe(Effect.orElseSucceed(() => ""));
120127

121128
// Old agents keep their scopes when the server is killed, and the restarted
@@ -303,6 +310,20 @@ export const make = Effect.gen(function* () {
303310
return false;
304311
});
305312

313+
const stop: AgentScopeShape["stop"] = (unit) =>
314+
Effect.gen(function* () {
315+
if (!unit.startsWith("t3code-") || !unit.endsWith(".scope")) return;
316+
const scope = yield* systemd;
317+
if (scope === undefined) return;
318+
// The kernel counts an OOM kill before it sends the kill, so the count is
319+
// visible here even when systemd has not handled the kill yet. A stop
320+
// from here could then end the scope as a plain stop, and the thread
321+
// would lose its OOM label. An empty scope has nothing left to stop.
322+
const state = yield* readScope(scope.systemctl, unit);
323+
if (state === undefined || classifyScope(state) !== "running") return;
324+
yield* run(scope.systemctl, ["--user", "stop", "--no-block", unit]);
325+
});
326+
306327
const clear: AgentScopeShape["clear"] = (threadId) =>
307328
Effect.gen(function* () {
308329
if (!scopeByThread.has(threadId)) return;
@@ -334,9 +355,10 @@ export const make = Effect.gen(function* () {
334355
// the unit even when it fails.
335356
options: threadId === undefined ? [...scope.scopeOptions, "--collect"] : scope.scopeOptions,
336357
};
337-
return agentScopeCommand({ command: resolved, args, scope: systemdScope });
358+
return { ...agentScopeCommand({ command: resolved, args, scope: systemdScope }), unit };
338359
}),
339360
oomKilled,
361+
stop,
340362
clear,
341363
} satisfies AgentScopeShape;
342364
});

‎apps/server/src/provider/acp/AcpJsonRpcConnection.test.ts‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import * as Stream from "effect/Stream";
1818
import { describe, expect } from "vite-plus/test";
1919

2020
import * as AcpSessionRuntime from "@t3tools/provider-acp/server/AcpSessionRuntime";
21+
import { AgentScope } from "@t3tools/shared/AgentScope";
2122
import type * as EffectAcpProtocol from "effect-acp/protocol";
2223
import * as EffectAcpErrors from "effect-acp/errors";
2324

@@ -94,6 +95,28 @@ describe("AcpSessionRuntime", () => {
9495
}).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
9596
);
9697

98+
// A daemon a tool detached with setsid leaves the agent's process group but
99+
// stays in its systemd scope.
100+
it.effect("stops the agent's scope once the runtime has ended the agent", () =>
101+
Effect.gen(function* () {
102+
const stopped: Array<string> = [];
103+
yield* Effect.gen(function* () {
104+
const runtime = yield* AcpSessionRuntime.make(mockRuntimeOptions);
105+
yield* runtime.start();
106+
expect(stopped).toEqual([]);
107+
}).pipe(
108+
Effect.scoped,
109+
Effect.provideService(AgentScope, {
110+
...AgentScope.defaultValue(),
111+
wrap: ({ command, args }) =>
112+
Effect.succeed({ command, args, unit: "t3code-acp-test.scope" }),
113+
stop: (unit) => Effect.sync(() => void stopped.push(unit)),
114+
}),
115+
);
116+
expect(stopped).toEqual(["t3code-acp-test.scope"]);
117+
}).pipe(Effect.provide(NodeServices.layer)),
118+
);
119+
97120
it.effect("publishes model changes returned by a config request and live notifications", () =>
98121
Effect.gen(function* () {
99122
const runtime = yield* AcpSessionRuntime.make(mockRuntimeOptions);

‎packages/provider-acp/src/server/AcpSessionRuntime.ts‎

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1437,19 +1437,24 @@ export const make = (
14371437
cause: new Error("Contained ACP command was not found on PATH"),
14381438
});
14391439
});
1440-
const scopedSpawnCommand =
1440+
const agentScope = yield* AgentScope;
1441+
const scopedLaunch =
14411442
linuxCgroupLease !== undefined || spawnCommand.shell
1442-
? spawnCommand
1443-
: {
1444-
...(yield* (yield* AgentScope).wrap({
1445-
command: spawnCommand.command,
1446-
args: spawnCommand.args,
1447-
name: "acp",
1448-
threadId: yield* AgentScopeThreadId,
1449-
env: { ...process.env, ...options.spawn.env },
1450-
})),
1451-
shell: false,
1452-
};
1443+
? undefined
1444+
: yield* agentScope.wrap({
1445+
command: spawnCommand.command,
1446+
args: spawnCommand.args,
1447+
name: "acp",
1448+
threadId: yield* AgentScopeThreadId,
1449+
env: { ...process.env, ...options.spawn.env },
1450+
});
1451+
const scopedSpawnCommand =
1452+
scopedLaunch === undefined ? spawnCommand : { ...scopedLaunch, shell: false };
1453+
// Registered before the spawn, so it runs after the finalizers that end the agent.
1454+
const scopeUnit = scopedLaunch?.unit;
1455+
if (scopeUnit !== undefined) {
1456+
yield* Scope.addFinalizer(runtimeScope, agentScope.stop(scopeUnit));
1457+
}
14531458
const containedSpawnCommand =
14541459
linuxCgroupLease === undefined
14551460
? scopedSpawnCommand

0 commit comments

Comments
 (0)