Skip to content

Commit e4025c8

Browse files
fix(server): reap recorded Linux Claude survivors before recovery (pingdotgg#22834)
1 parent 0902adc commit e4025c8

6 files changed

Lines changed: 384 additions & 23 deletions

File tree

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ import {
8282
mcpAppToolResourceUri,
8383
} from "@t3tools/shared/mcpApp";
8484
import { AgentScope } from "@t3tools/shared/AgentScope";
85+
import * as ProviderProcesses from "../../process/providerProcesses.ts";
8586
import * as HostProcess from "@t3tools/shared/HostProcess";
8687
import { applyClaudePromptEffortPrefix } from "@t3tools/shared/model";
8788
import { compareSemverVersions } from "@t3tools/shared/semver";
@@ -372,21 +373,26 @@ const CLAUDE_DESCENDANT_POLL = "1 second";
372373
const CLAUDE_ROOT_REFUSAL =
373374
"--dangerously-skip-permissions cannot be used with root/sudo privileges for security reasons";
374375

376+
// fd 4 gates the launch until the server has recorded the process it is about to become.
375377
const CLAUDE_GROUP_WATCH_SCRIPT = [
378+
"read -r _ <&4 || exit 1",
379+
"exec 4<&-",
376380
"{ trap '' TERM; read -r _ <&3; kill -TERM 0; sleep 5; kill -KILL 0; } </dev/null >/dev/null 2>&1 &",
377381
'exec "$@" 3<&-',
378382
].join("\n");
379383

380384
/**
381385
* Kill the CLI's group on exit to stop MCP grandchildren; `onExit` stops its agent scope for
382386
* setsid daemons. The CLI keeps working after its pipes close, so a watcher stops the group on
383-
* EOF from a pipe only this server holds, including when the server crashes.
387+
* EOF from a pipe only this server holds, including when the server crashes. `register` records
388+
* the CLI before it may run, so a restarted server can reap it if the watcher did not.
384389
*/
385390
export function spawnClaudeProcessGroup(
386391
options: ClaudeSpawnOptions,
387392
onExit?: () => void,
388393
onStderr: (text: string) => void = () => {},
389394
onStderrClose: () => void = () => {},
395+
register?: (pid: number) => () => void,
390396
): ClaudeSpawnedProcess & Pick<NodeChildProcess.ChildProcess, "pid"> {
391397
const child = NodeChildProcess.spawn(
392398
"/bin/sh",
@@ -395,18 +401,34 @@ export function spawnClaudeProcessGroup(
395401
cwd: options.cwd,
396402
env: options.env,
397403
signal: options.signal,
398-
stdio: ["pipe", "pipe", "pipe", "pipe"],
404+
stdio: ["pipe", "pipe", "pipe", "pipe", "pipe"],
399405
detached: true,
400406
},
401407
);
408+
let forget: (() => void) | undefined;
402409
// Always drained, so a chatty CLI never blocks on a full stderr pipe.
403410
child.stderr?.setEncoding("utf8").on("data", onStderr).once("close", onStderrClose);
404411
child.once("exit", () => {
405412
try {
406413
signalProcessGroup(child.pid ?? 0, "SIGKILL");
407414
} catch {}
415+
try {
416+
forget?.();
417+
} catch {}
408418
onExit?.();
409419
});
420+
const gate = child.stdio[4];
421+
if (child.pid !== undefined && gate !== null && gate !== undefined && "end" in gate) {
422+
try {
423+
forget = register?.(child.pid);
424+
gate.on("error", () => child.kill("SIGKILL"));
425+
gate.end("ready\n");
426+
} catch (cause) {
427+
child.kill("SIGKILL");
428+
gate.destroy();
429+
throw cause;
430+
}
431+
}
410432
// Node's typings make stdin and stdout nullable once stdio has a fourth entry; both are pipes.
411433
return child as ClaudeSpawnedProcess & Pick<NodeChildProcess.ChildProcess, "pid">;
412434
}
@@ -1256,6 +1278,7 @@ export const layerQueryRunner: Layer.Layer<
12561278
const path = yield* Path.Path;
12571279
const { native: nativeEventLogger } = yield* ProviderEventLoggers.ProviderEventLoggers;
12581280
const agentScope = yield* AgentScope;
1281+
const providerProcesses = yield* ProviderProcesses.ProviderProcesses;
12591282
const platform = yield* HostProcess.Platform;
12601283
const runFork = Effect.runForkWith(yield* Effect.context<never>());
12611284
const hostEnvironment = yield* HostProcess.Environment;
@@ -1372,6 +1395,7 @@ export const layerQueryRunner: Layer.Layer<
13721395
stopScope,
13731396
appendStderr,
13741397
stderrDone,
1398+
providerProcesses.register,
13751399
);
13761400
if (unit === undefined && child.pid !== undefined) {
13771401
const tracker = trackDescendants(child.pid);

‎apps/server/src/orchestration-v2/Adapters/ClaudeAgentSdkQueryRunner.test.ts‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -748,6 +748,47 @@ describe.skipIf(HostProcess.Platform.defaultValue() === "win32")("Claude CLI pro
748748
}),
749749
);
750750

751+
// A restarted server reaps recorded CLIs, so a CLI must never run before its record exists.
752+
it.effect("records the CLI before it runs and forgets it once it exits", () =>
753+
Effect.gen(function* () {
754+
const cli = fakeClaudeCli(
755+
['dir=$(dirname "$0")', 'printf \'%s %s\' "$$" "$(cat "$dir/record")" > "$dir/seen"'].join(
756+
"\n",
757+
),
758+
);
759+
const dir = NodePath.dirname(cli);
760+
NodeChildProcess.execFileSync("mkfifo", [NodePath.join(dir, "seen")]);
761+
yield* Effect.addFinalizer(() =>
762+
Effect.sync(() => NodeFS.rmSync(dir, { recursive: true, force: true })),
763+
);
764+
const forgotten = yield* Deferred.make<void>();
765+
let registered = 0;
766+
spawnClaudeProcessGroup(
767+
{
768+
command: cli,
769+
args: [],
770+
cwd: dir,
771+
env: process.env,
772+
// @effect-diagnostics-next-line abortControllerInEffect:off - The CLI exits on its own.
773+
signal: new AbortController().signal,
774+
},
775+
undefined,
776+
undefined,
777+
undefined,
778+
(pid) => {
779+
registered = pid;
780+
NodeFS.writeFileSync(NodePath.join(dir, "record"), String(pid));
781+
return () => Deferred.doneUnsafe(forgotten, Effect.void);
782+
},
783+
);
784+
const seen = yield* Effect.promise(() =>
785+
NodeFS.promises.readFile(NodePath.join(dir, "seen"), "utf8"),
786+
);
787+
assert.equal(seen, `${registered} ${registered}`);
788+
yield* Deferred.await(forgotten);
789+
}),
790+
);
791+
751792
it.effect("marks the CLI's jobs with its thread and counts it as an agent until it exits", () => {
752793
let runningAgents = 0;
753794
let releaseAgent!: () => void;

‎apps/server/src/process/agentScope.test.ts‎

Lines changed: 39 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import { ChildProcess, ChildProcessSpawner } from "effect/process";
1616
import * as ServerConfig from "../config.ts";
1717
import { makeCodexAppServerSpawnCommand } from "../orchestration-v2/Adapters/CodexAdapterV2.ts";
1818
import * as ProcessRunner from "../processRunner.ts";
19+
import * as ProviderProcesses from "./providerProcesses.ts";
1920
import { classifyScope, make, ownedAgentSlice, type ScopeState } from "./agentScope.ts";
2021

2122
const running: ScopeState = {
@@ -55,7 +56,14 @@ const withFakeSystemd = <A, E>(
5556
{
5657
oomPolicy = true,
5758
stopFails = false,
58-
}: { readonly oomPolicy?: boolean; readonly stopFails?: boolean } = {},
59+
leftovers = false,
60+
liveOwner = false,
61+
}: {
62+
readonly oomPolicy?: boolean;
63+
readonly stopFails?: boolean;
64+
readonly leftovers?: boolean;
65+
readonly liveOwner?: boolean;
66+
} = {},
5967
) =>
6068
Effect.gen(function* () {
6169
const bin = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-agent-scope-"));
@@ -102,7 +110,12 @@ const withFakeSystemd = <A, E>(
102110
}),
103111
});
104112
const fileSystem = yield* FileSystem.FileSystem;
113+
const { stateDir } = yield* ServerConfig.ServerConfig;
114+
if (leftovers) {
115+
units.set(ownedAgentSlice(stateDir), { activeState: "active", result: "success" });
116+
}
105117
const scope = yield* make.pipe(
118+
Effect.provideService(ProviderProcesses.ProviderProcesses, { hasOtherLiveOwner: liveOwner }),
106119
Effect.provideService(ProcessRunner.ProcessRunner, runner),
107120
Effect.provideService(HostProcess.Platform, "linux"),
108121
Effect.provideService(HostProcess.Environment, { PATH: bin, XDG_RUNTIME_DIR: "/run/user/1" }),
@@ -114,7 +127,6 @@ const withFakeSystemd = <A, E>(
114127
fileSystem.writeFileString(path.replace("/sys/fs/cgroup", cgroupRoot), data, options),
115128
}),
116129
);
117-
const { stateDir } = yield* ServerConfig.ServerConfig;
118130
return yield* body({ scope, units, calls, stateDir, cgroupRoot }).pipe(
119131
Effect.ensuring(Effect.sync(() => NodeFS.rmSync(bin, { recursive: true, force: true }))),
120132
);
@@ -308,27 +320,34 @@ describe("AgentScope service", () => {
308320
),
309321
);
310322

311-
it.effect("stops agents a killed server left before the first launch", () =>
312-
withFakeSystemd(({ scope, units, calls, stateDir }) =>
313-
Effect.gen(function* () {
314-
units.set(ownedAgentSlice(stateDir), { activeState: "active", result: "success" });
315-
const agent = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
316-
const stops = calls.filter(([, verb]) => verb === "stop");
317-
expect(stops).toEqual([["--user", "stop", ownedAgentSlice(stateDir)]]);
318-
expect(sliceOf(agent)).toBe(ownedAgentSlice(stateDir));
319-
expect(agent.args).toContain("--property=TimeoutStopSec=10s");
320-
const terminal = yield* scope.wrap({
321-
command: "agent",
322-
args: [],
323-
name: "terminal",
324-
survivesRestart: true,
325-
});
326-
expect(sliceOf(terminal)).toBe("app-t3code-agents.slice");
327-
expect(calls.filter(([, verb]) => verb === "stop")).toHaveLength(1);
328-
}),
323+
it.effect("stops agents a killed server left before startup completes", () =>
324+
withFakeSystemd(
325+
({ scope, calls, stateDir }) =>
326+
Effect.gen(function* () {
327+
expect(stopsOf(calls)).toEqual([["--user", "stop", ownedAgentSlice(stateDir)]]);
328+
const agent = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
329+
expect(sliceOf(agent)).toBe(ownedAgentSlice(stateDir));
330+
expect(agent.args).toContain("--property=TimeoutStopSec=10s");
331+
const terminal = yield* scope.wrap({
332+
command: "agent",
333+
args: [],
334+
name: "terminal",
335+
survivesRestart: true,
336+
});
337+
expect(sliceOf(terminal)).toBe("app-t3code-agents.slice");
338+
expect(stopsOf(calls)).toHaveLength(1);
339+
}),
340+
{ leftovers: true },
329341
),
330342
);
331343

344+
it.effect("leaves the slice to another live server sharing this home", () =>
345+
withFakeSystemd(({ calls }) => Effect.sync(() => expect(stopsOf(calls)).toEqual([])), {
346+
leftovers: true,
347+
liveOwner: true,
348+
}),
349+
);
350+
332351
it.effect("stops leftover processes only in an active agent scope", () =>
333352
withFakeSystemd(({ scope, units, calls, cgroupRoot }) =>
334353
Effect.gen(function* () {

‎apps/server/src/process/agentScope.ts‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import * as Path from "effect/Path";
1717

1818
import * as ServerConfig from "../config.ts";
1919
import * as ProcessRunner from "../processRunner.ts";
20+
import * as ProviderProcesses from "./providerProcesses.ts";
2021

2122
const AGENT_SLICE = "app-t3code-agents.slice";
2223
// Owns every agent slice and competes with the server's unit or scope under app.slice.
@@ -106,6 +107,7 @@ export const make = Effect.gen(function* () {
106107
const path = yield* Path.Path;
107108
const hostEnvironment = yield* HostProcess.Environment;
108109
const { stateDir } = yield* ServerConfig.ServerConfig;
110+
const { hasOtherLiveOwner } = yield* ProviderProcesses.ProviderProcesses;
109111
const ownedSlice = ownedAgentSlice(stateDir);
110112
const layerScope = yield* Effect.scope;
111113

@@ -130,6 +132,7 @@ export const make = Effect.gen(function* () {
130132
const stopLeftoverAgents = Effect.fn("agentScope.stopLeftoverAgents")(function* (
131133
systemctl: string,
132134
) {
135+
if (hasOtherLiveOwner) return;
133136
const shown = yield* run(systemctl, [
134137
"--user",
135138
"show",
@@ -215,6 +218,10 @@ export const make = Effect.gen(function* () {
215218
}),
216219
);
217220

221+
// Layer acquisition is the startup barrier: recovery must not terminalize or
222+
// continue old runs while a previous server's providers can still execute.
223+
yield* systemd;
224+
218225
// `shared` scopes serve other threads too: tracked only for OOM diagnosis on old systemd.
219226
const scopeByThread = new Map<string, { unit: string; oomKilled: boolean; shared: boolean }>();
220227

@@ -427,4 +434,7 @@ export const make = Effect.gen(function* () {
427434
} satisfies AgentScopeShape;
428435
});
429436

430-
export const layer = Layer.effect(AgentScope, make).pipe(Layer.provide(ProcessRunner.layer));
437+
export const layer = Layer.effect(AgentScope, make).pipe(
438+
Layer.provide(ProcessRunner.layer),
439+
Layer.provideMerge(ProviderProcesses.layer),
440+
);

0 commit comments

Comments
 (0)