Skip to content

Commit dd60673

Browse files
fix(server): an unkillable recorded provider no longer blocks startup
Reaping logs and keeps the record for the next start instead of dying, matching how a failed slice stop is handled. The reaping test moves to providerProcesses.test.ts and also asserts that a live owner is seen. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 8716139 commit dd60673

4 files changed

Lines changed: 143 additions & 131 deletions

File tree

‎apps/server/src/process/agentScope.test.ts‎

Lines changed: 1 addition & 118 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,12 @@
11
// @effect-diagnostics nodeBuiltinImport:off -- fake systemd binaries live in a temp dir.
2-
import * as NodeChildProcess from "node:child_process";
3-
import * as NodeCrypto from "node:crypto";
42
import * as NodeFS from "node:fs";
53
import * as NodeOS from "node:os";
64
import * as NodePath from "node:path";
75

86
import * as NodeServices from "@effect/platform-node/NodeServices";
97
import { assert, describe, expect, it } from "@effect/vitest";
10-
import * as AcpSessionRuntime from "@t3tools/provider-acp/server/AcpSessionRuntime";
118
import { AgentScope } from "@t3tools/shared/AgentScope";
129
import * as HostProcess from "@t3tools/shared/HostProcess";
13-
import * as Deferred from "effect/Deferred";
1410
import * as Effect from "effect/Effect";
1511
import * as FileSystem from "effect/FileSystem";
1612
import * as Layer from "effect/Layer";
@@ -21,7 +17,7 @@ import * as ServerConfig from "../config.ts";
2117
import { makeCodexAppServerSpawnCommand } from "../orchestration-v2/Adapters/CodexAdapterV2.ts";
2218
import * as ProcessRunner from "../processRunner.ts";
2319
import * as ProviderProcesses from "./providerProcesses.ts";
24-
import { classifyScope, layer, make, ownedAgentSlice, type ScopeState } from "./agentScope.ts";
20+
import { classifyScope, make, ownedAgentSlice, type ScopeState } from "./agentScope.ts";
2521

2622
const running: ScopeState = {
2723
loadState: "loaded",
@@ -438,116 +434,3 @@ describe("AgentScope service", () => {
438434
),
439435
);
440436
});
441-
442-
// Real child processes stand in for servers and CLIs; only PIDs captured at spawn are signalled.
443-
const ownedProcess = Effect.gen(function* () {
444-
const ready = yield* Deferred.make<void>();
445-
const exited = yield* Deferred.make<void>();
446-
const child = yield* Effect.acquireRelease(
447-
Effect.sync(() => {
448-
const child = NodeChildProcess.spawn(
449-
process.execPath,
450-
["-e", 'process.stdin.resume(); process.send("ready");'],
451-
{ stdio: ["pipe", "ignore", "ignore", "ipc"] },
452-
);
453-
child.once("message", () => Deferred.doneUnsafe(ready, Effect.void));
454-
child.once("error", (error) => Deferred.doneUnsafe(ready, Effect.die(error)));
455-
child.once("exit", () => Deferred.doneUnsafe(exited, Effect.void));
456-
return child;
457-
}),
458-
(child) =>
459-
Effect.sync(() => child.kill("SIGKILL")).pipe(Effect.andThen(Deferred.await(exited))),
460-
);
461-
yield* Deferred.await(ready);
462-
const pid = child.pid;
463-
assert(pid !== undefined);
464-
const controller = AcpSessionRuntime.makePosixProcessTreeController("linux");
465-
const identity = controller.identity(pid);
466-
assert(identity !== undefined);
467-
const boot = NodeFS.readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim();
468-
return {
469-
pid,
470-
startTime: `${boot}:${identity.startTime}`,
471-
pgid: 0,
472-
running: () => {
473-
const observed = controller.identity(pid);
474-
return observed !== undefined && !AcpSessionRuntime.posixProcessIsZombie(observed);
475-
},
476-
stop: Effect.sync(() => child.kill("SIGKILL")).pipe(Effect.andThen(Deferred.await(exited))),
477-
};
478-
});
479-
480-
describe.runIf(HostProcess.Platform.defaultValue() === "linux")(
481-
"provider restart ownership",
482-
() => {
483-
it.live("reaps only recorded CLIs of dead servers before startup completes", () =>
484-
Effect.gen(function* () {
485-
const config = yield* ServerConfig.ServerConfig;
486-
const directory = NodePath.join(config.stateDir, "provider-processes");
487-
NodeFS.mkdirSync(directory, { recursive: true });
488-
type Identity = { readonly pid: number; readonly startTime: string };
489-
const record = (owner: Identity, processes: ReadonlyArray<Identity>) => {
490-
const instanceId = NodeCrypto.randomUUID();
491-
NodeFS.writeFileSync(
492-
NodePath.join(directory, `${instanceId}.json`),
493-
JSON.stringify({
494-
version: 1,
495-
instanceId,
496-
hostname: NodeOS.hostname(),
497-
stateDir: config.stateDir,
498-
owner: { pid: owner.pid, startTime: owner.startTime },
499-
processes: processes.map(({ pid, startTime }) => ({ pid, startTime, pgid: 0 })),
500-
}),
501-
);
502-
};
503-
const reused = (identity: Identity) => ({
504-
...identity,
505-
startTime: identity.startTime.replace(/:\d+$/, ":1"),
506-
});
507-
const deadOwner = yield* ownedProcess;
508-
const liveOwner = yield* ownedProcess;
509-
const reusedOwner = yield* ownedProcess;
510-
const liveProvider = yield* ownedProcess;
511-
const reusedProvider = yield* ownedProcess;
512-
const orphan = yield* ownedProcess;
513-
const orphanOfReusedOwner = yield* ownedProcess;
514-
const unrecorded = yield* ownedProcess;
515-
record(liveOwner, [liveProvider]);
516-
record(deadOwner, [reused(reusedProvider)]);
517-
record(deadOwner, [orphan]);
518-
record(reused(reusedOwner), [orphanOfReusedOwner]);
519-
yield* deadOwner.stop;
520-
521-
yield* Layer.build(layer).pipe(
522-
Effect.provideService(ServerConfig.ServerConfig, config),
523-
Effect.provide(NodeServices.layer),
524-
);
525-
526-
expect({
527-
orphan: orphan.running(),
528-
orphanOfReusedOwner: orphanOfReusedOwner.running(),
529-
liveProvider: liveProvider.running(),
530-
reusedProvider: reusedProvider.running(),
531-
reusedOwner: reusedOwner.running(),
532-
unrecorded: unrecorded.running(),
533-
}).toEqual({
534-
orphan: false,
535-
orphanOfReusedOwner: false,
536-
liveProvider: true,
537-
reusedProvider: true,
538-
reusedOwner: true,
539-
unrecorded: true,
540-
});
541-
}).pipe(
542-
Effect.scoped,
543-
Effect.provideService(HostProcess.Platform, "linux"),
544-
Effect.provideService(HostProcess.Environment, { PATH: process.env.PATH }),
545-
Effect.provide(
546-
ServerConfig.layerTest(process.cwd(), { prefix: "t3-restart-ownership-" }).pipe(
547-
Layer.provideMerge(NodeServices.layer),
548-
),
549-
),
550-
),
551-
);
552-
},
553-
);

‎apps/server/src/process/agentScope.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ export const make = Effect.gen(function* () {
132132
const stopLeftoverAgents = Effect.fn("agentScope.stopLeftoverAgents")(function* (
133133
systemctl: string,
134134
) {
135-
if (hasOtherLiveOwner === true) return;
135+
if (hasOtherLiveOwner) return;
136136
const shown = yield* run(systemctl, [
137137
"--user",
138138
"show",
Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
// @effect-diagnostics nodeBuiltinImport:off -- real processes stand in for servers and CLIs.
2+
import * as NodeChildProcess from "node:child_process";
3+
import * as NodeCrypto from "node:crypto";
4+
import * as NodeFS from "node:fs";
5+
import * as NodeOS from "node:os";
6+
import * as NodePath from "node:path";
7+
8+
import * as NodeServices from "@effect/platform-node/NodeServices";
9+
import { assert, describe, expect, it } from "@effect/vitest";
10+
import * as AcpSessionRuntime from "@t3tools/provider-acp/server/AcpSessionRuntime";
11+
import * as HostProcess from "@t3tools/shared/HostProcess";
12+
import * as Context from "effect/Context";
13+
import * as Deferred from "effect/Deferred";
14+
import * as Effect from "effect/Effect";
15+
import * as Layer from "effect/Layer";
16+
17+
import * as ServerConfig from "../config.ts";
18+
import * as ProviderProcesses from "./providerProcesses.ts";
19+
20+
// Real child processes stand in for servers and CLIs; only PIDs captured at spawn are signalled.
21+
const ownedProcess = Effect.gen(function* () {
22+
const ready = yield* Deferred.make<void>();
23+
const exited = yield* Deferred.make<void>();
24+
const child = yield* Effect.acquireRelease(
25+
Effect.sync(() => {
26+
const child = NodeChildProcess.spawn(
27+
process.execPath,
28+
["-e", 'process.stdin.resume(); process.send("ready");'],
29+
{ stdio: ["pipe", "ignore", "ignore", "ipc"] },
30+
);
31+
child.once("message", () => Deferred.doneUnsafe(ready, Effect.void));
32+
child.once("error", (error) => Deferred.doneUnsafe(ready, Effect.die(error)));
33+
child.once("exit", () => Deferred.doneUnsafe(exited, Effect.void));
34+
return child;
35+
}),
36+
(child) =>
37+
Effect.sync(() => child.kill("SIGKILL")).pipe(Effect.andThen(Deferred.await(exited))),
38+
);
39+
yield* Deferred.await(ready);
40+
const pid = child.pid;
41+
assert(pid !== undefined);
42+
const controller = AcpSessionRuntime.makePosixProcessTreeController("linux");
43+
const identity = controller.identity(pid);
44+
assert(identity !== undefined);
45+
const boot = NodeFS.readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim();
46+
return {
47+
pid,
48+
startTime: `${boot}:${identity.startTime}`,
49+
pgid: 0,
50+
running: () => {
51+
const observed = controller.identity(pid);
52+
return observed !== undefined && !AcpSessionRuntime.posixProcessIsZombie(observed);
53+
},
54+
stop: Effect.sync(() => child.kill("SIGKILL")).pipe(Effect.andThen(Deferred.await(exited))),
55+
};
56+
});
57+
58+
describe.runIf(HostProcess.Platform.defaultValue() === "linux")("ProviderProcesses", () => {
59+
it.live("reaps only recorded CLIs of dead servers before startup completes", () =>
60+
Effect.gen(function* () {
61+
const config = yield* ServerConfig.ServerConfig;
62+
const directory = NodePath.join(config.stateDir, "provider-processes");
63+
NodeFS.mkdirSync(directory, { recursive: true });
64+
type Identity = { readonly pid: number; readonly startTime: string };
65+
const record = (owner: Identity, processes: ReadonlyArray<Identity>) => {
66+
const instanceId = NodeCrypto.randomUUID();
67+
NodeFS.writeFileSync(
68+
NodePath.join(directory, `${instanceId}.json`),
69+
JSON.stringify({
70+
version: 1,
71+
instanceId,
72+
hostname: NodeOS.hostname(),
73+
stateDir: config.stateDir,
74+
owner: { pid: owner.pid, startTime: owner.startTime },
75+
processes: processes.map(({ pid, startTime }) => ({ pid, startTime, pgid: 0 })),
76+
}),
77+
);
78+
};
79+
const reused = (identity: Identity) => ({
80+
...identity,
81+
startTime: identity.startTime.replace(/:\d+$/, ":1"),
82+
});
83+
const deadOwner = yield* ownedProcess;
84+
const liveOwner = yield* ownedProcess;
85+
const reusedOwner = yield* ownedProcess;
86+
const liveProvider = yield* ownedProcess;
87+
const reusedProvider = yield* ownedProcess;
88+
const orphan = yield* ownedProcess;
89+
const orphanOfReusedOwner = yield* ownedProcess;
90+
const unrecorded = yield* ownedProcess;
91+
record(liveOwner, [liveProvider]);
92+
record(deadOwner, [reused(reusedProvider)]);
93+
record(deadOwner, [orphan]);
94+
record(reused(reusedOwner), [orphanOfReusedOwner]);
95+
yield* deadOwner.stop;
96+
97+
const context = yield* Layer.build(ProviderProcesses.layer);
98+
expect(Context.get(context, ProviderProcesses.ProviderProcesses).hasOtherLiveOwner).toBe(
99+
true,
100+
);
101+
102+
expect({
103+
orphan: orphan.running(),
104+
orphanOfReusedOwner: orphanOfReusedOwner.running(),
105+
liveProvider: liveProvider.running(),
106+
reusedProvider: reusedProvider.running(),
107+
reusedOwner: reusedOwner.running(),
108+
unrecorded: unrecorded.running(),
109+
}).toEqual({
110+
orphan: false,
111+
orphanOfReusedOwner: false,
112+
liveProvider: true,
113+
reusedProvider: true,
114+
reusedOwner: true,
115+
unrecorded: true,
116+
});
117+
}).pipe(
118+
Effect.scoped,
119+
Effect.provideService(HostProcess.Platform, "linux"),
120+
Effect.provide(
121+
ServerConfig.layerTest(process.cwd(), { prefix: "t3-restart-ownership-" }).pipe(
122+
Layer.provideMerge(NodeServices.layer),
123+
),
124+
),
125+
),
126+
);
127+
});

‎apps/server/src/process/providerProcesses.ts‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -38,12 +38,12 @@ const readRecord = (recordPath: string) => {
3838

3939
export const ProviderProcesses = Context.Reference<{
4040
readonly register?: (pid: number) => () => void;
41-
readonly hasOtherLiveOwner?: boolean;
42-
}>("t3/process/ProviderProcesses", { defaultValue: () => ({}) });
41+
readonly hasOtherLiveOwner: boolean;
42+
}>("t3/process/ProviderProcesses", { defaultValue: () => ({ hasOtherLiveOwner: false }) });
4343

4444
const make = Effect.gen(function* () {
4545
const platform = yield* HostProcess.Platform;
46-
if (platform !== "linux") return {};
46+
if (platform !== "linux") return { hasOtherLiveOwner: false };
4747
const { stateDir } = yield* ServerConfig.ServerConfig;
4848
const directory = NodePath.join(stateDir, "provider-processes");
4949
const hostname = NodeOS.hostname();
@@ -57,13 +57,10 @@ const make = Effect.gen(function* () {
5757
};
5858
const matches = (expected: typeof ProcessIdentity.Type) =>
5959
identity(expected.pid)?.startTime === expected.startTime;
60-
// A failed signal only matters while the recorded process is still the same one.
6160
const kill = (survivor: typeof ProcessIdentity.Type) => {
6261
try {
6362
controller.signalProcess(survivor.pid, "SIGKILL");
64-
} catch (cause) {
65-
if (matches(survivor)) throw cause;
66-
}
63+
} catch {}
6764
};
6865
const owner = identity(process.pid);
6966
if (owner === undefined) return yield* Effect.die(new Error("Cannot identify provider owner"));
@@ -105,19 +102,24 @@ const make = Effect.gen(function* () {
105102
if (record.owner.pid !== process.pid) hasOtherLiveOwner = true;
106103
continue;
107104
}
105+
let survived = false;
108106
for (const survivor of record.processes) {
109107
if (survivor.pid === process.pid || !matches(survivor)) continue;
110108
// Never infer ownership from argv, executable names, or group membership.
111109
// As in the ACP controller, starttime blocks reuse between observations.
112110
// Node has no pidfd_send_signal, so the final read-to-signal race remains.
113111
kill(survivor);
114-
for (let attempt = 0; matches(survivor); attempt++) {
115-
if (attempt === 100) {
116-
return yield* Effect.die(new Error("Recorded provider survived restart cleanup"));
117-
}
118-
yield* Effect.sleep("10 millis");
112+
for (let attempt = 0; matches(survivor) && !survived; attempt++) {
113+
survived = attempt === 100;
114+
if (!survived) yield* Effect.sleep("10 millis");
119115
}
120116
}
117+
// Like a failed slice stop, an unkillable CLI must not keep the server down;
118+
// its record stays so the next start retries.
119+
if (survived) {
120+
yield* Effect.logWarning("Could not stop a provider left by a previous server", { file });
121+
continue;
122+
}
121123
NodeFS.rmSync(recordPath, { force: true });
122124
}
123125

0 commit comments

Comments
 (0)