Skip to content

Commit 3fff3c1

Browse files
committed
fix(server): keep owned ACP work in agent resource scopes
1 parent 5307c23 commit 3fff3c1

5 files changed

Lines changed: 529 additions & 37 deletions

File tree

‎apps/server/src/process/agentScope.test.ts‎

Lines changed: 178 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,11 @@ import * as NodeServices from "@effect/platform-node/NodeServices";
77
import { describe, expect, it } from "@effect/vitest";
88
import * as HostProcess from "@t3tools/shared/HostProcess";
99
import * as Effect from "effect/Effect";
10+
import * as Deferred from "effect/Deferred";
11+
import * as Fiber from "effect/Fiber";
12+
import * as TestClock from "effect/testing/TestClock";
1013
import * as Layer from "effect/Layer";
14+
import * as FileSystem from "effect/FileSystem";
1115
import { ChildProcessSpawner } from "effect/process";
1216

1317
import * as ServerConfig from "../config.ts";
@@ -61,21 +65,41 @@ describe("agentSliceMemoryLimits", () => {
6165
});
6266
});
6367

68+
interface FakeScopeState {
69+
activeState: string;
70+
result: string;
71+
oomKills?: number | undefined;
72+
populated?: boolean;
73+
}
74+
6475
const withFakeSystemd = <A, E>(
6576
body: (input: {
6677
readonly scope: Effect.Success<typeof make>;
67-
readonly units: Map<string, { activeState: string; result: string }>;
78+
readonly units: Map<string, FakeScopeState>;
6879
readonly calls: ReadonlyArray<ReadonlyArray<string>>;
6980
readonly stateDir: string;
81+
readonly handlers: Map<
82+
string,
83+
(
84+
input: ProcessRunner.ProcessRunInput,
85+
) => Effect.Effect<ProcessRunner.ProcessRunOutput | undefined>
86+
>;
87+
readonly output: (stdout: string) => ProcessRunner.ProcessRunOutput;
7088
}) => Effect.Effect<A, E>,
7189
) =>
7290
Effect.gen(function* () {
7391
const bin = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-agent-scope-"));
7492
for (const name of ["systemd-run", "systemctl", "agent"]) {
7593
NodeFS.writeFileSync(NodePath.join(bin, name), "", { mode: 0o755 });
7694
}
77-
const units = new Map<string, { activeState: string; result: string }>();
95+
const units = new Map<string, FakeScopeState>();
7896
const calls: Array<ReadonlyArray<string>> = [];
97+
const handlers = new Map<
98+
string,
99+
(
100+
input: ProcessRunner.ProcessRunInput,
101+
) => Effect.Effect<ProcessRunner.ProcessRunOutput | undefined>
102+
>();
79103
const output = (stdout: string): ProcessRunner.ProcessRunOutput => ({
80104
stdout,
81105
stderr: "",
@@ -87,27 +111,46 @@ const withFakeSystemd = <A, E>(
87111
stderrInvalidUtf8: false,
88112
});
89113
const runner = ProcessRunner.ProcessRunner.of({
90-
run: ({ args }) =>
91-
Effect.sync(() => {
114+
run: (input) =>
115+
Effect.gen(function* () {
116+
const { args } = input;
92117
calls.push(args);
93-
const [, verb, unit = ""] = args;
118+
const [, verb = "", unit = ""] = args;
119+
const handled = yield* handlers.get(verb)?.(input) ?? Effect.succeed(undefined);
120+
if (handled !== undefined) return handled;
121+
if (verb === "stop") units.delete(unit);
94122
if (verb === "reset-failed") units.delete(unit);
95123
if (verb !== "show") return output("");
96124
const state = units.get(unit);
97125
return output(
98126
state === undefined
99127
? "LoadState=not-found\nActiveState=inactive\nResult=success\n"
100-
: `LoadState=loaded\nActiveState=${state.activeState}\nResult=${state.result}\n`,
128+
: `LoadState=loaded\nActiveState=${state.activeState}\nResult=${state.result}\nControlGroup=/t3-agent-scope-test/${unit}\n`,
101129
);
102130
}),
103131
});
132+
const fileSystem = yield* FileSystem.FileSystem;
104133
const scope = yield* make.pipe(
134+
Effect.provideService(FileSystem.FileSystem, {
135+
...fileSystem,
136+
readFileString: (path, encoding) => {
137+
if (!path.startsWith("/sys/fs/cgroup/t3-agent-scope-test/")) {
138+
return fileSystem.readFileString(path, encoding);
139+
}
140+
const state = units.get(NodePath.basename(NodePath.dirname(path)));
141+
return Effect.succeed(
142+
path.endsWith("memory.events")
143+
? `oom_kill ${state?.oomKills ?? 0}\n`
144+
: `populated ${state?.populated === false ? 0 : 1}\n`,
145+
);
146+
},
147+
}),
105148
Effect.provideService(ProcessRunner.ProcessRunner, runner),
106149
Effect.provideService(HostProcess.Platform, "linux"),
107150
Effect.provideService(HostProcess.Environment, { PATH: bin, XDG_RUNTIME_DIR: "/run/user/1" }),
108151
);
109152
const { stateDir } = yield* ServerConfig.ServerConfig;
110-
return yield* body({ scope, units, calls, stateDir }).pipe(
153+
return yield* body({ scope, units, calls, stateDir, handlers, output }).pipe(
111154
Effect.ensuring(Effect.sync(() => NodeFS.rmSync(bin, { recursive: true, force: true }))),
112155
);
113156
}).pipe(
@@ -126,6 +169,134 @@ const sliceOf = (launch: { readonly args: ReadonlyArray<string> }) =>
126169
launch.args.find((arg) => arg.startsWith("--slice="))?.slice("--slice=".length) ?? "";
127170

128171
describe("AgentScope service", () => {
172+
it.effect("waits for scope shutdown with a timeout longer than TimeoutStopSec", () =>
173+
withFakeSystemd(({ scope, units, calls, handlers, output }) =>
174+
Effect.gen(function* () {
175+
const launch = yield* scope.wrap({ command: "agent", args: [], name: "acp" });
176+
const unit = unitOf(launch);
177+
expect(launch.unit).toBe(unit);
178+
expect(launch.args).toContain("--property=TimeoutStopSec=10s");
179+
units.set(unit, { activeState: "active", result: "success", populated: true });
180+
const stopping = yield* Deferred.make<ProcessRunner.ProcessRunInput>();
181+
const stopped = yield* Deferred.make<void>();
182+
handlers.set("stop", (input) =>
183+
Deferred.succeed(stopping, input).pipe(
184+
Effect.andThen(Deferred.await(stopped)),
185+
Effect.andThen(Effect.sync(() => units.delete(unit))),
186+
Effect.as(output("")),
187+
),
188+
);
189+
const fiber = yield* scope.stop(unit).pipe(Effect.forkChild);
190+
expect((yield* Deferred.await(stopping)).timeout).toBe("15 seconds");
191+
expect(fiber.pollUnsafe()).toBeUndefined();
192+
yield* Deferred.succeed(stopped, undefined);
193+
yield* Fiber.join(fiber);
194+
expect(calls.filter(([, verb]) => verb === "stop")).toEqual([["--user", "stop", unit]]);
195+
}),
196+
),
197+
);
198+
199+
it.effect.each(["kernel OOM", "OOM result", "deactivating"] as const)(
200+
"awaits %s shutdown without replacing its diagnosis",
201+
(pending) =>
202+
withFakeSystemd(({ scope, units, calls, handlers }) =>
203+
Effect.gen(function* () {
204+
const launch = yield* scope.wrap({
205+
command: "agent",
206+
args: [],
207+
name: "acp",
208+
threadId: "a",
209+
});
210+
const unit = unitOf(launch);
211+
units.set(unit, {
212+
activeState: pending === "deactivating" ? "deactivating" : "active",
213+
result: pending === "OOM result" ? "oom-kill" : "success",
214+
oomKills: pending === "kernel OOM" ? 1 : 0,
215+
populated: true,
216+
});
217+
const waiting = yield* Deferred.make<void>();
218+
const completed = yield* Deferred.make<void>();
219+
let reads = 0;
220+
handlers.set("show", () =>
221+
++reads === 2
222+
? Deferred.succeed(waiting, undefined).pipe(
223+
Effect.andThen(Deferred.await(completed)),
224+
Effect.as(undefined),
225+
)
226+
: Effect.succeed(undefined),
227+
);
228+
const fiber = yield* scope.stop(unit).pipe(Effect.forkChild);
229+
yield* Deferred.await(waiting);
230+
expect(fiber.pollUnsafe()).toBeUndefined();
231+
expect(calls.filter(([, verb]) => verb === "stop")).toEqual([]);
232+
units.set(unit, { activeState: "failed", result: "oom-kill", populated: false });
233+
yield* Deferred.succeed(completed, undefined);
234+
yield* Fiber.join(fiber);
235+
expect(yield* scope.oomKilled("a")).toBe(true);
236+
expect(calls.filter(([, verb]) => verb === "stop")).toEqual([]);
237+
}),
238+
),
239+
);
240+
241+
it.effect("fails when pending OOM shutdown never finishes", () =>
242+
withFakeSystemd(({ scope, units, handlers }) =>
243+
Effect.gen(function* () {
244+
const launch = yield* scope.wrap({ command: "agent", args: [], name: "acp" });
245+
const unit = unitOf(launch);
246+
units.set(unit, { activeState: "active", result: "success", oomKills: 1, populated: true });
247+
const waiting = yield* Deferred.make<void>();
248+
let reads = 0;
249+
handlers.set("show", () =>
250+
++reads === 2
251+
? Deferred.succeed(waiting, undefined).pipe(Effect.andThen(Effect.never))
252+
: Effect.succeed(undefined),
253+
);
254+
const fiber = yield* scope.stop(unit).pipe(Effect.uninterruptible, Effect.forkChild);
255+
yield* Deferred.await(waiting);
256+
yield* TestClock.adjust("15 seconds");
257+
expect(yield* Effect.flip(Fiber.join(fiber))).toMatchObject({
258+
_tag: "AgentScopeStopError",
259+
detail: "Agent scope did not finish stopping",
260+
});
261+
}),
262+
),
263+
);
264+
265+
it.effect.each(["failure", "timeout"] as const)("reports a scope stop %s", (failure) =>
266+
withFakeSystemd(({ scope, units, handlers, output }) =>
267+
Effect.gen(function* () {
268+
const launch = yield* scope.wrap({ command: "agent", args: [], name: "acp" });
269+
const unit = unitOf(launch);
270+
units.set(unit, { activeState: "active", result: "success", populated: true });
271+
handlers.set("stop", () =>
272+
Effect.succeed({
273+
...output(""),
274+
code: ChildProcessSpawner.ExitCode(failure === "failure" ? 1 : 0),
275+
timedOut: failure === "timeout",
276+
}),
277+
);
278+
expect(yield* Effect.flip(scope.stop(unit))).toMatchObject({
279+
_tag: "AgentScopeStopError",
280+
unit,
281+
});
282+
}),
283+
),
284+
);
285+
286+
it.effect("does not report an unreadable scope as stopped", () =>
287+
withFakeSystemd(({ scope, handlers, output }) =>
288+
Effect.gen(function* () {
289+
const launch = yield* scope.wrap({ command: "agent", args: [], name: "acp" });
290+
handlers.set("show", () =>
291+
Effect.succeed({ ...output(""), code: ChildProcessSpawner.ExitCode(1) }),
292+
);
293+
expect(yield* Effect.flip(scope.stop(unitOf(launch)))).toMatchObject({
294+
_tag: "AgentScopeStopError",
295+
});
296+
}),
297+
),
298+
);
299+
129300
it.effect("labels a thread from its newest scope until its next session clears it", () =>
130301
withFakeSystemd(({ scope, units }) =>
131302
Effect.gen(function* () {

0 commit comments

Comments
 (0)