Skip to content

Commit d7b6cf7

Browse files
fix(server): agents yield CPU, tasks, and reclaim to the server under pressure (pingdotgg#27176)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent f58fad0 commit d7b6cf7

4 files changed

Lines changed: 72 additions & 14 deletions

File tree

‎apps/server/src/cloud/bootService.test.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,19 @@ it("reads the served T3 home back out of a rendered unit or plist", () => {
4848
expect(BootService.bootServiceBaseDirOf("[Service]\nExecStart=/x\n")).toBeUndefined();
4949
});
5050

51+
it("protects the server unit from agents under CPU and memory pressure", () => {
52+
const unit = BootService.renderBootServiceUnit({
53+
program: ["/home/theo/.t3/runtime/versions/1.2.3/t3", "__service-launcher"],
54+
baseDir: "/home/theo/.t3",
55+
logPath: "/home/theo/.t3/userdata/logs/boot-service.log",
56+
unitPath: "/home/theo/.config/systemd/user/t3code.service",
57+
});
58+
const service = unit.slice(unit.indexOf("[Service]"), unit.indexOf("[Install]"));
59+
for (const line of ["CPUWeight=1000", "IOWeight=1000", "MemoryLow=1536M", "OOMPolicy=continue"]) {
60+
expect(service.split("\n")).toContain(line);
61+
}
62+
});
63+
5164
it.each([
5265
["a unit", '/home/theo/T3 "Data"/100%\\x', BootService.renderBootServiceUnit],
5366
[

‎apps/server/src/cloud/bootService.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,9 +110,12 @@ export function renderBootServiceUnit(plan: BootServicePlan): string {
110110
"OOMPolicy=continue",
111111
"Restart=always",
112112
"RestartSec=5",
113-
// Weights favor the server over the sibling agents slice during CPU or disk saturation.
113+
// Weights favor the server over the sibling agents slice; IOWeight needs a delegated io controller.
114114
"CPUWeight=1000",
115115
"IOWeight=1000",
116+
// Reclaim takes the agents' unprotected pages first. The kernel caps this at the
117+
// protection of user.slice and [email protected], which only root can grant.
118+
"MemoryLow=1536M",
116119
// systemd-oomd ignores this preference when the watched cgroup is root-owned, as user-1000.slice usually is.
117120
"ManagedOOMPreference=avoid",
118121
`StandardOutput=append:${escapeSystemdSpecifiers(plan.logPath)}`,

‎apps/server/src/process/agentScope.test.ts‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -245,6 +245,34 @@ describe("AgentScope service", () => {
245245
),
246246
);
247247

248+
it.effect("caps agent memory and yields CPU, disk, and tasks to the server", () =>
249+
withFakeSystemd(({ scope, calls }) =>
250+
Effect.gen(function* () {
251+
yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
252+
const setProperties = calls.filter(([, verb]) => verb === "set-property");
253+
expect(setProperties).toEqual([
254+
[
255+
"--user",
256+
"set-property",
257+
"--runtime",
258+
"app-t3code-agents.slice",
259+
expect.stringMatching(/^MemoryHigh=\d+$/),
260+
expect.stringMatching(/^MemoryMax=\d+$/),
261+
],
262+
[
263+
"--user",
264+
"set-property",
265+
"--runtime",
266+
"app-t3code.slice",
267+
"CPUWeight=10",
268+
"IOWeight=10",
269+
"TasksMax=25%",
270+
],
271+
]);
272+
}),
273+
),
274+
);
275+
248276
it.effect("stops agents a killed server left before the first launch", () =>
249277
withFakeSystemd(({ scope, units, calls, stateDir }) =>
250278
Effect.gen(function* () {

‎apps/server/src/process/agentScope.ts‎

Lines changed: 27 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,12 @@ import * as ServerConfig from "../config.ts";
1919
import * as ProcessRunner from "../processRunner.ts";
2020

2121
const AGENT_SLICE = "app-t3code-agents.slice";
22+
// Owns every agent slice and competes with the server's unit or scope under app.slice.
23+
const AGENTS_PARENT_SLICE = "app-t3code.slice";
24+
// Low weights yield CPU and disk to the server under contention, where the user manager
25+
// has those controllers: current systemd delegates cpu, rarely io. TasksMax keeps agents
26+
// below logind's 33% per-user task limit, leaving forks for the server.
27+
const AGENTS_PARENT_PROPERTIES = ["CPUWeight=10", "IOWeight=10", "TasksMax=25%"];
2228

2329
export function ownedAgentSlice(stateDir: string): string {
2430
const hash = NodeCrypto.createHash("sha256").update(stateDir).digest("hex").slice(0, 12);
@@ -173,20 +179,28 @@ export const make = Effect.gen(function* () {
173179
yield* Effect.logInfo("Agent scopes are off: no systemd user manager");
174180
return undefined;
175181
}
182+
const setProperties = Effect.fn("agentScope.setProperties")(function* (
183+
slice: string,
184+
properties: ReadonlyArray<string>,
185+
) {
186+
const result = yield* run(systemctl, [
187+
"--user",
188+
"set-property",
189+
"--runtime",
190+
slice,
191+
...properties,
192+
]);
193+
if (result._tag === "None" || result.value.code !== 0) {
194+
yield* Effect.logWarning("Could not set resource limits on an agents slice", {
195+
slice,
196+
properties,
197+
stderr: result._tag === "Some" ? result.value.stderr.trim() : undefined,
198+
});
199+
}
200+
});
176201
const limits = agentSliceMemoryLimits(NodeOS.totalmem());
177-
const limited = yield* run(systemctl, [
178-
"--user",
179-
"set-property",
180-
"--runtime",
181-
AGENT_SLICE,
182-
`MemoryHigh=${limits.high}`,
183-
`MemoryMax=${limits.max}`,
184-
]);
185-
if (limited._tag === "None" || limited.value.code !== 0) {
186-
yield* Effect.logWarning("Could not set memory limits on the agents slice", {
187-
stderr: limited._tag === "Some" ? limited.value.stderr.trim() : undefined,
188-
});
189-
}
202+
yield* setProperties(AGENT_SLICE, [`MemoryHigh=${limits.high}`, `MemoryMax=${limits.max}`]);
203+
yield* setProperties(AGENTS_PARENT_SLICE, AGENTS_PARENT_PROPERTIES);
190204
yield* Effect.logInfo("Agent scopes are on", {
191205
slice: AGENT_SLICE,
192206
...limits,

0 commit comments

Comments
 (0)