Skip to content

Commit ff990dd

Browse files
feat(server): t3 service protect keeps the server alive when agents fill memory
Memory protection for the server only works when user.slice and the user manager grant it, and only root can set that. Until now the only setup that did was a hand-made drop-in set on one machine. `t3 service protect` shows the drop-ins it will write, asks, and writes the system ones through sudo: MemoryLow along user.slice, user-$UID.slice and user@$UID.service, and OOMScoreAdjust=-900 for the user manager and, after a separate question, tailscaled and sshd. As the user it writes app.slice, init.scope and t3code.service drop-ins and pins the manager's DefaultOOMScoreAdjust=200 first, so lowering the manager never leaves every user service at -800. `--undo` removes exactly its own files, and `t3 service status` reports the memory protection and OOM score the kernel actually applies. The unit sets OOMScoreAdjust=100, the lowest an unprivileged manager may. The preview browser now runs in an agent scope through a self-deleting launcher, so it inherits neither the server's protection nor its score. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent eb3287d commit ff990dd

10 files changed

Lines changed: 710 additions & 11 deletions

File tree

‎apps/server/src/cli/service.ts‎

Lines changed: 128 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,14 @@ import * as HostProcess from "@t3tools/shared/HostProcess";
22
import * as Console from "effect/Console";
33
import * as Effect from "effect/Effect";
44
import * as Layer from "effect/Layer";
5+
import * as Path from "effect/Path";
56
import * as Terminal from "effect/Terminal";
67
import { Command, Flag, GlobalFlag, Prompt } from "effect/cli";
78
import { FetchHttpClient } from "effect/http";
89

910
import packageJson from "../../package.json" with { type: "json" };
1011
import * as BootService from "../cloud/bootService.ts";
12+
import * as ServiceProtection from "../cloud/serviceProtection.ts";
1113
import { compareExactServiceVersions } from "../cloud/serviceProtocol.ts";
1214
import type * as ServerConfig from "../config.ts";
1315
import * as ProcessRunner from "../processRunner.ts";
@@ -68,7 +70,23 @@ function formatServiceChange(
6870
return `${result.previouslyInstalled ? "Updated" : "Installed"} T3 Code service with t3@${packageJson.version}.\nLogs: ${result.plan.logPath}`;
6971
}
7072

71-
function formatServiceStatus(status: BootService.BootServiceStatus, cliVersion: string): string {
73+
function formatProtection(
74+
protection: ServiceProtection.ServiceProtectionState | undefined,
75+
): ReadonlyArray<string> {
76+
if (protection === undefined) return [];
77+
return [
78+
protection.unprotected === undefined
79+
? ` Memory protection: on${protection.setByProtect ? " (t3 service protect)" : ""}`
80+
: ` Memory protection: off, ${protection.unprotected} grants none. Run \`t3 service protect\`.`,
81+
...(protection.oomScoreAdj === undefined ? [] : [` OOM score: ${protection.oomScoreAdj}`]),
82+
];
83+
}
84+
85+
function formatServiceStatus(
86+
status: BootService.BootServiceStatus,
87+
cliVersion: string,
88+
protection?: ServiceProtection.ServiceProtectionState,
89+
): string {
7290
if (!status.supported) {
7391
return "T3 Code service\n Status: unavailable on this machine\n Supported on: Linux with systemd, macOS with launchd";
7492
}
@@ -89,6 +107,7 @@ function formatServiceStatus(status: BootService.BootServiceStatus, cliVersion:
89107
` Status: installed · t3@${installedVersion} (newer than this t3@${cliVersion} CLI)`,
90108
` Unit: ${status.unitPath}`,
91109
` Logs: ${status.logPath}`,
110+
...formatProtection(protection),
92111
...problems,
93112
` Next: Run \`t3 update ${installedVersion}\` to match it, or pass \`--allow-downgrade\` to \`t3 service install\` explicitly.`,
94113
].join("\n");
@@ -98,6 +117,7 @@ function formatServiceStatus(status: BootService.BootServiceStatus, cliVersion:
98117
` Status: ${status.current ? `installed · t3@${installedVersion}` : "needs an update or repair"}`,
99118
` Unit: ${status.unitPath}`,
100119
` Logs: ${status.logPath}`,
120+
...formatProtection(protection),
101121
...problems,
102122
...(status.current ? [] : [" Next: Run `t3 service install` to repair it."]),
103123
].join("\n");
@@ -210,12 +230,117 @@ const serviceStatusCommand = Command.make("status", projectLocationFlags).pipe(
210230
flags,
211231
Effect.gen(function* () {
212232
const service = yield* BootService.BootService;
213-
yield* Console.log(formatServiceStatus(yield* service.status, packageJson.version));
233+
const path = yield* Path.Path;
234+
const status = yield* service.status;
235+
const protection =
236+
status.installed && (yield* HostProcess.Platform) === "linux"
237+
? yield* ServiceProtection.readProtection(path.dirname(status.unitPath)).pipe(
238+
Effect.provide(ProcessRunner.layer),
239+
)
240+
: undefined;
241+
yield* Console.log(formatServiceStatus(status, packageJson.version, protection));
214242
}),
215243
),
216244
),
217245
);
218246

247+
const formatDropIns = (dropIns: ReadonlyArray<ServiceProtection.DropIn>) =>
248+
dropIns.map(
249+
({ path, contents }) => ` ${path}: ${contents.trimEnd().split("\n").slice(1).join(" ")}`,
250+
);
251+
252+
const confirm = (yes: boolean, message: string) =>
253+
yes ? Effect.succeed(true) : Prompt.run(Prompt.Confirm({ message, initial: false }));
254+
255+
const serviceProtectCommand = Command.make("protect", {
256+
...projectLocationFlags,
257+
undo: Flag.Boolean("undo").pipe(
258+
Flag.withDescription("Remove the protection this command set up."),
259+
Flag.withDefault(false),
260+
),
261+
yes: Flag.Boolean("yes").pipe(
262+
Flag.withDescription("Answer yes to every question, including SSH and Tailscale."),
263+
Flag.withDefault(false),
264+
),
265+
}).pipe(
266+
Command.withDescription(
267+
"Protect the T3 Code server from agents under memory pressure (Linux with systemd). Asks for sudo.",
268+
),
269+
Command.withHandler((flags) =>
270+
runServiceCommand(
271+
flags,
272+
Effect.gen(function* () {
273+
const unsupported = "Service protection needs Linux with systemd.";
274+
if ((yield* HostProcess.Platform) !== "linux") return yield* Console.log(unsupported);
275+
const uid = yield* HostProcess.UserId;
276+
if (uid === 0) {
277+
return yield* Console.log(
278+
"Run `t3 service protect` as the user that runs T3 Code, without sudo. It asks for sudo when it needs root.",
279+
);
280+
}
281+
const remote = yield* ServiceProtection.detectRemoteAccessUnits;
282+
const { unitPath } = yield* (yield* BootService.BootService).status;
283+
if (uid === undefined || remote === undefined || unitPath === "") {
284+
return yield* Console.log(unsupported);
285+
}
286+
const userUnitDir = (yield* Path.Path).dirname(unitPath);
287+
288+
if (flags.undo) {
289+
const installed = yield* ServiceProtection.installedProtection(
290+
ServiceProtection.protectionPlan({
291+
uid,
292+
userUnitDir,
293+
remoteAccessUnits: ServiceProtection.REMOTE_ACCESS_UNITS,
294+
}),
295+
);
296+
if (installed === undefined) {
297+
return yield* Console.log("T3 Code service protection is not set up.");
298+
}
299+
yield* Console.log(
300+
[
301+
"Removes:",
302+
...[...installed.system, ...installed.user, installed.managerDefaults].map(
303+
({ path }) => ` ${path}`,
304+
),
305+
].join("\n"),
306+
);
307+
if (!(yield* confirm(flags.yes, "Remove T3 Code service protection?"))) return;
308+
yield* ServiceProtection.removeProtection(installed);
309+
return yield* Console.log(
310+
"Removed T3 Code service protection. The server's OOM score returns to normal after `t3 service restart`.",
311+
);
312+
}
313+
314+
const names = remote.map((unit) => unit.replace(/\.service$/, "")).join(" and ");
315+
const protectRemote =
316+
remote.length > 0 &&
317+
(yield* confirm(
318+
flags.yes,
319+
`Also protect ${names} so you can still reach this machine under memory pressure?`,
320+
));
321+
const plan = ServiceProtection.protectionPlan({
322+
uid,
323+
userUnitDir,
324+
remoteAccessUnits: protectRemote ? remote : [],
325+
});
326+
yield* Console.log(
327+
[
328+
"As root, through sudo:",
329+
...formatDropIns(plan.system),
330+
"As you:",
331+
...formatDropIns([plan.managerDefaults, ...plan.user]),
332+
].join("\n"),
333+
);
334+
if (!(yield* confirm(flags.yes, "Apply these settings?"))) return;
335+
yield* ServiceProtection.applyProtection(plan);
336+
yield* Console.log(
337+
"Protected T3 Code. Memory protection applies now; the server takes its OOM score after `t3 service restart`. Undo with `t3 service protect --undo`.",
338+
);
339+
}).pipe(Effect.provide(ProcessRunner.layer)),
340+
),
341+
),
342+
);
343+
219344
export type ServiceOnboardingOutcome = "ready" | "pending" | "skipped";
220345

221346
export const offerServiceDuringOnboarding = Effect.gen(function* () {
@@ -305,6 +430,7 @@ export const serviceCommand = Command.make("service").pipe(
305430
serviceRestartCommand,
306431
serviceUninstallCommand,
307432
serviceStatusCommand,
433+
serviceProtectCommand,
308434
serviceUpdateCommand,
309435
]),
310436
);

‎apps/server/src/cloud/bootService.test.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,13 @@ it("protects the server unit from agents under CPU and memory pressure", () => {
5656
unitPath: "/home/theo/.config/systemd/user/t3code.service",
5757
});
5858
const service = unit.slice(unit.indexOf("[Service]"), unit.indexOf("[Install]"));
59-
for (const line of ["CPUWeight=1000", "IOWeight=1000", "MemoryLow=1536M", "OOMPolicy=continue"]) {
59+
for (const line of [
60+
"CPUWeight=1000",
61+
"IOWeight=1000",
62+
"MemoryLow=1536M",
63+
"OOMScoreAdjust=100",
64+
"OOMPolicy=continue",
65+
]) {
6066
expect(service.split("\n")).toContain(line);
6167
}
6268
});

‎apps/server/src/cloud/bootService.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,9 @@ export function renderBootServiceUnit(plan: BootServicePlan): string {
116116
// Reclaim takes the agents' unprotected pages first. The kernel caps this at the
117117
// protection of user.slice and [email protected], which only root can grant.
118118
"MemoryLow=1536M",
119+
// A user manager's own score, the lowest it may set: below its other services (200) and agents (800).
120+
// `t3 service protect` lowers it further through a drop-in.
121+
"OOMScoreAdjust=100",
119122
// systemd-oomd ignores this preference when the watched cgroup is root-owned, as user-1000.slice usually is.
120123
"ManagedOOMPreference=avoid",
121124
`StandardOutput=append:${escapeSystemdSpecifiers(plan.logPath)}`,

0 commit comments

Comments
 (0)