Skip to content

Commit 84d7f72

Browse files
committed
fix(server): leave a scope alone while its OOM count can't be read
1 parent 773ef26 commit 84d7f72

2 files changed

Lines changed: 26 additions & 7 deletions

File tree

‎apps/server/src/process/agentScope.test.ts‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -146,18 +146,21 @@ const exitedAgent = (
146146
units: Map<string, FakeUnit>,
147147
cgroupRoot: string,
148148
unit: string,
149-
{ populated, oomKills }: { readonly populated: boolean; readonly oomKills: number },
149+
{ populated, oomKills }: { readonly populated: boolean; readonly oomKills: number | undefined },
150150
) => {
151151
const controlGroup = `/${unit}`;
152152
NodeFS.mkdirSync(NodePath.join(cgroupRoot, controlGroup), { recursive: true });
153153
NodeFS.writeFileSync(
154154
NodePath.join(cgroupRoot, controlGroup, "cgroup.events"),
155155
`populated ${populated ? 1 : 0}\nfrozen 0\n`,
156156
);
157-
NodeFS.writeFileSync(
158-
NodePath.join(cgroupRoot, controlGroup, "memory.events"),
159-
`low 0\nhigh 0\nmax 0\noom ${oomKills}\noom_kill ${oomKills}\noom_group_kill 0\n`,
160-
);
157+
// An undefined count leaves memory.events unreadable.
158+
if (oomKills !== undefined) {
159+
NodeFS.writeFileSync(
160+
NodePath.join(cgroupRoot, controlGroup, "memory.events"),
161+
`low 0\nhigh 0\nmax 0\noom ${oomKills}\noom_kill ${oomKills}\noom_group_kill 0\n`,
162+
);
163+
}
161164
units.set(unit, { activeState: "active", result: "success", controlGroup });
162165
};
163166

@@ -247,6 +250,17 @@ describe("AgentScope service", () => {
247250
),
248251
);
249252

253+
it.effect("leaves a scope alone while its OOM count can't be read", () =>
254+
withFakeSystemd(({ scope, units, calls, cgroupRoot }) =>
255+
Effect.gen(function* () {
256+
const agent = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
257+
exitedAgent(units, cgroupRoot, unitOf(agent), { populated: true, oomKills: undefined });
258+
yield* scope.stop(unitOf(agent));
259+
expect(stopsOf(calls)).toEqual([]);
260+
}),
261+
),
262+
);
263+
250264
it.effect("stops an OOM-killed scope itself when systemd has no OOMPolicy for scopes", () =>
251265
withFakeSystemd(
252266
({ scope, units, calls, cgroupRoot }) =>

‎apps/server/src/process/agentScope.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ export interface ScopeState {
5151
readonly activeState: string;
5252
readonly result: string;
5353
readonly oomKills: number;
54+
/** False when `memory.events` could not be read, so `oomKills` is unknown. */
55+
readonly oomKillsKnown?: boolean | undefined;
5456
readonly populated: boolean | undefined;
5557
}
5658

@@ -232,11 +234,13 @@ export const make = Effect.gen(function* () {
232234
const memoryEvents = controlGroup ? yield* readCgroupFile(controlGroup, "memory.events") : "";
233235
const cgroupEvents = controlGroup ? yield* readCgroupFile(controlGroup, "cgroup.events") : "";
234236
const populated = parseCgroupCounter(cgroupEvents, "populated");
237+
const oomKills = parseCgroupCounter(memoryEvents, "oom_kill");
235238
return {
236239
loadState: values.get("LoadState") ?? "",
237240
activeState: values.get("ActiveState") ?? "",
238241
result: values.get("Result") ?? "",
239-
oomKills: parseCgroupCounter(memoryEvents, "oom_kill") ?? 0,
242+
oomKills: oomKills ?? 0,
243+
oomKillsKnown: oomKills !== undefined,
240244
populated: populated === undefined ? undefined : populated === 1,
241245
} satisfies ScopeState;
242246
});
@@ -322,8 +326,9 @@ export const make = Effect.gen(function* () {
322326
// would lose its OOM label. An empty scope has nothing left to stop.
323327
// Without OOMPolicy=stop (systemd before 253) nothing else stops an
324328
// OOM-killed scope, and there is no label to keep, so it is stopped here.
329+
// With it, a scope whose OOM count can't be read is left alone.
325330
const state = yield* readScope(scope.systemctl, unit);
326-
if (state === undefined) return;
331+
if (state === undefined || (scope.oomPolicy && state.oomKillsKnown === false)) return;
327332
const status = classifyScope(state);
328333
const oomKilledWithoutPolicy =
329334
!scope.oomPolicy &&

0 commit comments

Comments
 (0)