Skip to content

Commit 4ea32cb

Browse files
fix(server): load balancing reads container cgroup limits instead of the host's (pingdotgg#28261)
1 parent 25c5998 commit 4ea32cb

2 files changed

Lines changed: 337 additions & 8 deletions

File tree

Lines changed: 227 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,227 @@
1+
import { assert, describe, it } from "@effect/vitest";
2+
import type * as NodeOS from "node:os";
3+
import * as HostProcess from "@t3tools/shared/HostProcess";
4+
import * as Effect from "effect/Effect";
5+
import * as Fiber from "effect/Fiber";
6+
import * as FileSystem from "effect/FileSystem";
7+
import * as Layer from "effect/Layer";
8+
import * as PlatformError from "effect/PlatformError";
9+
import { ChildProcessSpawner } from "effect/process";
10+
import * as TestClock from "effect/testing/TestClock";
11+
import { vi } from "vite-plus/test";
12+
13+
import * as HostResources from "./HostResources.ts";
14+
15+
// An 8-CPU host; every cpus() call advances each CPU by 100 ms at `host.busy` utilization.
16+
const host = vi.hoisted(() => ({ busy: 0, reads: 0 }));
17+
vi.mock("node:os", async (importOriginal) => {
18+
const actual = await importOriginal<typeof NodeOS>();
19+
return {
20+
...actual,
21+
totalmem: () => 4 * 1024 ** 3,
22+
cpus: () => {
23+
const elapsed = 100 * host.reads++;
24+
const times = {
25+
user: elapsed * host.busy,
26+
nice: 0,
27+
sys: 0,
28+
idle: elapsed * (1 - host.busy),
29+
irq: 0,
30+
};
31+
return Array.from({ length: 8 }, () => ({ model: "test", speed: 1, times }));
32+
},
33+
};
34+
});
35+
const setup = (busy = 0) => {
36+
host.busy = busy;
37+
host.reads = 0;
38+
};
39+
40+
const GiB = 1024 ** 3;
41+
const MiB = 1024 ** 2;
42+
const cgroup = "/sys/fs/cgroup/t3.slice/server.scope";
43+
const unlimitedCgroup = {
44+
"/proc/self/cgroup": ["0::/t3.slice/server.scope\n"],
45+
"/proc/meminfo": [`MemTotal: ${4 * 1024 * 1024} kB\nMemAvailable: ${2 * 1024 * 1024} kB\n`],
46+
[`${cgroup}/cpu.max`]: ["max 100000\n"],
47+
[`${cgroup}/cpu.stat`]: ["usage_usec 5000000\n"],
48+
[`${cgroup}/memory.max`]: ["max\n"],
49+
[`${cgroup}/memory.current`]: [`${600 * MiB}\n`],
50+
[`${cgroup}/memory.stat`]: [`anon ${450 * MiB}\ninactive_file ${100 * MiB}\n`],
51+
};
52+
53+
// Each path serves its values in order and repeats the last one.
54+
const linuxHost = (files: Record<string, ReadonlyArray<string>>) => {
55+
const reads = new Map<string, number>();
56+
return HostResources.layer.pipe(
57+
Layer.provide(
58+
Layer.mergeAll(
59+
FileSystem.layerNoop({
60+
readFileString: (path) => {
61+
const values = files[path];
62+
if (!values) {
63+
return Effect.fail(
64+
PlatformError.systemError({
65+
_tag: "NotFound",
66+
module: "FileSystem",
67+
method: "readFileString",
68+
pathOrDescriptor: path,
69+
}),
70+
);
71+
}
72+
const index = reads.get(path) ?? 0;
73+
reads.set(path, index + 1);
74+
return Effect.succeed(values[Math.min(index, values.length - 1)]!);
75+
},
76+
}),
77+
Layer.succeed(HostProcess.Platform, "linux"),
78+
Layer.succeed(
79+
ChildProcessSpawner.ChildProcessSpawner,
80+
ChildProcessSpawner.make(() => Effect.die("HostResources spawned on linux")),
81+
),
82+
),
83+
),
84+
);
85+
};
86+
87+
const read = Effect.gen(function* () {
88+
const host = yield* HostResources.HostResources;
89+
const fiber = yield* Effect.forkChild(host.read);
90+
yield* TestClock.adjust("200 millis");
91+
return yield* Fiber.join(fiber);
92+
});
93+
94+
describe("HostResources.layer", () => {
95+
it.effect("reports the CPU quota and usage of the server's cgroup", () =>
96+
Effect.gen(function* () {
97+
setup();
98+
const snapshot = yield* read;
99+
100+
// One CPU of quota, 150 ms of it used during the 200 ms sample.
101+
assert.strictEqual(snapshot.cpuCount, 1);
102+
assert.strictEqual(snapshot.cpuUtilization, 0.75);
103+
}).pipe(
104+
Effect.provide(
105+
linuxHost({
106+
...unlimitedCgroup,
107+
[`${cgroup}/cpu.max`]: ["100000 100000\n"],
108+
[`${cgroup}/cpu.stat`]: [
109+
"usage_usec 5000000\nuser_usec 4000000\nsystem_usec 1000000\n",
110+
"usage_usec 5150000\nuser_usec 4100000\nsystem_usec 1050000\n",
111+
],
112+
}),
113+
),
114+
),
115+
);
116+
117+
it.effect("reports memory against the server cgroup's memory.max", () =>
118+
Effect.gen(function* () {
119+
setup();
120+
const snapshot = yield* read;
121+
122+
assert.strictEqual(snapshot.totalMemoryBytes, GiB);
123+
// 600 MiB charged, of which 100 MiB is reclaimable page cache.
124+
assert.strictEqual(snapshot.availableMemoryBytes, GiB - 500 * MiB);
125+
}).pipe(
126+
Effect.provide(
127+
linuxHost({
128+
...unlimitedCgroup,
129+
[`${cgroup}/memory.max`]: [`${GiB}\n`],
130+
}),
131+
),
132+
),
133+
);
134+
135+
it.effect("counts a busy host against an idle, quota-limited server cgroup", () =>
136+
Effect.gen(function* () {
137+
// Agents in sibling cgroups keep the host 90% busy; the server's own cgroup is idle.
138+
setup(0.9);
139+
const snapshot = yield* read;
140+
141+
assert.strictEqual(snapshot.cpuCount, 2);
142+
// Only 0.8 of 8 host CPUs are free, which is 60% of the 2-CPU quota in use.
143+
assert.closeTo(snapshot.cpuUtilization ?? Number.NaN, 0.6, 1e-9);
144+
}).pipe(
145+
Effect.provide(
146+
linuxHost({
147+
...unlimitedCgroup,
148+
[`${cgroup}/cpu.max`]: ["200000 100000\n"],
149+
}),
150+
),
151+
),
152+
);
153+
154+
it.effect("samples the CPUs the server is pinned to by id, not by os.cpus() index", () =>
155+
Effect.gen(function* () {
156+
setup();
157+
const snapshot = yield* read;
158+
159+
assert.strictEqual(snapshot.cpuCount, 1);
160+
assert.strictEqual(snapshot.cpuUtilization, 1);
161+
}).pipe(
162+
Effect.provide(
163+
linuxHost({
164+
...unlimitedCgroup,
165+
"/proc/self/status": ["Name:\tnode\nCpus_allowed_list:\t4\n"],
166+
// CPU 1 is offline; the server's CPU 4 is saturated, everything else is idle.
167+
"/proc/stat": [0, 100].map((elapsed) =>
168+
[0, 2, 3, 4, 5, 6, 7]
169+
.map((id) =>
170+
id === 4
171+
? `cpu${id} ${elapsed} 0 0 0 0 0 0 0`
172+
: `cpu${id} 0 0 0 ${elapsed} 0 0 0 0`,
173+
)
174+
.join("\n"),
175+
),
176+
}),
177+
),
178+
),
179+
);
180+
181+
it.effect("keeps host memory when the cgroup's memory.current is unreadable", () =>
182+
Effect.gen(function* () {
183+
setup();
184+
const snapshot = yield* read;
185+
186+
assert.strictEqual(snapshot.totalMemoryBytes, 4 * GiB);
187+
assert.strictEqual(snapshot.availableMemoryBytes, 2 * GiB);
188+
}).pipe(
189+
Effect.provide(
190+
linuxHost({
191+
...unlimitedCgroup,
192+
[`${cgroup}/memory.max`]: [`${GiB}\n`],
193+
[`${cgroup}/memory.current`]: [""],
194+
}),
195+
),
196+
),
197+
);
198+
199+
it.effect("keeps host values when the cgroup has no limits", () =>
200+
Effect.gen(function* () {
201+
setup(0.25);
202+
const snapshot = yield* read;
203+
204+
assert.strictEqual(snapshot.cpuCount, 8);
205+
assert.closeTo(snapshot.cpuUtilization ?? Number.NaN, 0.25, 1e-9);
206+
assert.strictEqual(snapshot.availableMemoryBytes, 2 * GiB);
207+
}).pipe(Effect.provide(linuxHost(unlimitedCgroup))),
208+
);
209+
210+
it.effect("falls back to host load when the cgroup's cpu.stat is unreadable", () =>
211+
Effect.gen(function* () {
212+
setup(0.9);
213+
const snapshot = yield* read;
214+
215+
assert.strictEqual(snapshot.cpuCount, 2);
216+
assert.closeTo(snapshot.cpuUtilization ?? Number.NaN, 0.6, 1e-9);
217+
}).pipe(
218+
Effect.provide(
219+
linuxHost({
220+
...unlimitedCgroup,
221+
[`${cgroup}/cpu.max`]: ["200000 100000\n"],
222+
[`${cgroup}/cpu.stat`]: [""],
223+
}),
224+
),
225+
),
226+
);
227+
});

‎apps/server/src/resourceTelemetry/HostResources.ts‎

Lines changed: 110 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import * as NodeOS from "node:os";
22
import type { HostResourcesSnapshot } from "@t3tools/contracts";
33
import * as HostProcess from "@t3tools/shared/HostProcess";
44
import * as Cache from "effect/Cache";
5+
import * as Clock from "effect/Clock";
56
import * as Context from "effect/Context";
67
import * as DateTime from "effect/DateTime";
78
import * as Effect from "effect/Effect";
@@ -14,6 +15,33 @@ export class HostResources extends Context.Service<
1415
{ readonly read: Effect.Effect<HostResourcesSnapshot> }
1516
>()("t3/resourceTelemetry/HostResources") {}
1617

18+
// Linux CPU list syntax, as in Cpus_allowed_list: "0-3,8,10-11".
19+
function parseCpuList(list: string): ReadonlySet<number> | null {
20+
const cpus = new Set<number>();
21+
for (const range of list.split(",")) {
22+
const match = /^(\d+)(?:-(\d+))?$/.exec(range.trim());
23+
if (!match) return null;
24+
for (let cpu = Number(match[1]); cpu <= Number(match[2] ?? match[1]); cpu++) cpus.add(cpu);
25+
}
26+
return cpus;
27+
}
28+
29+
// os.cpus() compacts online CPUs, so its indices are not CPU ids; /proc/stat lines are keyed.
30+
function pinnedCpu(stat: string, allowed: ReadonlySet<number>) {
31+
const cpu = { idle: 0, total: 0, count: 0 };
32+
for (const [, id, fields] of stat.matchAll(/^cpu(\d+) (.+)$/gm)) {
33+
if (!allowed.has(Number(id))) continue;
34+
const counters = (fields ?? "").trim().split(/\s+/).map(Number);
35+
const at = (index: number) => counters[index] ?? 0;
36+
// user nice system idle iowait irq softirq steal
37+
const idle = at(3) + at(4);
38+
cpu.idle += idle;
39+
cpu.total += idle + at(0) + at(1) + at(2) + at(5) + at(6) + at(7);
40+
cpu.count++;
41+
}
42+
return cpu.count > 0 ? cpu : null;
43+
}
44+
1745
function readCpu() {
1846
const cpus = NodeOS.cpus();
1947
const cpu = cpus.reduce(
@@ -37,28 +65,84 @@ function darwinAvailableMemory(output: string): number | null {
3765
return Number.isSafeInteger(available) && Number(pageSize) > 0 ? available : null;
3866
}
3967

68+
function cgroupNumber(text: string, pattern: RegExp): number | null {
69+
const value = pattern.exec(text)?.[1];
70+
return value === undefined ? null : Number(value);
71+
}
72+
4073
const make = Effect.fn("makeHostResources")(function* () {
4174
const fs = yield* FileSystem.FileSystem;
4275
const platform = yield* HostProcess.Platform;
4376
const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
4477

78+
const readFile = (path: string) => fs.readFileString(path).pipe(Effect.orElseSucceed(() => ""));
79+
4580
const sample = Effect.fn("HostResources.sample")(function* () {
46-
const previousCpu = readCpu();
81+
// cgroup v2 only; limits of the server's own cgroup, which is `/` inside a container.
82+
const cgroupPath =
83+
platform === "linux"
84+
? /^0::(\/.*)$/m.exec(yield* readFile("/proc/self/cgroup"))?.[1]
85+
: undefined;
86+
const cgroup =
87+
cgroupPath === undefined ? null : `/sys/fs/cgroup${cgroupPath.replace(/\/$/, "")}`;
88+
const cpuMax = cgroup
89+
? /^(\d+) (\d+)$/.exec((yield* readFile(`${cgroup}/cpu.max`)).trim())
90+
: null;
91+
const readCpuUsage = () =>
92+
cpuMax
93+
? Effect.map(readFile(`${cgroup}/cpu.stat`), (stat) =>
94+
cgroupNumber(stat, /^usage_usec (\d+)$/m),
95+
)
96+
: Effect.succeed(null);
97+
// Affinity (cpuset) pins the server to some CPUs; idle CPUs outside it are of no use.
98+
const allowedCpus =
99+
platform === "linux"
100+
? parseCpuList(
101+
/^Cpus_allowed_list:\s*(\S+)$/m.exec(yield* readFile("/proc/self/status"))?.[1] ?? "",
102+
)
103+
: null;
104+
105+
const sampleCpu = () =>
106+
allowedCpus
107+
? Effect.map(readFile("/proc/stat"), (stat) => pinnedCpu(stat, allowedCpus) ?? readCpu())
108+
: Effect.sync(readCpu);
109+
110+
const previousCpu = yield* sampleCpu();
111+
const previousUsage = yield* readCpuUsage();
112+
const startedAt = yield* Clock.currentTimeMillis;
47113
yield* Effect.sleep("200 millis");
48-
const cpu = readCpu();
114+
const cpu = yield* sampleCpu();
115+
const usage = yield* readCpuUsage();
116+
const elapsedMicros = ((yield* Clock.currentTimeMillis) - startedAt) * 1000;
49117
const totalDelta = cpu.total - previousCpu.total;
50118
const idleDelta = cpu.idle - previousCpu.idle;
51-
const cpuUtilization =
119+
let cpuCount = cpu.count;
120+
const hostUtilization =
52121
previousCpu.count === cpu.count && totalDelta > 0 && idleDelta >= 0
53122
? Math.min(1, Math.max(0, 1 - idleDelta / totalDelta))
54123
: null;
55-
const totalMemoryBytes = NodeOS.totalmem();
124+
let cpuUtilization = hostUtilization;
125+
if (cpuMax) {
126+
const capacity = Math.min(Number(cpuMax[1]) / Number(cpuMax[2]), cpu.count);
127+
cpuCount = Math.max(1, Math.floor(capacity));
128+
const cgroupIdle =
129+
previousUsage !== null && usage !== null && elapsedMicros > 0
130+
? capacity - (usage - previousUsage) / elapsedMicros
131+
: null;
132+
// Agents can run in sibling cgroups (agentScope.ts), so a busy host still counts.
133+
const hostIdle = hostUtilization === null ? null : cpu.count * (1 - hostUtilization);
134+
const idle =
135+
cgroupIdle === null || hostIdle === null
136+
? (cgroupIdle ?? hostIdle)
137+
: Math.min(cgroupIdle, hostIdle);
138+
cpuUtilization = idle === null ? null : Math.min(1, Math.max(0, 1 - idle / capacity));
139+
}
140+
141+
let totalMemoryBytes = NodeOS.totalmem();
56142
// On Windows libuv returns GlobalMemoryStatusEx.ullAvailPhys, including standby memory.
57143
let availableMemoryBytes = NodeOS.freemem();
58144
if (platform === "linux") {
59-
const meminfo = yield* fs
60-
.readFileString("/proc/meminfo")
61-
.pipe(Effect.orElseSucceed(() => ""));
145+
const meminfo = yield* readFile("/proc/meminfo");
62146
const available = /^MemAvailable:\s+(\d+)\s+kB$/m.exec(meminfo)?.[1];
63147
if (available) availableMemoryBytes = Number(available) * 1024;
64148
} else if (platform === "darwin") {
@@ -70,10 +154,28 @@ const make = Effect.fn("makeHostResources")(function* () {
70154
);
71155
availableMemoryBytes = darwinAvailableMemory(output) ?? availableMemoryBytes;
72156
}
157+
const memoryMax = cgroup
158+
? cgroupNumber(yield* readFile(`${cgroup}/memory.max`), /^(\d+)$/m)
159+
: null;
160+
const memoryCurrent =
161+
memoryMax === null
162+
? null
163+
: cgroupNumber(yield* readFile(`${cgroup}/memory.current`), /^(\d+)$/m);
164+
// Without usage, a capped total would read as a near-empty cgroup; keep host values instead.
165+
if (memoryMax !== null && memoryCurrent !== null) {
166+
totalMemoryBytes = Math.min(totalMemoryBytes, memoryMax);
167+
// inactive_file is page cache the kernel reclaims before hitting memory.max.
168+
const stat = yield* readFile(`${cgroup}/memory.stat`);
169+
const reclaimable = cgroupNumber(stat, /^inactive_file (\d+)$/m) ?? 0;
170+
availableMemoryBytes = Math.min(
171+
availableMemoryBytes,
172+
memoryMax - (memoryCurrent - reclaimable),
173+
);
174+
}
73175
return {
74176
sampledAt: DateTime.toEpochMillis(yield* DateTime.now),
75177
cpuUtilization,
76-
cpuCount: cpu.count,
178+
cpuCount,
77179
availableMemoryBytes: Math.min(totalMemoryBytes, Math.max(0, availableMemoryBytes)),
78180
totalMemoryBytes,
79181
};

0 commit comments

Comments
 (0)