Skip to content

Commit 694e82f

Browse files
committed
fix(server): skip the busy share when a cgroup quota limits us
availableParallelism() rounds a fractional quota up, so a container held to half a core on one vCPU still reports one core and passes the core-count check. Host-wide busy time then looks idle while the container is being throttled, and the notice blamed the disk. The share is now emitted only when this process's own cgroup sets no quota below the core count, read through /proc/self/cgroup the way libuv resolves it.
1 parent 4024eda commit 694e82f

3 files changed

Lines changed: 65 additions & 2 deletions

File tree

‎apps/server/src/observability/EventLoopMonitor.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import * as Semaphore from "effect/Semaphore";
1919
import * as ServerConfig from "../config.ts";
2020
import {
2121
cpuBusyBetween,
22+
cpuLimitCoversHost,
2223
loadAverageCountsDiskWait,
2324
readCpu,
2425
} from "../resourceTelemetry/HostResources.ts";
@@ -114,7 +115,9 @@ const makeNodeSampler = Effect.gen(function* () {
114115
// Host-wide busy time says nothing about the cores an affinity mask or quota leaves us,
115116
// and a load average without disk wait leaves idle cores with no other cause to name.
116117
cpuBusy:
117-
loadAverageCountsDiskWait(platform) && nextCpu?.count === cpuCount
118+
loadAverageCountsDiskWait(platform) &&
119+
nextCpu?.count === cpuCount &&
120+
cpuLimitCoversHost(cpuCount)
118121
? (busy ?? undefined)
119122
: undefined,
120123
cpuCount,

‎apps/server/src/resourceTelemetry/HostResources.test.ts‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { assert, describe, it } from "@effect/vitest";
22

3-
import { cpuBusyBetween, parseLinuxCpu } from "./HostResources.ts";
3+
import { cpuBusyBetween, cpuQuotaCores, parseLinuxCpu } from "./HostResources.ts";
44

55
const reading = (idle: number, total: number, count = 8) => ({ idle, total, count });
66

@@ -24,6 +24,18 @@ describe("parseLinuxCpu", () => {
2424
});
2525
});
2626

27+
describe("cpuQuotaCores", () => {
28+
it("is the fraction of a core a quota allows", () => {
29+
assert.strictEqual(cpuQuotaCores(50_000, 100_000), 0.5);
30+
assert.strictEqual(cpuQuotaCores(400_000, 100_000), 4);
31+
});
32+
33+
it("is nothing when no quota applies", () => {
34+
assert.isNull(cpuQuotaCores(-1, 100_000));
35+
assert.isNull(cpuQuotaCores(50_000, 0));
36+
});
37+
});
38+
2739
describe("cpuBusyBetween", () => {
2840
it("is the share of core time not spent idle", () => {
2941
assert.strictEqual(cpuBusyBetween(reading(1_000, 2_000), reading(1_750, 3_000)), 0.25);

‎apps/server/src/resourceTelemetry/HostResources.ts‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,54 @@ export function parseLinuxCpu(stat: string): { idle: number; total: number } | n
4040
*/
4141
export const loadAverageCountsDiskWait = (platform: NodeJS.Platform) => platform === "linux";
4242

43+
/** Cores a cgroup CPU quota allows, or null when it sets none. */
44+
export function cpuQuotaCores(quota: number, period: number): number | null {
45+
return quota > 0 && period > 0 ? quota / period : null;
46+
}
47+
48+
const readIfPresent = (path: string) => {
49+
try {
50+
return NodeFS.readFileSync(path, "utf8");
51+
} catch {
52+
return undefined;
53+
}
54+
};
55+
56+
/** A controller file from this process's own cgroup, which is not the mount root under systemd. */
57+
function selfCgroupFile(controller: string, file: string): string | undefined {
58+
for (const line of (readIfPresent("/proc/self/cgroup") ?? "").split("\n")) {
59+
const [, controllers = "", path = ""] = line.split(":");
60+
// A cgroup v2 line names no controllers; a v1 line lists them in no fixed order.
61+
const hierarchy = controllers === "" ? ["unified"] : controllers.split(",");
62+
if (!hierarchy.includes(controller)) continue;
63+
const found = readIfPresent(`/sys/fs/cgroup${path}/${file}`);
64+
if (found !== undefined) return found;
65+
}
66+
return undefined;
67+
}
68+
69+
// availableParallelism() rounds a fractional quota up, so a container held to half a core on a
70+
// single-core host still reports one, and host-wide busy time looks idle while it is throttled.
71+
function cgroupCpuQuotaCores(): number | null {
72+
const v2 = selfCgroupFile("unified", "cpu.max") ?? readIfPresent("/sys/fs/cgroup/cpu.max");
73+
if (v2 !== undefined) {
74+
const [quota, period] = v2.trim().split(/\s+/);
75+
return quota === undefined || quota === "max"
76+
? null
77+
: cpuQuotaCores(Number(quota), Number(period));
78+
}
79+
const cpu = (file: string) => selfCgroupFile("cpu", file);
80+
const quota = cpu("cpu.cfs_quota_us");
81+
if (quota === undefined) return null;
82+
return cpuQuotaCores(Number(quota), Number(cpu("cpu.cfs_period_us") ?? "100000"));
83+
}
84+
85+
/** Whether host-wide core time describes what this process can use. */
86+
export const cpuLimitCoversHost = (cores: number) => {
87+
const quota = cgroupCpuQuotaCores();
88+
return quota === null || quota >= cores;
89+
};
90+
4391
export interface CpuTimes {
4492
readonly idle: number;
4593
readonly total: number;

0 commit comments

Comments
 (0)