Skip to content

Commit 4024eda

Browse files
committed
fix(server): build the CPU sample from the whole /proc/stat line
os.cpus() reports user, nice, sys, idle and irq only, so softirq and steal work was missing from the total and a host busy on interrupts read as idle. readCpu now parses the aggregate line itself on Linux: idle plus iowait as idle, user, nice, system, irq, softirq and steal as work. A line it cannot parse leaves the reading unavailable instead of silently off.
1 parent d1d4fa5 commit 4024eda

3 files changed

Lines changed: 72 additions & 42 deletions

File tree

‎apps/server/src/observability/EventLoopMonitor.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ const makeNodeSampler = Effect.gen(function* () {
8787

8888
return Effect.sync(() => {
8989
const nextCpu = readCpu(platform);
90+
const busy = cpu === null || nextCpu === null ? null : cpuBusyBetween(cpu, nextCpu);
9091
const cpuCount = NodeOS.availableParallelism();
9192
const nextElu = NodePerfHooks.performance.eventLoopUtilization();
9293
const nextUsage = process.resourceUsage();
@@ -113,8 +114,8 @@ const makeNodeSampler = Effect.gen(function* () {
113114
// Host-wide busy time says nothing about the cores an affinity mask or quota leaves us,
114115
// and a load average without disk wait leaves idle cores with no other cause to name.
115116
cpuBusy:
116-
loadAverageCountsDiskWait(platform) && nextCpu.count === cpuCount
117-
? (cpuBusyBetween(cpu, nextCpu) ?? undefined)
117+
loadAverageCountsDiskWait(platform) && nextCpu?.count === cpuCount
118+
? (busy ?? undefined)
118119
: undefined,
119120
cpuCount,
120121
};
Lines changed: 27 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,40 @@
11
import { assert, describe, it } from "@effect/vitest";
22

3-
import { cpuBusyBetween } from "./HostResources.ts";
3+
import { cpuBusyBetween, parseLinuxCpu } from "./HostResources.ts";
44

5-
const reading = (idle: number, total: number, count = 8, diskWait: number | null = 0) => ({
6-
idle,
7-
total,
8-
count,
9-
diskWait,
5+
const reading = (idle: number, total: number, count = 8) => ({ idle, total, count });
6+
7+
const parsed = (line: string) => {
8+
const cpu = parseLinuxCpu(line);
9+
if (cpu === null) throw new Error(`unparseable /proc/stat line: ${line}`);
10+
return { ...cpu, count: 8 };
11+
};
12+
13+
describe("parseLinuxCpu", () => {
14+
it("counts disk wait as idle and interrupts as work", () => {
15+
// cpu user nice system idle iowait irq softirq steal guest guest_nice
16+
assert.deepInclude(parseLinuxCpu("cpu 100 0 200 600 100 0 100 0 0 0\n"), {
17+
idle: 7_000,
18+
total: 11_000,
19+
});
20+
});
21+
22+
it("returns nothing for a line it cannot read", () => {
23+
assert.isNull(parseLinuxCpu("cpu 100 0 nonsense\n"));
24+
});
1025
});
1126

1227
describe("cpuBusyBetween", () => {
1328
it("is the share of core time not spent idle", () => {
1429
assert.strictEqual(cpuBusyBetween(reading(1_000, 2_000), reading(1_750, 3_000)), 0.25);
1530
});
1631

32+
it("counts softirq as busy rather than idle", () => {
33+
const before = parsed("cpu 0 0 0 0 0 0 0 0 0 0\n");
34+
const after = parsed("cpu 0 0 0 200 0 0 800 0 0 0\n");
35+
assert.strictEqual(cpuBusyBetween(before, after), 0.8);
36+
});
37+
1738
it("stays within 0 to 1 when counters move inconsistently", () => {
1839
assert.strictEqual(cpuBusyBetween(reading(1_000, 2_000), reading(2_100, 3_000)), 0);
1940
assert.isNull(cpuBusyBetween(reading(1_000, 2_000), reading(980, 3_000)));
@@ -23,9 +44,4 @@ describe("cpuBusyBetween", () => {
2344
it("is unknown when the core count changed between readings", () => {
2445
assert.isNull(cpuBusyBetween(reading(1_000, 2_000, 8), reading(1_500, 3_000, 7)));
2546
});
26-
27-
it("is unknown when disk wait could not be read", () => {
28-
const unknown = reading(1_000, 2_000, 8, null);
29-
assert.isNull(cpuBusyBetween(unknown, reading(1_200, 3_000)));
30-
});
3147
});

‎apps/server/src/resourceTelemetry/HostResources.ts‎

Lines changed: 42 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -16,18 +16,22 @@ export class HostResources extends Context.Service<
1616
{ readonly read: Effect.Effect<HostResourcesSnapshot> }
1717
>()("t3/resourceTelemetry/HostResources") {}
1818

19-
// os.cpus() drops iowait from idle and from the total, so a disk-blocked core reads as busy.
20-
// libuv turns the 100 Hz tick counters behind os.cpus() into milliseconds, hence the scale here.
21-
function readDiskWait(platform: NodeJS.Platform): number | null {
22-
if (platform !== "linux") return 0;
23-
try {
24-
const stat = NodeFS.readFileSync("/proc/stat", "utf8");
25-
// cpu user nice system idle iowait irq softirq steal guest guest_nice
26-
const ioWaitTicks = Number(stat.slice(0, stat.indexOf("\n")).split(/\s+/)[5]);
27-
return Number.isFinite(ioWaitTicks) ? ioWaitTicks * 10 : null;
28-
} catch {
29-
return null;
30-
}
19+
/**
20+
* Core time from a /proc/stat "cpu" line, in the milliseconds os.cpus() reports: disk wait
21+
* counts as idle, and softirq and steal as work. os.cpus() leaves all three out of the total,
22+
* so a core blocked on a disk or spent on interrupts reads as the wrong one of the two.
23+
*/
24+
export function parseLinuxCpu(stat: string): { idle: number; total: number } | null {
25+
const end = stat.indexOf("\n");
26+
// cpu user nice system idle iowait irq softirq steal guest guest_nice
27+
const ticks = (end === -1 ? stat : stat.slice(0, end)).split(/\s+/).slice(1, 9).map(Number);
28+
if (ticks.length !== 8 || !ticks.every(Number.isFinite)) return null;
29+
const [user = 0, nice = 0, system = 0, idle = 0, ioWait = 0, irq = 0, softirq = 0, steal = 0] =
30+
ticks;
31+
// libuv turns these 100 Hz counters into the milliseconds os.cpus() reports.
32+
const idleMs = (idle + ioWait) * 10;
33+
const busyMs = (user + nice + system + irq + softirq + steal) * 10;
34+
return { idle: idleMs, total: idleMs + busyMs };
3135
}
3236

3337
/**
@@ -36,29 +40,37 @@ function readDiskWait(platform: NodeJS.Platform): number | null {
3640
*/
3741
export const loadAverageCountsDiskWait = (platform: NodeJS.Platform) => platform === "linux";
3842

39-
export function readCpu(platform: NodeJS.Platform) {
43+
export interface CpuTimes {
44+
readonly idle: number;
45+
readonly total: number;
46+
readonly count: number;
47+
}
48+
49+
/** Cumulative core time, or nothing when this platform's counters cannot tell work from its absence. */
50+
export function readCpu(platform: NodeJS.Platform): CpuTimes | null {
4051
const cpus = NodeOS.cpus();
41-
const cpu = cpus.reduce(
42-
(sum, { times }) => ({
43-
idle: sum.idle + times.idle,
44-
total: sum.total + times.user + times.nice + times.sys + times.idle + times.irq,
45-
}),
46-
{ idle: 0, total: 0 },
47-
);
48-
const diskWait = readDiskWait(platform);
52+
if (platform === "linux") {
53+
try {
54+
const parsed = parseLinuxCpu(NodeFS.readFileSync("/proc/stat", "utf8"));
55+
return parsed === null ? null : { ...parsed, count: cpus.length };
56+
} catch {
57+
return null;
58+
}
59+
}
4960
return {
50-
idle: cpu.idle + (diskWait ?? 0),
51-
total: cpu.total + (diskWait ?? 0),
61+
...cpus.reduce(
62+
(sum, { times }) => ({
63+
idle: sum.idle + times.idle,
64+
total: sum.total + times.user + times.nice + times.sys + times.idle + times.irq,
65+
}),
66+
{ idle: 0, total: 0 },
67+
),
5268
count: cpus.length,
53-
diskWait,
5469
};
5570
}
5671

57-
type CpuTimes = ReturnType<typeof readCpu>;
58-
5972
/** Share of core time spent running work between two readings, null if they are not comparable. */
6073
export function cpuBusyBetween(previous: CpuTimes, next: CpuTimes): number | null {
61-
if (previous.diskWait === null || next.diskWait === null) return null;
6274
const totalDelta = next.total - previous.total;
6375
const idleDelta = next.idle - previous.idle;
6476
return previous.count === next.count && totalDelta > 0 && idleDelta >= 0
@@ -86,7 +98,8 @@ const make = Effect.fn("makeHostResources")(function* () {
8698
const previousCpu = readCpu(platform);
8799
yield* Effect.sleep("200 millis");
88100
const cpu = readCpu(platform);
89-
const cpuUtilization = cpuBusyBetween(previousCpu, cpu);
101+
const cpuUtilization =
102+
previousCpu === null || cpu === null ? null : cpuBusyBetween(previousCpu, cpu);
90103
const totalMemoryBytes = NodeOS.totalmem();
91104
// On Windows libuv returns GlobalMemoryStatusEx.ullAvailPhys, including standby memory.
92105
let availableMemoryBytes = NodeOS.freemem();
@@ -108,7 +121,7 @@ const make = Effect.fn("makeHostResources")(function* () {
108121
return {
109122
sampledAt: DateTime.toEpochMillis(yield* DateTime.now),
110123
cpuUtilization,
111-
cpuCount: cpu.count,
124+
cpuCount: cpu?.count ?? NodeOS.cpus().length,
112125
availableMemoryBytes: Math.min(totalMemoryBytes, Math.max(0, availableMemoryBytes)),
113126
totalMemoryBytes,
114127
};

0 commit comments

Comments
 (0)