Skip to content

Commit d1d4fa5

Browse files
committed
fix(server): count disk wait as idle in the CPU sample
os.cpus() leaves iowait out of both idle and the total, so a core blocked on a saturated disk reads as 100% busy. On the host that motivated the freeze notice that is backwards: the stall was disk wait. readCpu now adds /proc/stat's iowait, and reports no busy share when it cannot read it. cpuBusy is also Linux-only now. A load average above the core count means disk wait only on Linux; macOS counts runnable threads alone, so the I/O wording would have named a cause macOS cannot have.
1 parent b6d3f1e commit d1d4fa5

4 files changed

Lines changed: 63 additions & 12 deletions

File tree

‎apps/server/src/observability/EventLoopMonitor.ts‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import * as NodeOS from "node:os";
33
import * as NodePerfHooks from "node:perf_hooks";
44

55
import type { ServerEventLoopStall } from "@t3tools/contracts";
6+
import * as HostProcess from "@t3tools/shared/HostProcess";
67
import { RotatingFileSink } from "@t3tools/shared/logging";
78
import * as Context from "effect/Context";
89
import * as DateTime from "effect/DateTime";
@@ -16,7 +17,11 @@ import type * as Scope from "effect/Scope";
1617
import * as Semaphore from "effect/Semaphore";
1718

1819
import * as ServerConfig from "../config.ts";
19-
import { cpuBusyBetween, readCpu } from "../resourceTelemetry/HostResources.ts";
20+
import {
21+
cpuBusyBetween,
22+
loadAverageCountsDiskWait,
23+
readCpu,
24+
} from "../resourceTelemetry/HostResources.ts";
2025
import * as Metrics from "./Metrics.ts";
2126

2227
// Subtract the histogram's timer resolution; this can undercount stalls by up to RESOLUTION_MS.
@@ -59,13 +64,14 @@ export interface EventLoopReadings {
5964
readonly elapsedMs: number;
6065
/** One-minute load average, undefined on Windows. */
6166
readonly loadAverage: number | undefined;
62-
/** Share of core time spent running work, undefined unless every core is available to this process. */
67+
/** Share of core time spent running work over the sample, undefined when it cannot explain a high load. */
6368
readonly cpuBusy: number | undefined;
6469
readonly cpuCount: number;
6570
}
6671

6772
// Node skips the first gap after a reset, so a stall right at a sample boundary can be missed.
6873
const makeNodeSampler = Effect.gen(function* () {
74+
const platform = yield* HostProcess.Platform;
6975
const histogram = yield* Effect.acquireRelease(
7076
Effect.sync(() => {
7177
const histogram = NodePerfHooks.monitorEventLoopDelay({ resolution: RESOLUTION_MS });
@@ -77,10 +83,10 @@ const makeNodeSampler = Effect.gen(function* () {
7783
let elu = NodePerfHooks.performance.eventLoopUtilization();
7884
let usage = process.resourceUsage();
7985
let sampledAt = NodePerfHooks.performance.now();
80-
let cpu = readCpu();
86+
let cpu = readCpu(platform);
8187

8288
return Effect.sync(() => {
83-
const nextCpu = readCpu();
89+
const nextCpu = readCpu(platform);
8490
const cpuCount = NodeOS.availableParallelism();
8591
const nextElu = NodePerfHooks.performance.eventLoopUtilization();
8692
const nextUsage = process.resourceUsage();
@@ -104,8 +110,12 @@ const makeNodeSampler = Effect.gen(function* () {
104110
elapsedMs: nextSampledAt - sampledAt,
105111
// Windows has no load average and reports 0, which a running process never sees elsewhere.
106112
loadAverage: NodeOS.loadavg()[0] || undefined,
107-
// Host-wide busy time says nothing about the cores an affinity mask or quota leaves us.
108-
cpuBusy: nextCpu.count === cpuCount ? (cpuBusyBetween(cpu, nextCpu) ?? undefined) : undefined,
113+
// Host-wide busy time says nothing about the cores an affinity mask or quota leaves us,
114+
// and a load average without disk wait leaves idle cores with no other cause to name.
115+
cpuBusy:
116+
loadAverageCountsDiskWait(platform) && nextCpu.count === cpuCount
117+
? (cpuBusyBetween(cpu, nextCpu) ?? undefined)
118+
: undefined,
109119
cpuCount,
110120
};
111121
histogram.reset();

‎apps/server/src/resourceTelemetry/HostResources.test.ts‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,12 @@ import { assert, describe, it } from "@effect/vitest";
22

33
import { cpuBusyBetween } from "./HostResources.ts";
44

5-
const reading = (idle: number, total: number, count = 8) => ({ idle, total, count });
5+
const reading = (idle: number, total: number, count = 8, diskWait: number | null = 0) => ({
6+
idle,
7+
total,
8+
count,
9+
diskWait,
10+
});
611

712
describe("cpuBusyBetween", () => {
813
it("is the share of core time not spent idle", () => {
@@ -18,4 +23,9 @@ describe("cpuBusyBetween", () => {
1823
it("is unknown when the core count changed between readings", () => {
1924
assert.isNull(cpuBusyBetween(reading(1_000, 2_000, 8), reading(1_500, 3_000, 7)));
2025
});
26+
27+
it("is unknown when disk wait could not be read", () => {
28+
const unknown = reading(1_000, 2_000, 8, null);
29+
assert.isNull(cpuBusyBetween(unknown, reading(1_200, 3_000)));
30+
});
2131
});

‎apps/server/src/resourceTelemetry/HostResources.ts‎

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
// @effect-diagnostics nodeBuiltinImport:off - iowait exists only in /proc/stat, which must be read at the same instant as os.cpus() and has no Effect equivalent.
2+
import * as NodeFS from "node:fs";
13
import * as NodeOS from "node:os";
24
import type { HostResourcesSnapshot } from "@t3tools/contracts";
35
import * as HostProcess from "@t3tools/shared/HostProcess";
@@ -14,7 +16,27 @@ export class HostResources extends Context.Service<
1416
{ readonly read: Effect.Effect<HostResourcesSnapshot> }
1517
>()("t3/resourceTelemetry/HostResources") {}
1618

17-
export function readCpu() {
19+
// os.cpus() drops iowait from idle and from the total, so a disk-blocked core reads as busy.
20+
// libuv turns the 100 Hz tick counters behind os.cpus() into milliseconds, hence the scale here.
21+
function readDiskWait(platform: NodeJS.Platform): number | null {
22+
if (platform !== "linux") return 0;
23+
try {
24+
const stat = NodeFS.readFileSync("/proc/stat", "utf8");
25+
// cpu user nice system idle iowait irq softirq steal guest guest_nice
26+
const ioWaitTicks = Number(stat.slice(0, stat.indexOf("\n")).split(/\s+/)[5]);
27+
return Number.isFinite(ioWaitTicks) ? ioWaitTicks * 10 : null;
28+
} catch {
29+
return null;
30+
}
31+
}
32+
33+
/**
34+
* A load average above the core count can mean disk wait on Linux alone:
35+
* macOS counts runnable threads only, and Windows reports no load average.
36+
*/
37+
export const loadAverageCountsDiskWait = (platform: NodeJS.Platform) => platform === "linux";
38+
39+
export function readCpu(platform: NodeJS.Platform) {
1840
const cpus = NodeOS.cpus();
1941
const cpu = cpus.reduce(
2042
(sum, { times }) => ({
@@ -23,13 +45,20 @@ export function readCpu() {
2345
}),
2446
{ idle: 0, total: 0 },
2547
);
26-
return { ...cpu, count: cpus.length };
48+
const diskWait = readDiskWait(platform);
49+
return {
50+
idle: cpu.idle + (diskWait ?? 0),
51+
total: cpu.total + (diskWait ?? 0),
52+
count: cpus.length,
53+
diskWait,
54+
};
2755
}
2856

2957
type CpuTimes = ReturnType<typeof readCpu>;
3058

3159
/** Share of core time spent running work between two readings, null if they are not comparable. */
3260
export function cpuBusyBetween(previous: CpuTimes, next: CpuTimes): number | null {
61+
if (previous.diskWait === null || next.diskWait === null) return null;
3362
const totalDelta = next.total - previous.total;
3463
const idleDelta = next.idle - previous.idle;
3564
return previous.count === next.count && totalDelta > 0 && idleDelta >= 0
@@ -54,9 +83,9 @@ const make = Effect.fn("makeHostResources")(function* () {
5483
const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
5584

5685
const sample = Effect.fn("HostResources.sample")(function* () {
57-
const previousCpu = readCpu();
86+
const previousCpu = readCpu(platform);
5887
yield* Effect.sleep("200 millis");
59-
const cpu = readCpu();
88+
const cpu = readCpu(platform);
6089
const cpuUtilization = cpuBusyBetween(previousCpu, cpu);
6190
const totalMemoryBytes = NodeOS.totalmem();
6291
// On Windows libuv returns GlobalMemoryStatusEx.ullAvailPhys, including standby memory.

‎packages/contracts/src/server.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -605,7 +605,9 @@ export const ServerEventLoopStall = Schema.Struct({
605605
loadAverage: Schema.optionalKey(Schema.Number),
606606
/**
607607
* Share of all cores' time spent running work over the sample that saw the
608-
* stall. Load counts tasks blocked on I/O too, so this tells CPU from I/O.
608+
* stall, with disk wait counted as idle. Load counts tasks blocked on I/O
609+
* too, so on Linux this tells CPU from I/O. Absent elsewhere, and absent when
610+
* every core is not available to this process.
609611
*/
610612
cpuBusy: Schema.optionalKey(Schema.Number.check(Schema.isBetween({ minimum: 0, maximum: 1 }))),
611613
cpuCount: PositiveInt,

0 commit comments

Comments
 (0)