Skip to content

Commit 8c5f25b

Browse files
committed
feat: resource queue shows every environment, with disk usage; composer gauge draws full rings
Build-Task-Run: 2026-11-04-multi-env-resource-queue
1 parent d41c140 commit 8c5f25b

28 files changed

Lines changed: 4993 additions & 938 deletions

‎apps/server/src/diagnostics/HostDisks.test.ts‎

Lines changed: 988 additions & 0 deletions
Large diffs are not rendered by default.
Lines changed: 315 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,315 @@
1+
import type { HostMetricsDisk } from "@t3tools/contracts";
2+
3+
/**
4+
* Shared disk-usage reader for host metrics. A hung statfs pins a libuv pool thread and cannot
5+
* be cancelled, so at most {@link MAX_IN_FLIGHT} reads are in flight and a source in flight is
6+
* never read again until it settles.
7+
*/
8+
9+
/** Never pins more than 2 of the 4 libuv pool threads. */
10+
const MAX_IN_FLIGHT = 2;
11+
/** Refreshes are at least this far apart (monotonic clock, from the last refresh start). */
12+
const REFRESH_MS = 10_000;
13+
/** A read in flight longer than this is reported stuck, once per episode. */
14+
const STUCK_MS = 30_000;
15+
/** Local block-device filesystems worth reporting. */
16+
const ALLOWED_TYPES: ReadonlySet<string> = new Set([
17+
"ext2",
18+
"ext3",
19+
"ext4",
20+
"xfs",
21+
"btrfs",
22+
"f2fs",
23+
"exfat",
24+
"ntfs3",
25+
]);
26+
const DARWIN_DATA = "/System/Volumes/Data";
27+
28+
export interface MountRow {
29+
readonly source: string;
30+
/** Mount point shown to the user: escapes decoded, control characters dropped. */
31+
readonly mount: string;
32+
readonly type: string;
33+
/** The real mount point, handed to statfs; identifies the mount. */
34+
readonly path: string;
35+
}
36+
37+
export interface StatFsLike {
38+
readonly bsize: number;
39+
readonly blocks: number;
40+
readonly bfree: number;
41+
readonly bavail: number;
42+
}
43+
44+
export interface HostDisksDeps {
45+
readonly statfs: (path: string) => Promise<StatFsLike>;
46+
/** A mount point that is not a directory (a bind-mounted file) is not a disk. */
47+
readonly stat: (path: string) => Promise<{ isDirectory(): boolean }>;
48+
readonly readMounts: () => string;
49+
/** Monotonic milliseconds; drives every timer. */
50+
readonly monoNow: () => number;
51+
/** Epoch milliseconds; stamped on each reading as `readAt`. */
52+
readonly epochNow: () => number;
53+
readonly log: {
54+
warn(msg: string, data: object): void;
55+
info(msg: string, data: object): void;
56+
};
57+
readonly platform: NodeJS.Platform;
58+
readonly exists: (path: string) => boolean;
59+
}
60+
61+
export interface HostDisks {
62+
/** Readings in mount-table order; undefined until a mount table has been parsed once. */
63+
snapshot(): ReadonlyArray<HostMetricsDisk> | undefined;
64+
}
65+
66+
/** `/proc/self/mounts` encodes space, tab, newline and backslash as `\ooo`. */
67+
const decodeOctal = (field: string) =>
68+
field.replace(/\\([0-7]{3})/g, (_, octal: string) => String.fromCharCode(parseInt(octal, 8)));
69+
70+
const CONTROL_CHARS = /[\u0000-\u001f\u007f]/g;
71+
72+
/**
73+
* Allowlisted local block devices from a `/proc/self/mounts` text, in table order. Of all rows
74+
* stacked on one mount point only the last (the visible top) counts, whatever its type; then the
75+
* allowlist applies. A source can have several rows; the reader picks the first directory.
76+
*/
77+
export function parseMounts(text: string): ReadonlyArray<MountRow> {
78+
const parsed: Array<{ source: string; rawMount: string; type: string; path: string }> = [];
79+
for (const line of text.split("\n")) {
80+
const [source, rawMount, type] = line.split(" ");
81+
if (source === undefined || rawMount === undefined || type === undefined) continue;
82+
parsed.push({ source, rawMount, type, path: decodeOctal(rawMount) });
83+
}
84+
const top = new Map(parsed.map((row) => [row.path, row]));
85+
const rows: MountRow[] = [];
86+
for (const row of parsed) {
87+
if (top.get(row.path) !== row) continue;
88+
if (!ALLOWED_TYPES.has(row.type)) continue;
89+
if (!row.source.startsWith("/dev/") || row.source.startsWith("/dev/loop")) continue;
90+
rows.push({
91+
source: row.source,
92+
mount: row.path.replace(CONTROL_CHARS, ""),
93+
type: row.type,
94+
path: row.path,
95+
});
96+
}
97+
return rows;
98+
}
99+
100+
function toDisk(mount: string, s: StatFsLike, readAt: number): HostMetricsDisk {
101+
const usedBytes = (s.blocks - s.bfree) * s.bsize;
102+
const availBytes = s.bavail * s.bsize;
103+
const denominator = usedBytes + availBytes;
104+
return {
105+
mount,
106+
usedBytes,
107+
totalBytes: s.blocks * s.bsize,
108+
pct: denominator > 0 ? (usedBytes / denominator) * 100 : 0,
109+
readAt,
110+
};
111+
}
112+
113+
const errorCode = (error: unknown): string => {
114+
const code = (error as { code?: unknown } | null)?.code;
115+
return typeof code === "string" ? code : "unknown";
116+
};
117+
const errorMessage = (error: unknown): string =>
118+
error instanceof Error ? error.message : String(error);
119+
120+
export function makeHostDisks(deps: HostDisksDeps): HostDisks {
121+
const { statfs, stat, monoNow, epochNow, log } = deps;
122+
/** Last reading per source, with the path it was taken at. */
123+
const cache = new Map<string, { readonly path: string; readonly disk: HostMetricsDisk }>();
124+
/** Each source's mount rows, in table order. */
125+
let rows = new Map<string, ReadonlyArray<MountRow>>();
126+
/** `row` is the candidate being read now. */
127+
const inFlight = new Map<string, { row: MountRow; readonly startedAt: number }>();
128+
/** Sources waiting for a slot, in mount-table order; never holds an in-flight source. */
129+
const queue = new Map<string, ReadonlyArray<MountRow>>();
130+
/** The row each stuck read was reported on, so its recovery names the same row. */
131+
const stuckLogged = new Map<string, MountRow>();
132+
/** Error codes already logged per source; cleared by a successful read. */
133+
const failureLogged = new Map<string, Set<string>>();
134+
let parsed = false;
135+
let mountsFailing = false;
136+
let saturated = false;
137+
/** Reads that settled since the last refresh after holding their slot a whole period. */
138+
let slowSettled = 0;
139+
let lastRefresh = 0;
140+
141+
function readRows(): ReadonlyArray<MountRow> | undefined {
142+
if (deps.platform === "linux") return parseMounts(deps.readMounts());
143+
if (deps.platform === "darwin") {
144+
const path = deps.exists(DARWIN_DATA) ? DARWIN_DATA : "/";
145+
return [{ source: "darwin-data", mount: "/", type: "apfs", path }];
146+
}
147+
return undefined;
148+
}
149+
150+
/** A source's mounts include this path. */
151+
const mountedAt = (source: string, path: string) =>
152+
rows.get(source)?.some((row) => row.path === path) === true;
153+
154+
function refresh() {
155+
lastRefresh = monoNow();
156+
try {
157+
const next = readRows();
158+
if (next === undefined) return;
159+
rows = new Map();
160+
for (const row of next) rows.set(row.source, [...(rows.get(row.source) ?? []), row]);
161+
parsed = true;
162+
mountsFailing = false;
163+
} catch (error) {
164+
if (!mountsFailing) {
165+
mountsFailing = true;
166+
log.warn("disk mount table unreadable", {
167+
code: errorCode(error),
168+
error: errorMessage(error),
169+
});
170+
}
171+
}
172+
// A reading is shown only under the mount it was taken at.
173+
for (const [source, entry] of cache) {
174+
if (!mountedAt(source, entry.path)) cache.delete(source);
175+
}
176+
for (const source of queue.keys()) if (!rows.has(source)) queue.delete(source);
177+
for (const [source, candidates] of rows) {
178+
const running = inFlight.get(source);
179+
if (running === undefined) queue.set(source, candidates);
180+
else warnIfStuck(source, running.row, running.startedAt);
181+
}
182+
reportSaturation();
183+
pump();
184+
}
185+
186+
function warnIfStuck(source: string, row: MountRow, startedAt: number) {
187+
const stuckMs = monoNow() - startedAt;
188+
if (stuckMs <= STUCK_MS || stuckLogged.has(source)) return;
189+
stuckLogged.set(source, row);
190+
log.warn("disk read stuck", {
191+
mount: row.mount,
192+
type: row.type,
193+
stuckMs,
194+
inFlight: inFlight.size,
195+
});
196+
}
197+
198+
function pump() {
199+
while (inFlight.size < MAX_IN_FLIGHT) {
200+
const next = queue.entries().next();
201+
if (next.done === true) break;
202+
const [source, candidates] = next.value;
203+
queue.delete(source);
204+
start(source, candidates);
205+
}
206+
}
207+
208+
/**
209+
* A source is named by the mount its last reading came from. A never-read source is named by
210+
* its device: its first row may be a file bind mount, not the disk.
211+
*/
212+
const label = (source: string) => cache.get(source)?.disk.mount ?? source;
213+
214+
/**
215+
* Saturated: every slot holds a read older than a period and some source is skipped. The
216+
* episode lasts while slow reads keep the slots busy (running past a period, or settled since
217+
* the last refresh after one), so 15 s reads are one episode, not a toggle.
218+
*/
219+
function reportSaturation() {
220+
const stuck = [...inFlight.values()].filter((e) => monoNow() - e.startedAt >= REFRESH_MS);
221+
const busy = stuck.length + slowSettled;
222+
slowSettled = 0;
223+
if (saturated && busy >= MAX_IN_FLIGHT) return;
224+
if (stuck.length >= MAX_IN_FLIGHT) {
225+
const skipped = [...rows.keys()].filter((source) => !inFlight.has(source));
226+
if (saturated || skipped.length === 0) return;
227+
saturated = true;
228+
log.warn("disk reads saturated", {
229+
inFlight: stuck.map((entry) => entry.row.mount),
230+
skipped: skipped.map(label),
231+
});
232+
} else if (saturated) {
233+
saturated = false;
234+
log.info("disk reads no longer saturated", {});
235+
}
236+
}
237+
238+
/**
239+
* One read of a source: its first row whose mount point is a directory. Every stat and statfs
240+
* runs inside this one in-flight slot, so a hang in either stays under the cap.
241+
*/
242+
async function readSource(entry: { row: MountRow }, candidates: ReadonlyArray<MountRow>) {
243+
// A row whose stat fails is skipped; the read fails only if no later row is a directory.
244+
let failed: { readonly row: MountRow; readonly error: unknown } | undefined;
245+
for (const row of candidates) {
246+
entry.row = row;
247+
let isDirectory: boolean;
248+
try {
249+
isDirectory = (await stat(row.path)).isDirectory();
250+
} catch (error) {
251+
failed ??= { row, error };
252+
continue;
253+
}
254+
if (isDirectory) return { row, stat: await statfs(row.path) };
255+
}
256+
if (failed === undefined) return undefined;
257+
entry.row = failed.row;
258+
throw failed.error;
259+
}
260+
261+
function start(source: string, candidates: ReadonlyArray<MountRow>) {
262+
const startedAt = monoNow();
263+
const entry = { row: candidates[0]!, startedAt };
264+
inFlight.set(source, entry);
265+
readSource(entry, candidates)
266+
.then((read) => {
267+
failureLogged.delete(source);
268+
// No directory among its mounts: a file bind mount is not a disk.
269+
if (read === undefined) cache.delete(source);
270+
// The source may have been unmounted or remounted while this read was parked.
271+
else if (mountedAt(source, read.row.path)) {
272+
cache.set(source, {
273+
path: read.row.path,
274+
disk: toDisk(read.row.mount, read.stat, epochNow()),
275+
});
276+
}
277+
})
278+
.catch((error: unknown) => {
279+
const code = errorCode(error);
280+
const logged = failureLogged.get(source) ?? new Set<string>();
281+
if (logged.has(code)) return;
282+
failureLogged.set(source, logged.add(code));
283+
log.warn("disk read failed", { mount: entry.row.mount, code, error: errorMessage(error) });
284+
})
285+
.finally(() => {
286+
inFlight.delete(source);
287+
if (monoNow() - startedAt >= REFRESH_MS) slowSettled += 1;
288+
const stuckRow = stuckLogged.get(source);
289+
if (stuckRow !== undefined) {
290+
stuckLogged.delete(source);
291+
log.info("disk read recovered", {
292+
mount: stuckRow.mount,
293+
afterMs: monoNow() - startedAt,
294+
});
295+
}
296+
pump();
297+
})
298+
// A throw from a log sink or pump here must not become an unhandled rejection.
299+
.catch(() => {});
300+
}
301+
302+
refresh();
303+
return {
304+
snapshot() {
305+
if (monoNow() - lastRefresh >= REFRESH_MS) refresh();
306+
if (!parsed) return undefined;
307+
const disks: HostMetricsDisk[] = [];
308+
for (const source of rows.keys()) {
309+
const entry = cache.get(source);
310+
if (entry !== undefined) disks.push(entry.disk);
311+
}
312+
return disks;
313+
},
314+
};
315+
}

0 commit comments

Comments
 (0)