|
| 1 | +import type { HostMetricsDisk } from "@t3tools/contracts"; |
| 2 | + |
| 3 | +/** |
| 4 | + * Shared disk-usage reader for host metrics. A hung statfs pins a libuv pool thread and cannot |
| 5 | + * be cancelled, so at most {@link MAX_IN_FLIGHT} reads are in flight and a source in flight is |
| 6 | + * never read again until it settles. |
| 7 | + */ |
| 8 | + |
| 9 | +/** Never pins more than 2 of the 4 libuv pool threads. */ |
| 10 | +const MAX_IN_FLIGHT = 2; |
| 11 | +/** Refreshes are at least this far apart (monotonic clock, from the last refresh start). */ |
| 12 | +const REFRESH_MS = 10_000; |
| 13 | +/** A read in flight longer than this is reported stuck, once per episode. */ |
| 14 | +const STUCK_MS = 30_000; |
| 15 | +/** Local block-device filesystems worth reporting. */ |
| 16 | +const ALLOWED_TYPES: ReadonlySet<string> = new Set([ |
| 17 | + "ext2", |
| 18 | + "ext3", |
| 19 | + "ext4", |
| 20 | + "xfs", |
| 21 | + "btrfs", |
| 22 | + "f2fs", |
| 23 | + "exfat", |
| 24 | + "ntfs3", |
| 25 | +]); |
| 26 | +const DARWIN_DATA = "/System/Volumes/Data"; |
| 27 | + |
| 28 | +export interface MountRow { |
| 29 | + readonly source: string; |
| 30 | + /** Mount point shown to the user: escapes decoded, control characters dropped. */ |
| 31 | + readonly mount: string; |
| 32 | + readonly type: string; |
| 33 | + /** The real mount point, handed to statfs; identifies the mount. */ |
| 34 | + readonly path: string; |
| 35 | +} |
| 36 | + |
| 37 | +export interface StatFsLike { |
| 38 | + readonly bsize: number; |
| 39 | + readonly blocks: number; |
| 40 | + readonly bfree: number; |
| 41 | + readonly bavail: number; |
| 42 | +} |
| 43 | + |
| 44 | +export interface HostDisksDeps { |
| 45 | + readonly statfs: (path: string) => Promise<StatFsLike>; |
| 46 | + /** A mount point that is not a directory (a bind-mounted file) is not a disk. */ |
| 47 | + readonly stat: (path: string) => Promise<{ isDirectory(): boolean }>; |
| 48 | + readonly readMounts: () => string; |
| 49 | + /** Monotonic milliseconds; drives every timer. */ |
| 50 | + readonly monoNow: () => number; |
| 51 | + /** Epoch milliseconds; stamped on each reading as `readAt`. */ |
| 52 | + readonly epochNow: () => number; |
| 53 | + readonly log: { |
| 54 | + warn(msg: string, data: object): void; |
| 55 | + info(msg: string, data: object): void; |
| 56 | + }; |
| 57 | + readonly platform: NodeJS.Platform; |
| 58 | + readonly exists: (path: string) => boolean; |
| 59 | +} |
| 60 | + |
| 61 | +export interface HostDisks { |
| 62 | + /** Readings in mount-table order; undefined until a mount table has been parsed once. */ |
| 63 | + snapshot(): ReadonlyArray<HostMetricsDisk> | undefined; |
| 64 | +} |
| 65 | + |
| 66 | +/** `/proc/self/mounts` encodes space, tab, newline and backslash as `\ooo`. */ |
| 67 | +const decodeOctal = (field: string) => |
| 68 | + field.replace(/\\([0-7]{3})/g, (_, octal: string) => String.fromCharCode(parseInt(octal, 8))); |
| 69 | + |
| 70 | +const CONTROL_CHARS = /[\u0000-\u001f\u007f]/g; |
| 71 | + |
| 72 | +/** |
| 73 | + * Allowlisted local block devices from a `/proc/self/mounts` text, in table order. Of all rows |
| 74 | + * stacked on one mount point only the last (the visible top) counts, whatever its type; then the |
| 75 | + * allowlist applies. A source can have several rows; the reader picks the first directory. |
| 76 | + */ |
| 77 | +export function parseMounts(text: string): ReadonlyArray<MountRow> { |
| 78 | + const parsed: Array<{ source: string; rawMount: string; type: string; path: string }> = []; |
| 79 | + for (const line of text.split("\n")) { |
| 80 | + const [source, rawMount, type] = line.split(" "); |
| 81 | + if (source === undefined || rawMount === undefined || type === undefined) continue; |
| 82 | + parsed.push({ source, rawMount, type, path: decodeOctal(rawMount) }); |
| 83 | + } |
| 84 | + const top = new Map(parsed.map((row) => [row.path, row])); |
| 85 | + const rows: MountRow[] = []; |
| 86 | + for (const row of parsed) { |
| 87 | + if (top.get(row.path) !== row) continue; |
| 88 | + if (!ALLOWED_TYPES.has(row.type)) continue; |
| 89 | + if (!row.source.startsWith("/dev/") || row.source.startsWith("/dev/loop")) continue; |
| 90 | + rows.push({ |
| 91 | + source: row.source, |
| 92 | + mount: row.path.replace(CONTROL_CHARS, ""), |
| 93 | + type: row.type, |
| 94 | + path: row.path, |
| 95 | + }); |
| 96 | + } |
| 97 | + return rows; |
| 98 | +} |
| 99 | + |
| 100 | +function toDisk(mount: string, s: StatFsLike, readAt: number): HostMetricsDisk { |
| 101 | + const usedBytes = (s.blocks - s.bfree) * s.bsize; |
| 102 | + const availBytes = s.bavail * s.bsize; |
| 103 | + const denominator = usedBytes + availBytes; |
| 104 | + return { |
| 105 | + mount, |
| 106 | + usedBytes, |
| 107 | + totalBytes: s.blocks * s.bsize, |
| 108 | + pct: denominator > 0 ? (usedBytes / denominator) * 100 : 0, |
| 109 | + readAt, |
| 110 | + }; |
| 111 | +} |
| 112 | + |
| 113 | +const errorCode = (error: unknown): string => { |
| 114 | + const code = (error as { code?: unknown } | null)?.code; |
| 115 | + return typeof code === "string" ? code : "unknown"; |
| 116 | +}; |
| 117 | +const errorMessage = (error: unknown): string => |
| 118 | + error instanceof Error ? error.message : String(error); |
| 119 | + |
| 120 | +export function makeHostDisks(deps: HostDisksDeps): HostDisks { |
| 121 | + const { statfs, stat, monoNow, epochNow, log } = deps; |
| 122 | + /** Last reading per source, with the path it was taken at. */ |
| 123 | + const cache = new Map<string, { readonly path: string; readonly disk: HostMetricsDisk }>(); |
| 124 | + /** Each source's mount rows, in table order. */ |
| 125 | + let rows = new Map<string, ReadonlyArray<MountRow>>(); |
| 126 | + /** `row` is the candidate being read now. */ |
| 127 | + const inFlight = new Map<string, { row: MountRow; readonly startedAt: number }>(); |
| 128 | + /** Sources waiting for a slot, in mount-table order; never holds an in-flight source. */ |
| 129 | + const queue = new Map<string, ReadonlyArray<MountRow>>(); |
| 130 | + /** The row each stuck read was reported on, so its recovery names the same row. */ |
| 131 | + const stuckLogged = new Map<string, MountRow>(); |
| 132 | + /** Error codes already logged per source; cleared by a successful read. */ |
| 133 | + const failureLogged = new Map<string, Set<string>>(); |
| 134 | + let parsed = false; |
| 135 | + let mountsFailing = false; |
| 136 | + let saturated = false; |
| 137 | + /** Reads that settled since the last refresh after holding their slot a whole period. */ |
| 138 | + let slowSettled = 0; |
| 139 | + let lastRefresh = 0; |
| 140 | + |
| 141 | + function readRows(): ReadonlyArray<MountRow> | undefined { |
| 142 | + if (deps.platform === "linux") return parseMounts(deps.readMounts()); |
| 143 | + if (deps.platform === "darwin") { |
| 144 | + const path = deps.exists(DARWIN_DATA) ? DARWIN_DATA : "/"; |
| 145 | + return [{ source: "darwin-data", mount: "/", type: "apfs", path }]; |
| 146 | + } |
| 147 | + return undefined; |
| 148 | + } |
| 149 | + |
| 150 | + /** A source's mounts include this path. */ |
| 151 | + const mountedAt = (source: string, path: string) => |
| 152 | + rows.get(source)?.some((row) => row.path === path) === true; |
| 153 | + |
| 154 | + function refresh() { |
| 155 | + lastRefresh = monoNow(); |
| 156 | + try { |
| 157 | + const next = readRows(); |
| 158 | + if (next === undefined) return; |
| 159 | + rows = new Map(); |
| 160 | + for (const row of next) rows.set(row.source, [...(rows.get(row.source) ?? []), row]); |
| 161 | + parsed = true; |
| 162 | + mountsFailing = false; |
| 163 | + } catch (error) { |
| 164 | + if (!mountsFailing) { |
| 165 | + mountsFailing = true; |
| 166 | + log.warn("disk mount table unreadable", { |
| 167 | + code: errorCode(error), |
| 168 | + error: errorMessage(error), |
| 169 | + }); |
| 170 | + } |
| 171 | + } |
| 172 | + // A reading is shown only under the mount it was taken at. |
| 173 | + for (const [source, entry] of cache) { |
| 174 | + if (!mountedAt(source, entry.path)) cache.delete(source); |
| 175 | + } |
| 176 | + for (const source of queue.keys()) if (!rows.has(source)) queue.delete(source); |
| 177 | + for (const [source, candidates] of rows) { |
| 178 | + const running = inFlight.get(source); |
| 179 | + if (running === undefined) queue.set(source, candidates); |
| 180 | + else warnIfStuck(source, running.row, running.startedAt); |
| 181 | + } |
| 182 | + reportSaturation(); |
| 183 | + pump(); |
| 184 | + } |
| 185 | + |
| 186 | + function warnIfStuck(source: string, row: MountRow, startedAt: number) { |
| 187 | + const stuckMs = monoNow() - startedAt; |
| 188 | + if (stuckMs <= STUCK_MS || stuckLogged.has(source)) return; |
| 189 | + stuckLogged.set(source, row); |
| 190 | + log.warn("disk read stuck", { |
| 191 | + mount: row.mount, |
| 192 | + type: row.type, |
| 193 | + stuckMs, |
| 194 | + inFlight: inFlight.size, |
| 195 | + }); |
| 196 | + } |
| 197 | + |
| 198 | + function pump() { |
| 199 | + while (inFlight.size < MAX_IN_FLIGHT) { |
| 200 | + const next = queue.entries().next(); |
| 201 | + if (next.done === true) break; |
| 202 | + const [source, candidates] = next.value; |
| 203 | + queue.delete(source); |
| 204 | + start(source, candidates); |
| 205 | + } |
| 206 | + } |
| 207 | + |
| 208 | + /** |
| 209 | + * A source is named by the mount its last reading came from. A never-read source is named by |
| 210 | + * its device: its first row may be a file bind mount, not the disk. |
| 211 | + */ |
| 212 | + const label = (source: string) => cache.get(source)?.disk.mount ?? source; |
| 213 | + |
| 214 | + /** |
| 215 | + * Saturated: every slot holds a read older than a period and some source is skipped. The |
| 216 | + * episode lasts while slow reads keep the slots busy (running past a period, or settled since |
| 217 | + * the last refresh after one), so 15 s reads are one episode, not a toggle. |
| 218 | + */ |
| 219 | + function reportSaturation() { |
| 220 | + const stuck = [...inFlight.values()].filter((e) => monoNow() - e.startedAt >= REFRESH_MS); |
| 221 | + const busy = stuck.length + slowSettled; |
| 222 | + slowSettled = 0; |
| 223 | + if (saturated && busy >= MAX_IN_FLIGHT) return; |
| 224 | + if (stuck.length >= MAX_IN_FLIGHT) { |
| 225 | + const skipped = [...rows.keys()].filter((source) => !inFlight.has(source)); |
| 226 | + if (saturated || skipped.length === 0) return; |
| 227 | + saturated = true; |
| 228 | + log.warn("disk reads saturated", { |
| 229 | + inFlight: stuck.map((entry) => entry.row.mount), |
| 230 | + skipped: skipped.map(label), |
| 231 | + }); |
| 232 | + } else if (saturated) { |
| 233 | + saturated = false; |
| 234 | + log.info("disk reads no longer saturated", {}); |
| 235 | + } |
| 236 | + } |
| 237 | + |
| 238 | + /** |
| 239 | + * One read of a source: its first row whose mount point is a directory. Every stat and statfs |
| 240 | + * runs inside this one in-flight slot, so a hang in either stays under the cap. |
| 241 | + */ |
| 242 | + async function readSource(entry: { row: MountRow }, candidates: ReadonlyArray<MountRow>) { |
| 243 | + // A row whose stat fails is skipped; the read fails only if no later row is a directory. |
| 244 | + let failed: { readonly row: MountRow; readonly error: unknown } | undefined; |
| 245 | + for (const row of candidates) { |
| 246 | + entry.row = row; |
| 247 | + let isDirectory: boolean; |
| 248 | + try { |
| 249 | + isDirectory = (await stat(row.path)).isDirectory(); |
| 250 | + } catch (error) { |
| 251 | + failed ??= { row, error }; |
| 252 | + continue; |
| 253 | + } |
| 254 | + if (isDirectory) return { row, stat: await statfs(row.path) }; |
| 255 | + } |
| 256 | + if (failed === undefined) return undefined; |
| 257 | + entry.row = failed.row; |
| 258 | + throw failed.error; |
| 259 | + } |
| 260 | + |
| 261 | + function start(source: string, candidates: ReadonlyArray<MountRow>) { |
| 262 | + const startedAt = monoNow(); |
| 263 | + const entry = { row: candidates[0]!, startedAt }; |
| 264 | + inFlight.set(source, entry); |
| 265 | + readSource(entry, candidates) |
| 266 | + .then((read) => { |
| 267 | + failureLogged.delete(source); |
| 268 | + // No directory among its mounts: a file bind mount is not a disk. |
| 269 | + if (read === undefined) cache.delete(source); |
| 270 | + // The source may have been unmounted or remounted while this read was parked. |
| 271 | + else if (mountedAt(source, read.row.path)) { |
| 272 | + cache.set(source, { |
| 273 | + path: read.row.path, |
| 274 | + disk: toDisk(read.row.mount, read.stat, epochNow()), |
| 275 | + }); |
| 276 | + } |
| 277 | + }) |
| 278 | + .catch((error: unknown) => { |
| 279 | + const code = errorCode(error); |
| 280 | + const logged = failureLogged.get(source) ?? new Set<string>(); |
| 281 | + if (logged.has(code)) return; |
| 282 | + failureLogged.set(source, logged.add(code)); |
| 283 | + log.warn("disk read failed", { mount: entry.row.mount, code, error: errorMessage(error) }); |
| 284 | + }) |
| 285 | + .finally(() => { |
| 286 | + inFlight.delete(source); |
| 287 | + if (monoNow() - startedAt >= REFRESH_MS) slowSettled += 1; |
| 288 | + const stuckRow = stuckLogged.get(source); |
| 289 | + if (stuckRow !== undefined) { |
| 290 | + stuckLogged.delete(source); |
| 291 | + log.info("disk read recovered", { |
| 292 | + mount: stuckRow.mount, |
| 293 | + afterMs: monoNow() - startedAt, |
| 294 | + }); |
| 295 | + } |
| 296 | + pump(); |
| 297 | + }) |
| 298 | + // A throw from a log sink or pump here must not become an unhandled rejection. |
| 299 | + .catch(() => {}); |
| 300 | + } |
| 301 | + |
| 302 | + refresh(); |
| 303 | + return { |
| 304 | + snapshot() { |
| 305 | + if (monoNow() - lastRefresh >= REFRESH_MS) refresh(); |
| 306 | + if (!parsed) return undefined; |
| 307 | + const disks: HostMetricsDisk[] = []; |
| 308 | + for (const source of rows.keys()) { |
| 309 | + const entry = cache.get(source); |
| 310 | + if (entry !== undefined) disks.push(entry.disk); |
| 311 | + } |
| 312 | + return disks; |
| 313 | + }, |
| 314 | + }; |
| 315 | +} |
0 commit comments