Skip to content

Commit 86d130a

Browse files
feat(server): export server and process-group resource metrics (pingdotgg#22122)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 9d68e1e commit 86d130a

7 files changed

Lines changed: 271 additions & 3 deletions

File tree

‎apps/server/src/observability/EventLoopMonitor.test.ts‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import * as Effect from "effect/Effect";
44
import * as FileSystem from "effect/FileSystem";
55
import * as Layer from "effect/Layer";
66
import * as Logger from "effect/Logger";
7+
import * as Metric from "effect/Metric";
78
import * as Tracer from "effect/Tracer";
89
import * as TestClock from "effect/testing/TestClock";
910

@@ -24,6 +25,8 @@ const stalled: EventLoopMonitor.EventLoopReadings = {
2425
involuntaryContextSwitches: 57,
2526
},
2627
rssBytes: 1536 * 1024 * 1024,
28+
heapUsedBytes: 512 * 1024 * 1024,
29+
elapsedMs: 30_000,
2730
};
2831
const quiet: EventLoopMonitor.EventLoopReadings = { ...stalled, delayMaxNs: ms(2_950) };
2932

@@ -78,6 +81,29 @@ describe("EventLoopMonitor", () => {
7881
}),
7982
);
8083

84+
it.effect("exports the server's memory, CPU and event loop from each sample", () =>
85+
Effect.gen(function* () {
86+
yield* monitor([quiet, stalled]);
87+
yield* TestClock.adjust("60 seconds");
88+
89+
const gauges = Object.fromEntries(
90+
(yield* Metric.snapshot).flatMap((snapshot) =>
91+
snapshot.id.startsWith("t3_server_") && snapshot.type === "Gauge"
92+
? [[snapshot.id, snapshot.state.value]]
93+
: [],
94+
),
95+
);
96+
assert.deepStrictEqual(gauges, {
97+
t3_server_resident_bytes: 1536 * 1024 * 1024,
98+
t3_server_heap_used_bytes: 512 * 1024 * 1024,
99+
t3_server_event_loop_utilization: 0.176,
100+
t3_server_event_loop_delay_max: 4_950,
101+
// 406 ms of CPU over 30 s of wall time.
102+
t3_server_cpu_utilization: 406 / 30_000,
103+
});
104+
}).pipe(Effect.scoped),
105+
);
106+
81107
it.effect("keeps each stall as one line in a capped stall log", () =>
82108
Effect.gen(function* () {
83109
const fs = yield* FileSystem.FileSystem;

‎apps/server/src/observability/EventLoopMonitor.ts‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,13 @@ import { RotatingFileSink } from "@t3tools/shared/logging";
55
import * as DateTime from "effect/DateTime";
66
import * as Effect from "effect/Effect";
77
import * as Layer from "effect/Layer";
8+
import * as Metric from "effect/Metric";
89
import * as Path from "effect/Path";
910
import * as Schema from "effect/Schema";
1011
import type * as Scope from "effect/Scope";
1112

1213
import * as ServerConfig from "../config.ts";
14+
import * as Metrics from "./Metrics.ts";
1315

1416
// Node's delay histogram wakes a native timer every RESOLUTION_MS and records the
1517
// gap between wakeups, so an idle loop reads about RESOLUTION_MS and a stall of S
@@ -34,6 +36,9 @@ export interface EventLoopReadings {
3436
| "involuntaryContextSwitches"
3537
>;
3638
readonly rssBytes: number;
39+
readonly heapUsedBytes: number;
40+
/** Wall time the readings cover. */
41+
readonly elapsedMs: number;
3742
}
3843

3944
// Node skips the first gap after a reset, so a stall right at a sample boundary can be missed.
@@ -48,11 +53,14 @@ const makeNodeSampler = Effect.gen(function* () {
4853
);
4954
let elu = NodePerfHooks.performance.eventLoopUtilization();
5055
let usage = process.resourceUsage();
56+
let sampledAt = NodePerfHooks.performance.now();
5157

5258
// @effect-diagnostics-next-line returnEffectInGen:off - the read effect is the result.
5359
return Effect.sync(() => {
5460
const nextElu = NodePerfHooks.performance.eventLoopUtilization();
5561
const nextUsage = process.resourceUsage();
62+
const nextSampledAt = NodePerfHooks.performance.now();
63+
const memory = process.memoryUsage();
5664
const loop = NodePerfHooks.performance.eventLoopUtilization(nextElu, elu);
5765
const readings: EventLoopReadings = {
5866
delayMaxNs: histogram.max,
@@ -66,11 +74,14 @@ const makeNodeSampler = Effect.gen(function* () {
6674
involuntaryContextSwitches:
6775
nextUsage.involuntaryContextSwitches - usage.involuntaryContextSwitches,
6876
},
69-
rssBytes: process.memoryUsage.rss(),
77+
rssBytes: memory.rss,
78+
heapUsedBytes: memory.heapUsed,
79+
elapsedMs: nextSampledAt - sampledAt,
7080
};
7181
histogram.reset();
7282
elu = nextElu;
7383
usage = nextUsage;
84+
sampledAt = nextSampledAt;
7485
return readings;
7586
});
7687
});
@@ -83,6 +94,21 @@ export const stallMs = ({ delayMaxNs, activeMs }: EventLoopReadings) => {
8394
return delayMs;
8495
};
8596

97+
// Untraced: it runs inside every 30-second tick, which already has the monitor's own span.
98+
const recordGauges = Effect.fnUntraced(function* (readings: EventLoopReadings) {
99+
const cpuMs = (readings.usage.userCPUTime + readings.usage.systemCPUTime) / 1000;
100+
yield* Metric.update(Metrics.serverResidentBytes, readings.rssBytes);
101+
yield* Metric.update(Metrics.serverHeapUsedBytes, readings.heapUsedBytes);
102+
yield* Metric.update(Metrics.serverEventLoopUtilization, readings.utilization);
103+
yield* Metric.update(
104+
Metrics.serverEventLoopDelayMax,
105+
Math.max(0, Math.round(readings.delayMaxNs / 1e6) - RESOLUTION_MS),
106+
);
107+
if (readings.elapsedMs > 0) {
108+
yield* Metric.update(Metrics.serverCpuUtilization, cpuMs / readings.elapsedMs);
109+
}
110+
});
111+
86112
const encodeStallLine = Schema.encodeUnknownEffect(Schema.fromJsonString(Schema.Unknown));
87113

88114
export interface StallLogOptions {
@@ -114,6 +140,7 @@ export const layerWith = (
114140
);
115141
const tick = Effect.gen(function* () {
116142
const readings = yield* sample;
143+
yield* recordGauges(readings);
117144
const delayMaxMs = stallMs(readings);
118145
if (delayMaxMs === undefined) return;
119146
const { utilization, usage, rssBytes } = readings;

‎apps/server/src/observability/Metrics.ts‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,30 @@ export const providerTurnDuration = Metric.timer("t3_provider_turn_duration", {
3939
description: "Time for the provider adapter to start a turn, not how long the turn runs.",
4040
});
4141

42+
// OTLP carries the unit, and Prometheus names the series by it: a unitless gauge gains `_ratio`.
43+
export const serverResidentBytes = Metric.gauge("t3_server_resident_bytes", {
44+
description: "Resident memory of the server process.",
45+
attributes: { unit: "By" },
46+
});
47+
48+
export const serverHeapUsedBytes = Metric.gauge("t3_server_heap_used_bytes", {
49+
description: "JavaScript heap in use by the server process.",
50+
attributes: { unit: "By" },
51+
});
52+
53+
export const serverCpuUtilization = Metric.gauge("t3_server_cpu_utilization", {
54+
description: "CPU time the server used per second over the last sample; 1 is one full core.",
55+
});
56+
57+
export const serverEventLoopUtilization = Metric.gauge("t3_server_event_loop_utilization", {
58+
description: "Share of the last sample the server's event loop spent busy.",
59+
});
60+
61+
export const serverEventLoopDelayMax = Metric.gauge("t3_server_event_loop_delay_max", {
62+
description: "Longest event loop delay over the last sample.",
63+
attributes: { unit: "ms" },
64+
});
65+
4266
export const gitCommandsTotal = Metric.counter("t3_git_commands_total", {
4367
description: "Total git commands executed by the server runtime.",
4468
});
Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
import { assert, describe, it } from "@effect/vitest";
2+
import type { ResourceTelemetryProcess } from "@t3tools/contracts";
3+
4+
import { groupProcesses } from "./ProcessGroupMetrics.ts";
5+
6+
const process = (
7+
pid: number,
8+
ppid: number,
9+
name: string,
10+
category: ResourceTelemetryProcess["category"] = "server-child",
11+
command = name,
12+
) =>
13+
({
14+
identity: { pid, startTimeMs: 1 },
15+
ppid,
16+
name,
17+
command,
18+
category,
19+
residentBytes: pid * 1_000,
20+
cpuPercent: pid,
21+
}) as ResourceTelemetryProcess;
22+
23+
describe("groupProcesses", () => {
24+
it("sums each direct child of the server with its whole subtree", () => {
25+
const totals = groupProcesses([
26+
process(10, 1, "node", "server"),
27+
process(11, 10, "t3-resource-mon", "resource-monitor"),
28+
process(20, 10, "claude"),
29+
process(21, 20, "bash"),
30+
process(22, 21, "cargo"),
31+
process(30, 10, "codex"),
32+
process(40, 10, "zsh"),
33+
process(50, 10, "chrome-headless"),
34+
process(60, 10, "git", "server-child", "git checkout claude/fix"),
35+
process(70, 99, "orphan"),
36+
// An older Claude package runs as a script under node.
37+
process(
38+
80,
39+
10,
40+
"node",
41+
"server-child",
42+
"node /opt/node_modules/@anthropic-ai/claude-code/cli.js",
43+
),
44+
process(90, 10, "node", "server-child", "node /srv/tools/report.js"),
45+
]);
46+
47+
assert.deepStrictEqual(Object.fromEntries(totals), {
48+
server: { residentBytes: 10_000, cpuPercent: 10, count: 1 },
49+
"resource-monitor": { residentBytes: 11_000, cpuPercent: 11, count: 1 },
50+
// The agent's shell and the build it ran belong to the agent, not the terminal group.
51+
claude: { residentBytes: 143_000, cpuPercent: 143, count: 4 },
52+
codex: { residentBytes: 30_000, cpuPercent: 30, count: 1 },
53+
terminal: { residentBytes: 40_000, cpuPercent: 40, count: 1 },
54+
browser: { residentBytes: 50_000, cpuPercent: 50, count: 1 },
55+
// `git` whose arguments mention claude, a parentless process, and a node script.
56+
other: { residentBytes: 220_000, cpuPercent: 220, count: 3 },
57+
});
58+
});
59+
});
Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
import type { ResourceTelemetryProcess } from "@t3tools/contracts";
2+
import * as Effect from "effect/Effect";
3+
import * as Layer from "effect/Layer";
4+
import * as Metric from "effect/Metric";
5+
6+
import * as ServerConfig from "../config.ts";
7+
import { metricAttributes } from "../observability/Metrics.ts";
8+
import * as ResourceTelemetry from "./ResourceTelemetry.ts";
9+
10+
const SAMPLE_INTERVAL = "1 minute";
11+
12+
const residentBytes = Metric.gauge("t3_process_group_resident_bytes", {
13+
description: "Resident memory of the server's processes, summed by group.",
14+
attributes: { unit: "By" },
15+
});
16+
const cpuPercent = Metric.gauge("t3_process_group_cpu_percent", {
17+
description: "CPU use of the server's processes, summed by group; 100 is one full core.",
18+
attributes: { unit: "%" },
19+
});
20+
const processCount = Metric.gauge("t3_process_group_count", {
21+
description: "Processes in each group.",
22+
attributes: { unit: "{process}" },
23+
});
24+
25+
// Each direct child of the server names the group of its whole subtree. A fixed set keeps one
26+
// series per group per server, however many agents and processes come and go.
27+
const PROVIDER_GROUPS: ReadonlyArray<readonly [RegExp, string]> = [
28+
[/codex/, "codex"],
29+
[/claude/, "claude"],
30+
[/cursor/, "cursor"],
31+
[/opencode/, "opencode"],
32+
[/grok/, "grok"],
33+
[/antigravity|gemini/, "antigravity"],
34+
];
35+
const OTHER_GROUPS: ReadonlyArray<readonly [RegExp, string]> = [
36+
[/chrom/, "browser"],
37+
[/^(?:ba|z|fi|da|c)?sh(?:\.exe)?$|^pwsh|^powershell|^cmd\.exe$/, "terminal"],
38+
];
39+
// A CLI shipped as a script runs under a JavaScript runtime, so its process name is the
40+
// runtime's (older Claude packages run `node .../@anthropic-ai/claude-code/cli.js`). Only those
41+
// are matched by command line; any other name is matched by name alone, so a shell or `git`
42+
// whose arguments mention a provider stays in its own group.
43+
const SCRIPT_RUNTIME = /^(?:node|bun|deno)(?:\.exe)?$/;
44+
45+
const match = (groups: ReadonlyArray<readonly [RegExp, string]>, text: string) =>
46+
groups.find(([pattern]) => pattern.test(text))?.[1];
47+
48+
const childGroup = (name: string, command: string) => {
49+
const lowerName = name.toLowerCase();
50+
if (SCRIPT_RUNTIME.test(lowerName))
51+
return match(PROVIDER_GROUPS, command.toLowerCase()) ?? "other";
52+
return match(PROVIDER_GROUPS, lowerName) ?? match(OTHER_GROUPS, lowerName) ?? "other";
53+
};
54+
55+
export interface ProcessGroupTotals {
56+
readonly residentBytes: number;
57+
readonly cpuPercent: number;
58+
readonly count: number;
59+
}
60+
61+
export function groupProcesses(
62+
processes: ReadonlyArray<ResourceTelemetryProcess>,
63+
): ReadonlyMap<string, ProcessGroupTotals> {
64+
const byPid = new Map(processes.map((process) => [process.identity.pid, process]));
65+
const serverPid = processes.find((process) => process.category === "server")?.identity.pid;
66+
const groupOf = (process: ResourceTelemetryProcess): string => {
67+
if (process.category === "server") return "server";
68+
if (process.category === "resource-monitor") return "resource-monitor";
69+
if (process.category.startsWith("electron-")) return "electron";
70+
let current = process;
71+
// Bounded by the table size, in case a reused PID makes the parent chain loop.
72+
for (let step = 0; step < processes.length; step++) {
73+
if (current.ppid === serverPid) return childGroup(current.name, current.command);
74+
const parent = byPid.get(current.ppid);
75+
if (parent === undefined) break;
76+
current = parent;
77+
}
78+
return "other";
79+
};
80+
const totals = new Map<string, ProcessGroupTotals>();
81+
for (const process of processes) {
82+
const group = groupOf(process);
83+
const total = totals.get(group) ?? { residentBytes: 0, cpuPercent: 0, count: 0 };
84+
totals.set(group, {
85+
residentBytes: total.residentBytes + process.residentBytes,
86+
cpuPercent: total.cpuPercent + process.cpuPercent,
87+
count: total.count + 1,
88+
});
89+
}
90+
return totals;
91+
}
92+
93+
/**
94+
* Exports process totals by group while OTLP metrics export is on. Sampling walks the process
95+
* table, which an idle server otherwise never does, so only a server exporting metrics pays for
96+
* it, once a minute.
97+
*/
98+
export const layer = Layer.effectDiscard(
99+
Effect.gen(function* () {
100+
const config = yield* ServerConfig.ServerConfig;
101+
if (config.otlpMetricsUrl === undefined) return;
102+
const telemetry = yield* ResourceTelemetry.ResourceTelemetry;
103+
// A group that empties reads zero rather than holding its last value.
104+
const seen = new Set<string>();
105+
const sample = Effect.fn("ProcessGroupMetrics.sample")(function* () {
106+
const totals = groupProcesses((yield* telemetry.refresh).processes);
107+
for (const group of totals.keys()) seen.add(group);
108+
for (const group of seen) {
109+
const total = totals.get(group) ?? { residentBytes: 0, cpuPercent: 0, count: 0 };
110+
const attributes = metricAttributes({ group });
111+
yield* Metric.update(Metric.withAttributes(residentBytes, attributes), total.residentBytes);
112+
yield* Metric.update(Metric.withAttributes(cpuPercent, attributes), total.cpuPercent);
113+
yield* Metric.update(Metric.withAttributes(processCount, attributes), total.count);
114+
}
115+
});
116+
yield* sample().pipe(
117+
Effect.catch((error) =>
118+
Effect.logDebug("process group metrics sample failed", { error: error.message }),
119+
),
120+
Effect.andThen(Effect.sleep(SAMPLE_INTERVAL)),
121+
Effect.forever,
122+
Effect.forkScoped,
123+
);
124+
}),
125+
);

‎apps/server/src/server.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,7 @@ import * as DesktopTelemetryReceiver from "./resourceTelemetry/DesktopTelemetryR
163163
import * as NativeTelemetryClient from "./resourceTelemetry/NativeTelemetryClient.ts";
164164
import * as ResourceAttribution from "./resourceTelemetry/ResourceAttribution.ts";
165165
import * as ResourceMonitorBinary from "./resourceTelemetry/ResourceMonitorBinary.ts";
166+
import * as ProcessGroupMetrics from "./resourceTelemetry/ProcessGroupMetrics.ts";
166167
import * as ResourceTelemetry from "./resourceTelemetry/ResourceTelemetry.ts";
167168
import * as AntigravityUsage from "./provider/Drivers/AntigravityUsage.ts";
168169
import * as CursorAccountReader from "@t3tools/provider-cursor/server/CursorAccountReader";
@@ -276,6 +277,7 @@ const layerResourceDiagnostics = Layer.mergeAll(
276277
layerResourceTelemetry,
277278
ProcessDiagnostics.layer.pipe(Layer.provide(layerResourceTelemetry)),
278279
ProcessResourceMonitor.layer.pipe(Layer.provide(layerResourceTelemetry)),
280+
ProcessGroupMetrics.layer.pipe(Layer.provide(layerResourceTelemetry)),
279281
);
280282

281283
const layerRelayClient = Layer.unwrap(

‎docs/internals/resource-telemetry.md‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,13 @@ protocol. A missing or failed collector leaves the server running.
1111
The native child owns sampling and bounded in-memory history. It samples on a timer
1212
only while diagnostics has live subscribers. With none, the server sends a sample
1313
interval of 0 and the child samples only on request, so an idle server never walks
14-
the process table. History therefore covers only time with a live subscriber, plus
15-
on-demand samples. Consuming host power for background scheduling must not retain
14+
the process table. The exception is OTLP metrics export: a server exporting metrics
15+
requests one sample a minute and exports process totals by group
16+
([`ProcessGroupMetrics`](../../apps/server/src/resourceTelemetry/ProcessGroupMetrics.ts)),
17+
because the user asked for continuous telemetry. Groups are a fixed set named after
18+
the server's direct children, never process IDs or threads, so the series count
19+
stays bounded as agents come and go. History therefore covers only time with a live
20+
subscriber, plus on-demand and export samples. Consuming host power for background scheduling must not retain
1621
live diagnostics. There is no telemetry database or recurring shell-probe fallback.
1722

1823
History has independent bounds for age, snapshot count, process rows, and retained

0 commit comments

Comments
 (0)