Skip to content

Commit b15c94e

Browse files
paulcam206Copilot
andcommitted
perf(bench): report Windows RSS + ready-anchored CPU in gateway startup bench
The gateway startup benchmark logged cpu/cpuCore/rss as n/a on Windows because readProcessRssMb and readProcessTreeCpuMs were Unix-only (ps). RSS: add a Windows implementation of readProcessRssMb via a CIM Win32_Process WorkingSetSize query (with pid validation); Unix ps path unchanged. CPU: measure it at the gateway's own ready boundary instead of via an external process query. The gateway already emits a memory.ready startup-trace snapshot; it now also reports process.cpuUsage() as cpuMs, and the bench reads memory.ready.cpuMs. This is exact and ready-anchored on every platform and retires the external ps/CIM CPU read from the startup bench, so post-ready work can no longer be charged to startup CPU. During startup the gateway is a single OS process (verified: no child processes at ready), so this equals the previous process-tree sum. readProcessTreeCpuMs stays for the Unix-only restart bench. Co-authored-by: Copilot <[email protected]> Copilot-Session: 8b7f11bb-0f17-4cd2-809d-a62dca26f62d
1 parent 8470a5b commit b15c94e

7 files changed

Lines changed: 195 additions & 22 deletions

File tree

scripts/bench-gateway-startup.ts

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ import {
1212
getFreePort,
1313
parseProcessRssKb,
1414
readProcessRssMb,
15-
readProcessTreeCpuMs,
1615
requestProbeStatus,
1716
} from "./lib/gateway-bench-probes.ts";
1817
import { selectSlowStartupTraceDurations } from "./lib/gateway-startup-trace-ranking.js";
@@ -798,16 +797,20 @@ async function runGatewaySample(options: {
798797
detached: process.platform !== "win32",
799798
env,
800799
});
801-
const cpuStartMs = readProcessTreeCpuMs(child.pid);
800+
const isWindows = process.platform === "win32";
802801
const sampleRss = () => {
803802
const rssMb = readProcessRssMb(child.pid);
804803
if (rssMb != null) {
805804
maxRssMb = maxRssMb == null ? rssMb : Math.max(maxRssMb, rssMb);
806805
}
807806
};
808-
sampleRss();
809-
const rssTimer = setInterval(sampleRss, 100);
810-
rssTimer.unref?.();
807+
// Reading RSS on Windows spawns a CIM query, so sample once at ready (below) instead of hot-polling;
808+
// Unix `ps` is cheap enough to poll for the startup peak.
809+
const rssTimer = isWindows ? null : setInterval(sampleRss, 100);
810+
if (!isWindows) {
811+
sampleRss();
812+
}
813+
rssTimer?.unref?.();
811814
const childExitPromise = new Promise<{ exitCode: number | null; signal: string | null }>(
812815
(resolve) => {
813816
child.once("exit", (exitCode, signal) => {
@@ -859,15 +862,21 @@ async function runGatewaySample(options: {
859862
}),
860863
]);
861864
const readyAt = performance.now();
862-
const cpuEndMs = readProcessTreeCpuMs(child.pid);
863-
const cpuMs = cpuStartMs == null || cpuEndMs == null ? null : Math.max(0, cpuEndMs - cpuStartMs);
864-
const cpuCoreRatio = cpuMs == null ? null : cpuMs / Math.max(1, readyAt - startAt);
865+
// Sample RSS while the gateway is still alive; on Windows this is the primary (non-polled) sample.
866+
sampleRss();
865867
const exit = await stopChild(child);
866-
clearInterval(rssTimer);
868+
if (rssTimer) {
869+
clearInterval(rssTimer);
870+
}
867871
sampleRss();
868872
// stopChild is the bounded teardown wait; the raw exit promise may never settle.
869873
void childExitPromise.catch(() => null);
870874
rmSync(root, { force: true, maxRetries: 3, recursive: true, retryDelay: 100 });
875+
// The gateway emits its own CPU total (process.cpuUsage) in the memory.ready startup trace, so read a
876+
// ready-anchored value here with no external process query. The value is fixed at the ready boundary,
877+
// so reading it after teardown drains stdout cannot let post-ready work inflate the reported CPU.
878+
const cpuMs = startupTrace["memory.ready.cpuMs"] ?? null;
879+
const cpuCoreRatio = cpuMs == null ? null : cpuMs / Math.max(1, readyAt - startAt);
871880

872881
return {
873882
cpuCoreRatio,

scripts/lib/gateway-bench-probes.ts

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
import { spawnSync } from "node:child_process";
33
import { request } from "node:http";
44
import { createServer } from "node:net";
5+
import path from "node:path";
56
import { expectDefined } from "../../packages/normalization-core/src/expect.js";
67

78
export async function getFreePort(): Promise<number> {
@@ -57,9 +58,12 @@ function classifyProbeErrorKind(error: unknown): string {
5758
}
5859

5960
export function readProcessRssMb(pid: number | undefined): number | null {
60-
if (!pid || process.platform === "win32") {
61+
if (!pid || !Number.isInteger(pid) || pid <= 0) {
6162
return null;
6263
}
64+
if (process.platform === "win32") {
65+
return readWin32ProcessWorkingSetMb(pid);
66+
}
6367
const result = spawnSync("ps", ["-o", "rss=", "-p", String(pid)], {
6468
encoding: "utf8",
6569
stdio: ["ignore", "pipe", "ignore"],
@@ -172,3 +176,47 @@ function parsePsCpuTimeMs(raw: string): number | null {
172176
}
173177
return null;
174178
}
179+
180+
// Windows has no fast `ps` equivalent (wmic is removed on recent builds), so per-process RSS comes from
181+
// a CIM Win32_Process query. Callers keep this to a single sample per run, never a hot poll.
182+
const WIN32_PROCESS_QUERY_TIMEOUT_MS = 10_000;
183+
184+
function readWin32ProcessWorkingSetMb(pid: number): number | null {
185+
const raw = runWin32ProcessQuery(
186+
`Get-CimInstance Win32_Process -Filter "ProcessId=${pid}" | Select-Object -ExpandProperty WorkingSetSize`,
187+
);
188+
if (raw === null) {
189+
return null;
190+
}
191+
const bytes = Number(raw.trim());
192+
if (!Number.isFinite(bytes) || bytes <= 0) {
193+
return null;
194+
}
195+
return bytes / (1024 * 1024);
196+
}
197+
198+
function runWin32ProcessQuery(script: string): string | null {
199+
const result = spawnSync(
200+
resolvePowershellPath(),
201+
["-NoProfile", "-NonInteractive", "-Command", script],
202+
{
203+
encoding: "utf8",
204+
maxBuffer: 16 * 1024 * 1024,
205+
stdio: ["ignore", "pipe", "ignore"],
206+
timeout: WIN32_PROCESS_QUERY_TIMEOUT_MS,
207+
windowsHide: true,
208+
},
209+
);
210+
if (result.status !== 0 || typeof result.stdout !== "string") {
211+
return null;
212+
}
213+
const trimmed = result.stdout.trim();
214+
return trimmed.length > 0 ? trimmed : null;
215+
}
216+
217+
function resolvePowershellPath(): string {
218+
const systemRoot = process.env.SystemRoot ?? process.env.windir;
219+
return systemRoot
220+
? path.join(systemRoot, "System32", "WindowsPowerShell", "v1.0", "powershell.exe")
221+
: "powershell.exe";
222+
}

src/gateway/restart-trace.test.ts

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
*/
44
import { describe, expect, it } from "vitest";
55
import {
6-
collectGatewayProcessMemoryUsageMb,
6+
collectGatewayProcessUsageMetrics,
77
createGatewayRestartTraceHandoffEnv,
88
} from "./restart-trace.js";
99

@@ -24,9 +24,10 @@ describe("gateway restart trace handoff", () => {
2424
});
2525

2626
it("includes restart resource counts with ready memory metrics", () => {
27-
const metrics = Object.fromEntries(collectGatewayProcessMemoryUsageMb());
27+
const metrics = Object.fromEntries(collectGatewayProcessUsageMetrics());
2828

2929
expect(metrics.rssMb).toEqual(expect.any(Number));
30+
expect(metrics.cpuMs).toEqual(expect.any(Number));
3031
expect(metrics.activeTimersCount).toEqual(expect.any(Number));
3132
expect(metrics.processSigusr1ListenersCount).toEqual(expect.any(Number));
3233
expect(metrics.processSigtermListenersCount).toEqual(expect.any(Number));
@@ -36,7 +37,7 @@ describe("gateway restart trace handoff", () => {
3637
it("counts active timer resources", () => {
3738
const timer = setTimeout(() => {}, 10_000);
3839
try {
39-
const metrics = Object.fromEntries(collectGatewayProcessMemoryUsageMb());
40+
const metrics = Object.fromEntries(collectGatewayProcessUsageMetrics());
4041

4142
expect(metrics.activeTimersCount).toBeGreaterThanOrEqual(1);
4243
} finally {

src/gateway/restart-trace.ts

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -196,16 +196,23 @@ export function recordGatewayRestartTraceDetail(name: string, metrics: RestartTr
196196
emitRestartTraceDetail(name, metrics);
197197
}
198198

199-
/** Collects process memory/resource metrics for restart trace diagnostics. */
200-
export function collectGatewayProcessMemoryUsageMb(): ReadonlyArray<readonly [string, number]> {
199+
/** Collects process CPU/memory/resource metrics for restart and startup trace diagnostics. */
200+
export function collectGatewayProcessUsageMetrics(): ReadonlyArray<readonly [string, number]> {
201201
const usage = process.memoryUsage();
202202
const toMb = (bytes: number) => bytes / 1024 / 1024;
203+
// process.cpuUsage() reports cumulative user+system CPU for this process and all its threads
204+
// (including worker threads) in microseconds. Emitted at the ready boundary, it gives the gateway
205+
// startup benchmark an exact, ready-anchored CPU total with no external process query. The gateway
206+
// is a single OS process during startup (plugins load in-process; supervisors spawn post-ready), so
207+
// this matches a process-tree sum; child-process CPU, if any were spawned before ready, is excluded.
208+
const cpu = process.cpuUsage();
203209
const metrics: Array<readonly [string, number]> = [
204210
["rssMb", toMb(usage.rss)],
205211
["heapTotalMb", toMb(usage.heapTotal)],
206212
["heapUsedMb", toMb(usage.heapUsed)],
207213
["externalMb", toMb(usage.external)],
208214
["arrayBuffersMb", toMb(usage.arrayBuffers)],
215+
["cpuMs", (cpu.user + cpu.system) / 1000],
209216
];
210217
const resources = collectGatewayProcessResourceCounts();
211218
if (resources) {

src/gateway/server-close.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ import {
2222
} from "./chat-abort.js";
2323
import { abortQueuedChatTurns, type QueuedChatTurnMap } from "./chat-queued-turns.js";
2424
import {
25-
collectGatewayProcessMemoryUsageMb,
25+
collectGatewayProcessUsageMetrics,
2626
measureGatewayRestartTrace,
2727
recordGatewayRestartTrace,
2828
} from "./restart-trace.js";
@@ -1046,7 +1046,7 @@ export function createGatewayCloseHandler(
10461046
recordGatewayRestartTrace("restart.close.total", durationMs, [
10471047
["reason", reason],
10481048
["restartExpectedMs", restartExpectedMs ?? "none"],
1049-
...collectGatewayProcessMemoryUsageMb(),
1049+
...collectGatewayProcessUsageMetrics(),
10501050
]);
10511051
return { durationMs, warnings };
10521052
};

src/gateway/server.impl.ts

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ import {
109109
pluginConfigTargetsChanged,
110110
} from "./plugin-channel-reload-targets.js";
111111
import {
112-
collectGatewayProcessMemoryUsageMb,
112+
collectGatewayProcessUsageMetrics,
113113
finishGatewayRestartTrace,
114114
recordGatewayRestartTraceDetail,
115115
recordGatewayRestartTraceSpan,
@@ -2172,12 +2172,12 @@ export async function startGatewayServer(
21722172
}),
21732173
),
21742174
));
2175-
startupTrace.detail("memory.ready", collectGatewayProcessMemoryUsageMb());
2175+
startupTrace.detail("memory.ready", collectGatewayProcessUsageMetrics());
21762176
startupTrace.mark("ready");
21772177
if (sidecarStartup === "defer") {
21782178
log.info("gateway ready");
21792179
}
2180-
finishGatewayRestartTrace("restart.ready", collectGatewayProcessMemoryUsageMb());
2180+
finishGatewayRestartTrace("restart.ready", collectGatewayProcessUsageMetrics());
21812181
postAttachRuntimeReturned = true;
21822182
activateScheduledServicesWhenReady();
21832183

@@ -2310,7 +2310,7 @@ export async function startGatewayServer(
23102310
logCron,
23112311
log,
23122312
recordPostReadyMemory: () => {
2313-
startupTrace.detail("memory.post-ready", collectGatewayProcessMemoryUsageMb());
2313+
startupTrace.detail("memory.post-ready", collectGatewayProcessUsageMetrics());
23142314
},
23152315
});
23162316
// The loop closes the previous server before this generation starts, so retired
@@ -2330,7 +2330,7 @@ export async function startGatewayServer(
23302330
errorMessage: "retained npm generation cleanup failed",
23312331
});
23322332
} else {
2333-
startupTrace.detail("memory.post-ready", collectGatewayProcessMemoryUsageMb());
2333+
startupTrace.detail("memory.post-ready", collectGatewayProcessUsageMetrics());
23342334
}
23352335
} catch (err) {
23362336
await closeOnStartupFailure();
Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
// Verifies the process probes: `ps` CPU/RSS on Unix, and Windows RSS via a CIM Win32_Process query.
2+
import { afterEach, describe, expect, it, vi } from "vitest";
3+
4+
const spawnSyncMock = vi.hoisted(() => vi.fn());
5+
vi.mock("node:child_process", () => ({ spawnSync: spawnSyncMock }));
6+
7+
import {
8+
parseProcessRssKb,
9+
readProcessRssMb,
10+
readProcessTreeCpuMs,
11+
} from "../../scripts/lib/gateway-bench-probes.js";
12+
13+
const originalPlatform = process.platform;
14+
15+
function setPlatform(platform: NodeJS.Platform): void {
16+
Object.defineProperty(process, "platform", { configurable: true, value: platform });
17+
}
18+
19+
function mockSpawn(result: { status: number | null; stdout: string }): void {
20+
spawnSyncMock.mockReturnValue(result);
21+
}
22+
23+
afterEach(() => {
24+
Object.defineProperty(process, "platform", { configurable: true, value: originalPlatform });
25+
spawnSyncMock.mockReset();
26+
});
27+
28+
describe("parseProcessRssKb", () => {
29+
it("parses a positive integer", () => {
30+
expect(parseProcessRssKb(" 144080 \n")).toBe(144_080);
31+
});
32+
33+
it("rejects zero, negatives, and non-numeric values", () => {
34+
expect(parseProcessRssKb("0")).toBeNull();
35+
expect(parseProcessRssKb("-5")).toBeNull();
36+
expect(parseProcessRssKb("abc")).toBeNull();
37+
expect(parseProcessRssKb("")).toBeNull();
38+
});
39+
});
40+
41+
describe("readProcessRssMb (Unix)", () => {
42+
it("converts `ps` RSS kilobytes to megabytes", () => {
43+
setPlatform("linux");
44+
mockSpawn({ status: 0, stdout: "144080\n" });
45+
expect(readProcessRssMb(1234)).toBeCloseTo(144_080 / 1024, 5);
46+
});
47+
48+
it("returns null when `ps` fails", () => {
49+
setPlatform("linux");
50+
mockSpawn({ status: 1, stdout: "" });
51+
expect(readProcessRssMb(1234)).toBeNull();
52+
});
53+
54+
it("returns null for an invalid pid without spawning", () => {
55+
setPlatform("linux");
56+
expect(readProcessRssMb(undefined)).toBeNull();
57+
expect(readProcessRssMb(0)).toBeNull();
58+
expect(spawnSyncMock).not.toHaveBeenCalled();
59+
});
60+
});
61+
62+
describe("readProcessTreeCpuMs (Unix)", () => {
63+
it("sums CPU time across the tree rooted at rootPid", () => {
64+
setPlatform("linux");
65+
mockSpawn({
66+
status: 0,
67+
stdout: ["100 1 00:10", "200 100 00:05", "300 100 01:00", "400 999 00:30"].join("\n"),
68+
});
69+
// 100=10s + 200=5s + 300=60s; 400 belongs to a different parent and is excluded.
70+
expect(readProcessTreeCpuMs(100)).toBe(75_000);
71+
});
72+
73+
it("returns null when the root pid is absent", () => {
74+
setPlatform("linux");
75+
mockSpawn({ status: 0, stdout: "200 100 00:05" });
76+
expect(readProcessTreeCpuMs(999_999)).toBeNull();
77+
});
78+
79+
it("returns null when `ps` fails", () => {
80+
setPlatform("linux");
81+
mockSpawn({ status: 1, stdout: "" });
82+
expect(readProcessTreeCpuMs(100)).toBeNull();
83+
});
84+
});
85+
86+
describe("readProcessRssMb (Windows)", () => {
87+
it("converts CIM WorkingSetSize bytes to megabytes", () => {
88+
setPlatform("win32");
89+
mockSpawn({ status: 0, stdout: "144080896\n" });
90+
expect(readProcessRssMb(1234)).toBeCloseTo(144_080_896 / (1024 * 1024), 5);
91+
});
92+
93+
it("returns null on empty or failed output", () => {
94+
setPlatform("win32");
95+
mockSpawn({ status: 0, stdout: " " });
96+
expect(readProcessRssMb(1234)).toBeNull();
97+
mockSpawn({ status: 1, stdout: "" });
98+
expect(readProcessRssMb(1234)).toBeNull();
99+
});
100+
});
101+
102+
describe("readProcessTreeCpuMs (Windows)", () => {
103+
it("is unavailable on Windows (startup CPU comes from the gateway ready trace)", () => {
104+
setPlatform("win32");
105+
expect(readProcessTreeCpuMs(100)).toBeNull();
106+
expect(spawnSyncMock).not.toHaveBeenCalled();
107+
});
108+
});

0 commit comments

Comments
 (0)