Skip to content

Commit cdba9a0

Browse files
t3dotggclaude
andauthored
perf(server): idle servers no longer scan every process every 5 seconds (#19399)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 8e018ff commit cdba9a0

4 files changed

Lines changed: 130 additions & 21 deletions

File tree

‎apps/server/src/resourceTelemetry/NativeTelemetryClient.test.ts‎

Lines changed: 108 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,30 @@
1+
import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
2+
import * as NodeServices from "@effect/platform-node/NodeServices";
13
import type { HostPowerSnapshot } from "@t3tools/contracts";
4+
import {
5+
RESOURCE_MONITOR_PROTOCOL_VERSION,
6+
ResourceMonitorCommand,
7+
type ResourceMonitorHelloEvent,
8+
} from "@t3tools/contracts";
29
import { describe, expect, it } from "@effect/vitest";
310
import * as DateTime from "effect/DateTime";
411
import * as Deferred from "effect/Deferred";
512
import * as Effect from "effect/Effect";
13+
import * as Exit from "effect/Exit";
614
import * as Fiber from "effect/Fiber";
15+
import * as Layer from "effect/Layer";
16+
import * as Queue from "effect/Queue";
717
import * as Ref from "effect/Ref";
18+
import * as Schema from "effect/Schema";
19+
import * as Scope from "effect/Scope";
820
import * as Semaphore from "effect/Semaphore";
21+
import * as Sink from "effect/Sink";
22+
import * as Stream from "effect/Stream";
23+
import { ChildProcessSpawner } from "effect/process";
924

25+
import * as ServerConfig from "../config.ts";
26+
import * as NativeTelemetryClient from "./NativeTelemetryClient.ts";
27+
import * as ResourceMonitorBinary from "./ResourceMonitorBinary.ts";
1028
import {
1129
canCommandNativeTelemetrySidecar,
1230
canRequestNativeTelemetryRetry,
@@ -40,25 +58,106 @@ describe("resolveNativeSampleIntervalMs", () => {
4058
expect(resolveNativeSampleIntervalMs({ ...basePower, onBattery: "true" }, 1)).toBe(5_000);
4159
});
4260

43-
it("slows background telemetry and serves live diagnostics at 1Hz", () => {
61+
it("stops timed sampling without a live subscriber and serves live diagnostics at 1Hz", () => {
4462
const unknown: HostPowerSnapshot = {
4563
...basePower,
4664
source: "unknown",
4765
stale: true,
4866
};
49-
expect(resolveNativeSampleIntervalMs(unknown, 0)).toBe(5_000);
67+
expect(resolveNativeSampleIntervalMs(unknown, 0)).toBe(0);
5068
expect(resolveNativeSampleIntervalMs(unknown, 1)).toBe(1_000);
51-
expect(
52-
resolveNativeSampleIntervalMs(
53-
{ ...basePower, stale: true, locked: "true", suspended: true },
54-
0,
55-
),
56-
).toBe(5_000);
57-
expect(resolveNativeSampleIntervalMs(basePower, 0)).toBe(5_000);
69+
expect(resolveNativeSampleIntervalMs({ ...basePower, locked: "true" }, 0)).toBe(0);
70+
expect(resolveNativeSampleIntervalMs({ ...basePower, onBattery: "true" }, 0)).toBe(0);
71+
expect(resolveNativeSampleIntervalMs(basePower, 0)).toBe(0);
5872
expect(resolveNativeSampleIntervalMs(basePower, 1)).toBe(1_000);
5973
});
6074
});
6175

76+
const decodeCommandLine = Schema.decodeUnknownSync(Schema.fromJsonString(ResourceMonitorCommand));
77+
78+
const hello: ResourceMonitorHelloEvent = {
79+
version: RESOURCE_MONITOR_PROTOCOL_VERSION,
80+
type: "hello",
81+
sidecarVersion: "test",
82+
sidecarPid: 4242,
83+
platform: "linux",
84+
arch: "x64",
85+
capabilities: {
86+
cumulativeCpuTime: true,
87+
currentCpuPercent: true,
88+
residentMemory: true,
89+
virtualMemory: true,
90+
ioBytes: true,
91+
processStartTime: true,
92+
processTree: true,
93+
},
94+
};
95+
96+
describe("NativeTelemetryClient", () => {
97+
it.effect("asks the sidecar for timed samples only while a live subscriber exists", () =>
98+
Effect.gen(function* () {
99+
const commands = yield* Queue.unbounded<ResourceMonitorCommand>();
100+
// A sidecar that says hello, stays alive, and records every command it receives.
101+
const spawner = ChildProcessSpawner.make(() =>
102+
Effect.succeed(
103+
ChildProcessSpawner.makeHandle({
104+
pid: ChildProcessSpawner.ProcessId(hello.sidecarPid),
105+
exitCode: Effect.never,
106+
isRunning: Effect.succeed(true),
107+
kill: () => Effect.void,
108+
unref: Effect.succeed(Effect.void),
109+
stdin: Sink.forEach((chunk: Uint8Array) =>
110+
Effect.forEach(new TextDecoder().decode(chunk).split("\n").filter(Boolean), (line) =>
111+
Queue.offer(commands, decodeCommandLine(line)),
112+
),
113+
),
114+
stdout: Stream.encodeText(Stream.make(`${JSON.stringify(hello)}\n`)).pipe(
115+
Stream.concat(Stream.never),
116+
),
117+
stderr: Stream.empty,
118+
all: Stream.empty,
119+
getInputFd: () => Sink.drain,
120+
getOutputFd: () => Stream.empty,
121+
}),
122+
),
123+
);
124+
const client = yield* NativeTelemetryClient.make().pipe(
125+
Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner),
126+
Effect.provideService(ResourceMonitorBinary.ResourceMonitorBinary, {
127+
resolve: Effect.succeed("/test/t3-resource-monitor"),
128+
}),
129+
);
130+
131+
const configure = yield* Queue.take(commands);
132+
expect(configure).toMatchObject({ type: "configure", sampleIntervalMs: 0 });
133+
expect(yield* Queue.take(commands)).toMatchObject({ type: "setExternalProcesses" });
134+
135+
const live = yield* Scope.make();
136+
yield* client.snapshots.pipe(Stream.runDrain, Effect.forkIn(live));
137+
expect(yield* Queue.take(commands)).toMatchObject({
138+
type: "setSampleInterval",
139+
sampleIntervalMs: 1_000,
140+
});
141+
expect(yield* Queue.take(commands)).toMatchObject({ type: "setStreaming", enabled: true });
142+
143+
yield* Scope.close(live, Exit.void);
144+
expect(yield* Queue.take(commands)).toMatchObject({
145+
type: "setSampleInterval",
146+
sampleIntervalMs: 0,
147+
});
148+
expect(yield* Queue.take(commands)).toMatchObject({ type: "setStreaming", enabled: false });
149+
expect((yield* client.health).sampleIntervalMs).toBe(0);
150+
}).pipe(
151+
Effect.provide(
152+
Layer.mergeAll(
153+
ServerConfig.layerTest(process.cwd(), { prefix: "t3-native-telemetry-test-" }),
154+
NodeCrypto.layer,
155+
).pipe(Layer.provideMerge(NodeServices.layer)),
156+
),
157+
),
158+
);
159+
});
160+
62161
describe("canRequestNativeTelemetryRetry", () => {
63162
it("only accepts retry while the supervisor is waiting without a live sidecar", () => {
64163
expect(canRequestNativeTelemetryRetry("degraded", false)).toBe(true);

‎apps/server/src/resourceTelemetry/NativeTelemetryClient.ts‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,8 @@ import * as ServerConfig from "../config.ts";
4040
import { subscribeBeforeSnapshotWithoutMutex } from "../utils/subscribeBeforeSnapshot.ts";
4141

4242
const SAMPLE_INTERVAL_MS = 1_000;
43-
const UNKNOWN_BACKGROUND_SAMPLE_INTERVAL_MS = 5_000;
43+
// The sidecar treats 0 as "sample only on request".
44+
const IDLE_SAMPLE_INTERVAL_MS = 0;
4445
const BATTERY_SAMPLE_INTERVAL_MS = 5_000;
4546
const CONSTRAINED_SAMPLE_INTERVAL_MS = 15_000;
4647
const HANDSHAKE_TIMEOUT = Duration.seconds(5);
@@ -258,13 +259,14 @@ function isThermallyConstrained(snapshot: HostPowerSnapshot): boolean {
258259
return snapshot.thermalState === "serious" || snapshot.thermalState === "critical";
259260
}
260261

262+
// Without a live subscriber the sidecar does not sample on a timer, so an idle server
263+
// never walks the process table. History then covers only time with a subscriber.
261264
export function resolveNativeSampleIntervalMs(
262265
snapshot: HostPowerSnapshot,
263266
liveSubscriberCount: number,
264267
): number {
265-
if (snapshot.stale || snapshot.source === "unknown") {
266-
return liveSubscriberCount > 0 ? SAMPLE_INTERVAL_MS : UNKNOWN_BACKGROUND_SAMPLE_INTERVAL_MS;
267-
}
268+
if (liveSubscriberCount === 0) return IDLE_SAMPLE_INTERVAL_MS;
269+
if (snapshot.stale || snapshot.source === "unknown") return SAMPLE_INTERVAL_MS;
268270
if (
269271
snapshot.suspended ||
270272
snapshot.locked === "true" ||
@@ -274,7 +276,7 @@ export function resolveNativeSampleIntervalMs(
274276
return CONSTRAINED_SAMPLE_INTERVAL_MS;
275277
}
276278
if (snapshot.onBattery === "true") return BATTERY_SAMPLE_INTERVAL_MS;
277-
return liveSubscriberCount > 0 ? SAMPLE_INTERVAL_MS : UNKNOWN_BACKGROUND_SAMPLE_INTERVAL_MS;
279+
return SAMPLE_INTERVAL_MS;
278280
}
279281

280282
export function commitCollectionControlUpdate<E, R>(
@@ -382,7 +384,7 @@ export const make = Effect.fn("resourceTelemetry.nativeTelemetryClient.make")(fu
382384
updatedAt: initializedAt,
383385
},
384386
liveSubscriberCount: 0,
385-
sampleIntervalMs: UNKNOWN_BACKGROUND_SAMPLE_INTERVAL_MS,
387+
sampleIntervalMs: IDLE_SAMPLE_INTERVAL_MS,
386388
});
387389
const appliedCollectionControl = yield* Ref.make(yield* Ref.get(collectionControl));
388390
const externalProcesses = yield* Ref.make<ReadonlyArray<ResourceMonitorExternalProcess>>([]);
@@ -1071,7 +1073,7 @@ export const layerTest = (
10711073
lastSampleAt: Option.none<DateTime.Utc>(),
10721074
lastError: Option.some("Resource monitor test implementation is unavailable."),
10731075
restartCount: 0,
1074-
sampleIntervalMs: UNKNOWN_BACKGROUND_SAMPLE_INTERVAL_MS,
1076+
sampleIntervalMs: IDLE_SAMPLE_INTERVAL_MS,
10751077
});
10761078
return Layer.succeed(
10771079
NativeTelemetryClient,

‎apps/web/src/components/settings/DiagnosticsSettings.tsx‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1038,11 +1038,17 @@ export function DiagnosticsSettingsPanel() {
10381038
<StatBlock
10391039
label="Samples"
10401040
value={resourceData ? formatCount(resourceData.retainedSampleCount) : "..."}
1041-
tooltip="In-memory process samples retained by the server. This resets when the server restarts."
1041+
tooltip="In-memory process samples retained by the server. The server samples only while diagnostics is open, and this resets when the server restarts."
10421042
/>
10431043
<StatBlock
10441044
label="Interval"
1045-
value={resourceData ? formatDuration(resourceData.sampleIntervalMs) : "..."}
1045+
value={
1046+
resourceData
1047+
? resourceData.sampleIntervalMs === 0
1048+
? "On demand"
1049+
: formatDuration(resourceData.sampleIntervalMs)
1050+
: "..."
1051+
}
10461052
/>
10471053
<StatBlock
10481054
label="Processes"

‎docs/internals/resource-telemetry.md‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,11 @@ protocol. A missing or failed collector leaves the server running.
88

99
## Collection cost
1010

11-
The native child owns sampling and bounded in-memory history. The server requests
12-
continuous snapshots only while diagnostics has live subscribers and fetches
13-
history on demand. Consuming host power for background scheduling must not retain
11+
The native child owns sampling and bounded in-memory history. It samples on a timer
12+
only while diagnostics has live subscribers. With none, the server sends a sample
13+
interval of 0 and the child samples only on request, so an idle server never walks
14+
the process table. History therefore covers only time with a live subscriber, plus
15+
on-demand samples. Consuming host power for background scheduling must not retain
1416
live diagnostics. There is no telemetry database or recurring shell-probe fallback.
1517

1618
History has independent bounds for age, snapshot count, process rows, and retained

0 commit comments

Comments
 (0)