Skip to content

Commit afe2734

Browse files
donjor-agent[bot]rsi-pr-agent
authored andcommitted
fix(server): log memory high-water marks and snapshot the heap at 4 GiB (#1718)
## Problem `t3 serve` grew to 60 GB RSS plus 16 GB swap in 74 minutes, then froze the desktop. Nothing recorded which memory pool grew, so the cause is still unknown. ## Change New fork layer `MemoryWatch.fork.ts`, hooked beside `HeapSnapshot.layer`: - **Sample:** `process.memoryUsage()` every minute - **Log:** a warning at each new GiB past 2 GiB - **Fields:** rss, heap, external, arrayBuffers - **Snapshot:** once per process at 4 GiB - **Written by:** `writeHeapSnapshot` into `logsDir` `external` and `arrayBuffers` show whether the growth is buffers or native heap. The snapshot names what retains them. ## Scope and approval Refs #1717. The maintainer approved telemetry as the next step there. Diagnostics only; no behavior change below 2 GiB RSS. ## Verification - **Unit test:** `MemoryWatch.fork.test.ts` passes - **Covers:** thresholds, repeat steps, one snapshot - **Typecheck:** `tsc` on `apps/server`, exit 0 - **Fork CI:** `vp run fork:ci`, exit 0 - **Not checked:** a live 4 GiB run; no repro yet Claude Code, claude-opus-5-5. ## Fork trailers Squashes: - 96a3e9f fix(server): log memory high-water marks and snapshot the heap at 4 GiB (#1718) (#1718) - 9db81d1 fixup! fix(server): log memory high-water marks and snapshot the heap at 4 GiB (#1718) (#1718) Fork-Domain: upstream-fixes Fork-Tier: bugfix Fork-Upstreamable: yes
1 parent 8df49c9 commit afe2734

4 files changed

Lines changed: 111 additions & 0 deletions

File tree

‎apps/server/src/observability/HeapSnapshot.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,3 +36,5 @@ export const layer = Layer.effectDiscard(
3636
);
3737
}),
3838
);
39+
40+
export { writeHeapSnapshot }; // fork-hook: upstream-fixes/heap-snapshot-export
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
import { describe, expect, it } from "vite-plus/test";
2+
3+
import { initialState, step } from "./MemoryWatch.fork.ts";
4+
5+
const GiB = 1024 ** 3;
6+
7+
const run = (samplesGiB: ReadonlyArray<number>) => {
8+
let state = initialState;
9+
return samplesGiB.map((gib) => {
10+
const decision = step(state, gib * GiB);
11+
state = decision.next;
12+
return { report: decision.report, snapshot: decision.snapshot };
13+
});
14+
};
15+
16+
describe("MemoryWatch step", () => {
17+
it("reports each new GiB high-water mark from 2 GiB and snapshots once at 4 GiB", () => {
18+
expect(run([1.5, 2.1, 2.5, 3.0, 2.0, 4.2, 9.5, 9.9, 10.0, 4.5])).toEqual([
19+
{ report: false, snapshot: false },
20+
{ report: true, snapshot: false },
21+
{ report: false, snapshot: false },
22+
{ report: true, snapshot: false },
23+
{ report: false, snapshot: false },
24+
{ report: true, snapshot: true },
25+
{ report: true, snapshot: false },
26+
{ report: false, snapshot: false },
27+
{ report: true, snapshot: false },
28+
{ report: false, snapshot: false },
29+
]);
30+
});
31+
});
Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
import * as Effect from "effect/Effect";
2+
import * as Layer from "effect/Layer";
3+
4+
import * as ServerConfig from "../config.ts";
5+
import { writeHeapSnapshot } from "./HeapSnapshot.ts";
6+
7+
const GiB = 1024 ** 3;
8+
const MiB = 1024 ** 2;
9+
const FIRST_REPORT_BYTES = 2 * GiB;
10+
const REPORT_STEP_BYTES = 1 * GiB;
11+
const SNAPSHOT_BYTES = 4 * GiB;
12+
const SAMPLE_INTERVAL = "1 minute";
13+
14+
export interface MemoryWatchState {
15+
readonly nextReportBytes: number;
16+
readonly snapshotTaken: boolean;
17+
}
18+
19+
export const initialState: MemoryWatchState = {
20+
nextReportBytes: FIRST_REPORT_BYTES,
21+
snapshotTaken: false,
22+
};
23+
24+
/**
25+
* Decides one sample: report each new 1 GiB high-water mark past 2 GiB, and
26+
* write one heap snapshot per process past 4 GiB.
27+
*/
28+
export const step = (state: MemoryWatchState, rssBytes: number) => {
29+
const report = rssBytes >= state.nextReportBytes;
30+
const snapshot = !state.snapshotTaken && rssBytes >= SNAPSHOT_BYTES;
31+
return {
32+
report,
33+
snapshot,
34+
next: {
35+
nextReportBytes: report
36+
? rssBytes - (rssBytes % REPORT_STEP_BYTES) + REPORT_STEP_BYTES
37+
: state.nextReportBytes,
38+
snapshotTaken: state.snapshotTaken || snapshot,
39+
},
40+
};
41+
};
42+
43+
/**
44+
* Logs the server's memory breakdown at each new high-water mark, so a leak
45+
* leaves a durable record of which pool grew and when. `external` and
46+
* `arrayBuffers` grow outside the V8 heap limit; a heap snapshot names what
47+
* retains them. See RSI-Software/t3code-hyprws#1717.
48+
*/
49+
export const layer = Layer.effectDiscard(
50+
Effect.gen(function* () {
51+
const { logsDir } = yield* ServerConfig.ServerConfig;
52+
let state = initialState;
53+
const tick = Effect.gen(function* () {
54+
const usage = process.memoryUsage();
55+
const decision = step(state, usage.rss);
56+
state = decision.next;
57+
if (decision.report) {
58+
yield* Effect.logWarning(`server memory reached ${Math.round(usage.rss / MiB)} MiB`, {
59+
rssMb: Math.round(usage.rss / MiB),
60+
heapUsedMb: Math.round(usage.heapUsed / MiB),
61+
heapTotalMb: Math.round(usage.heapTotal / MiB),
62+
externalMb: Math.round(usage.external / MiB),
63+
arrayBuffersMb: Math.round(usage.arrayBuffers / MiB),
64+
});
65+
}
66+
if (decision.snapshot) {
67+
yield* writeHeapSnapshot(logsDir);
68+
}
69+
});
70+
yield* Effect.sleep(SAMPLE_INTERVAL).pipe(
71+
Effect.andThen(tick),
72+
Effect.forever,
73+
Effect.forkScoped,
74+
);
75+
}),
76+
);

‎apps/server/src/server.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@ import * as SourceControlRepositoryService from "./sourceControl/SourceControlRe
138138
import * as WorktreeSetupTracker from "./project/WorktreeSetupTracker.ts";
139139
import * as Observability from "./observability/Observability.ts";
140140
import * as HeapSnapshot from "./observability/HeapSnapshot.ts";
141+
import * as MemoryWatch from "./observability/MemoryWatch.fork.ts"; // fork-hook: upstream-fixes/memory-watch-import
141142
import * as EventLoopMonitor from "./observability/EventLoopMonitor.ts";
142143
import * as ServerEnvironment from "./environment/ServerEnvironment.ts";
143144
import * as DirectEndpoints from "./environment/DirectEndpoints.ts";
@@ -1130,6 +1131,7 @@ const layerMakeServer = Layer.unwrap(
11301131
layerTailscaleServe,
11311132
layerCloudDesiredLinkReconcile,
11321133
HeapSnapshot.layer,
1134+
MemoryWatch.layer, // fork-hook: upstream-fixes/memory-watch
11331135
);
11341136

11351137
return layerServerApplication.pipe(

0 commit comments

Comments
 (0)