Skip to content

Commit 13e3774

Browse files
t3dotggclaude
andcommitted
fix(observability): remove a partial heap snapshot when the write fails
If v8.writeHeapSnapshot failed after it opened the file, for example on a full disk, the partial file stayed in the logs dir. It can hold secrets and uses disk space. The failure handler now removes the file and still logs the original warning. A focused test fakes a failed write and checks that the logs dir is empty. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent ece83d5 commit 13e3774

2 files changed

Lines changed: 61 additions & 17 deletions

File tree

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
// @effect-diagnostics nodeBuiltinImport:off - tests fake a failed write at the native v8 boundary.
2+
import * as NodeServices from "@effect/platform-node/NodeServices";
3+
import * as NodeFS from "node:fs";
4+
import * as NodePath from "node:path";
5+
import * as NodeV8 from "node:v8";
6+
import { assert, it } from "@effect/vitest";
7+
import * as Effect from "effect/Effect";
8+
import * as FileSystem from "effect/FileSystem";
9+
import { vi } from "vite-plus/test";
10+
11+
import { writeHeapSnapshot } from "./HeapSnapshot.ts";
12+
13+
vi.mock("node:v8", async (importOriginal) => {
14+
const actual = await importOriginal<typeof NodeV8>();
15+
return { ...actual, writeHeapSnapshot: vi.fn(actual.writeHeapSnapshot) };
16+
});
17+
18+
it.layer(NodeServices.layer)("writeHeapSnapshot", (it) => {
19+
it.effect("removes the partial file when the write fails", () =>
20+
Effect.gen(function* () {
21+
const fs = yield* FileSystem.FileSystem;
22+
const logsDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-heap-snapshot-test-" });
23+
let partialPath: string | undefined;
24+
vi.mocked(NodeV8.writeHeapSnapshot).mockImplementationOnce((path) => {
25+
partialPath = path;
26+
if (path) NodeFS.writeFileSync(path, "partial");
27+
throw new Error("ENOSPC: no space left on device");
28+
});
29+
30+
yield* writeHeapSnapshot(logsDir);
31+
32+
assert.strictEqual(NodePath.dirname(partialPath ?? ""), logsDir);
33+
assert.deepEqual(yield* fs.readDirectory(logsDir), []);
34+
}),
35+
);
36+
});

‎apps/server/src/observability/HeapSnapshot.ts‎

Lines changed: 25 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,34 @@ import * as NodeV8 from "node:v8";
55
import { HostProcessPlatform } from "@t3tools/shared/hostProcess";
66
import * as DateTime from "effect/DateTime";
77
import * as Effect from "effect/Effect";
8+
import * as FileSystem from "effect/FileSystem";
89
import * as Layer from "effect/Layer";
910

1011
import * as ServerConfig from "../config.ts";
1112

1213
/**
13-
* Writes a V8 heap snapshot of the server into its logs dir when the process
14-
* gets SIGUSR2 (`kill -USR2 <pid>`), so a maintainer can see what a
15-
* long-running server holds. See "Heap Snapshots" in
16-
* docs/operations/observability.md.
14+
* Writes one V8 heap snapshot into `logsDir` and logs its path. A failed write
15+
* logs a warning and removes any partial file, because that file can hold
16+
* secrets and the failure is often a full disk.
17+
*/
18+
export const writeHeapSnapshot = Effect.fn("server.heapSnapshot", { root: true })(
19+
function* (logsDir: string) {
20+
const fs = yield* FileSystem.FileSystem;
21+
const timestamp = DateTime.formatIso(yield* DateTime.now).replaceAll(":", "-");
22+
const path = NodePath.join(logsDir, `server-${process.pid}-${timestamp}.heapsnapshot`);
23+
yield* Effect.annotateCurrentSpan({ path });
24+
yield* Effect.try(() => NodeV8.writeHeapSnapshot(path)).pipe(
25+
Effect.tapError(() => fs.remove(path, { force: true }).pipe(Effect.ignore)),
26+
);
27+
yield* Effect.logInfo("Wrote heap snapshot.", { path });
28+
},
29+
Effect.catch((cause) => Effect.logWarning("Failed to write heap snapshot.", { cause })),
30+
);
31+
32+
/**
33+
* Writes a heap snapshot when the process gets SIGUSR2 (`kill -USR2 <pid>`),
34+
* so a maintainer can see what a long-running server holds. See "Heap
35+
* Snapshots" in docs/operations/observability.md.
1736
*
1837
* `writeHeapSnapshot` is synchronous on the only JS thread, so two snapshots
1938
* never overlap: a signal sent during a write waits until it finishes. Windows
@@ -23,21 +42,10 @@ export const layer = Layer.effectDiscard(
2342
Effect.gen(function* () {
2443
if ((yield* HostProcessPlatform) === "win32") return;
2544
const { logsDir } = yield* ServerConfig.ServerConfig;
26-
const runFork = Effect.runForkWith(yield* Effect.context<never>());
27-
28-
const writeSnapshot = Effect.gen(function* () {
29-
const timestamp = DateTime.formatIso(yield* DateTime.now).replaceAll(":", "-");
30-
const path = NodePath.join(logsDir, `server-${process.pid}-${timestamp}.heapsnapshot`);
31-
yield* Effect.annotateCurrentSpan({ path });
32-
yield* Effect.sync(() => NodeV8.writeHeapSnapshot(path));
33-
yield* Effect.logInfo("Wrote heap snapshot.", { path });
34-
}).pipe(
35-
Effect.catchDefect((cause) => Effect.logWarning("Failed to write heap snapshot.", { cause })),
36-
Effect.withSpan("server.heapSnapshot", { root: true }),
37-
);
45+
const runFork = Effect.runForkWith(yield* Effect.context<FileSystem.FileSystem>());
3846

3947
const onSignal = () => {
40-
runFork(writeSnapshot);
48+
runFork(writeHeapSnapshot(logsDir));
4149
};
4250
yield* Effect.acquireRelease(
4351
Effect.sync(() => {

0 commit comments

Comments
 (0)