Skip to content

Commit dd237e7

Browse files
t3dotggclaude
andcommitted
fix(observability): warn when a stall cannot be written to the stall log
The stall sink swallowed append failures, so a stall log that could not be written lost every line without a warning. It now throws, and the monitor logs a fixed warning with the file path. The stall span is recorded either way. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 10898e6 commit dd237e7

2 files changed

Lines changed: 31 additions & 1 deletion

File tree

‎apps/server/src/observability/EventLoopMonitor.test.ts‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { assert, describe, it } from "@effect/vitest";
33
import * as Effect from "effect/Effect";
44
import * as FileSystem from "effect/FileSystem";
55
import * as Layer from "effect/Layer";
6+
import * as Logger from "effect/Logger";
67
import * as Tracer from "effect/Tracer";
78
import * as TestClock from "effect/testing/TestClock";
89

@@ -114,6 +115,30 @@ describe("EventLoopMonitor", () => {
114115
}).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
115116
);
116117

118+
it.effect("warns when a stall cannot be written to the stall log", () => {
119+
const warnings: Array<unknown> = [];
120+
const logger = Logger.make(({ message }) => {
121+
warnings.push(Array.isArray(message) ? message[0] : message);
122+
});
123+
return Effect.gen(function* () {
124+
const fs = yield* FileSystem.FileSystem;
125+
// A directory at the log path opens, but every append to it fails. The cap
126+
// is large so the sink never rotates the directory out of the way.
127+
const filePath = yield* fs.makeTempDirectoryScoped({ prefix: "t3-event-loop-" });
128+
yield* monitor([stalled, stalled], { filePath, maxBytes: 1024 * 1024 });
129+
yield* TestClock.adjust("60 seconds");
130+
assert.deepStrictEqual(warnings, [
131+
"event loop stalled for 4950 ms",
132+
"event loop stall log write failed",
133+
]);
134+
}).pipe(
135+
Effect.scoped,
136+
Effect.provide(
137+
Layer.merge(NodeServices.layer, Logger.layer([logger], { mergeWithExisting: false })),
138+
),
139+
);
140+
});
141+
117142
it("ignores delay the loop spent idle, such as a system sleep", () => {
118143
// Waking from sleep reads as a long gap, but the loop was idle in poll for it.
119144
const asleep: EventLoopMonitor.EventLoopReadings = {

‎apps/server/src/observability/EventLoopMonitor.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,7 @@ export const layerWith = (
121121
filePath: stallLog.filePath,
122122
maxBytes: stallLog.maxBytes,
123123
maxFiles: 1,
124+
throwOnError: true,
124125
}),
125126
).pipe(
126127
Effect.catch(() =>
@@ -153,7 +154,11 @@ export const layerWith = (
153154
const time = DateTime.formatIso(yield* DateTime.now);
154155
yield* encodeStallLine({ time, ...attributes }).pipe(
155156
Effect.flatMap((line) => Effect.try(() => stallSink.write(`${line}\n`))),
156-
Effect.ignore,
157+
Effect.catch(() =>
158+
Effect.logWarning("event loop stall log write failed", {
159+
filePath: stallLog.filePath,
160+
}),
161+
),
157162
);
158163
}
159164
});

0 commit comments

Comments
 (0)