Skip to content

Commit c0fdab2

Browse files
t3dotggclaude
andcommitted
fix(observability): a failing trace disk no longer stalls the server
When a trace file write failed, the sink put the whole backlog back in its buffer. Every later push then retried the full backlog, which took about 40% of the event loop after a few minutes of a failed disk. At about 120k records, the unshift threw RangeError, lost the backlog, and stopped the timed flush. Now a failed write drops the rest of its batch and counts it. The next flush logs the count once as a warning. The buffer stays at or below one batch, and writes continue when the disk recovers. Also fix the documented trace batch window default: it is 1000 ms, not 200 ms. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent a21b42c commit c0fdab2

3 files changed

Lines changed: 76 additions & 10 deletions

File tree

‎docs/operations/observability.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -584,7 +584,7 @@ Local trace file:
584584
- `T3CODE_TRACE_FILE`: override trace file path
585585
- `T3CODE_TRACE_MAX_BYTES`: per-file rotation size, default `10485760`
586586
- `T3CODE_TRACE_MAX_FILES`: rotated file count, default `10`
587-
- `T3CODE_TRACE_BATCH_WINDOW_MS`: flush window, default `200`
587+
- `T3CODE_TRACE_BATCH_WINDOW_MS`: flush window, default `1000`
588588
- `T3CODE_TRACE_MIN_LEVEL`: minimum trace level, default `Info`
589589
- `T3CODE_TRACE_TIMING_ENABLED`: enable timing metadata, default `true`
590590

‎packages/shared/src/observability.test.ts‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,9 @@ import * as Ref from "effect/Ref";
1212
import * as References from "effect/References";
1313
import * as Schema from "effect/Schema";
1414
import * as Tracer from "effect/Tracer";
15+
import { vi } from "vite-plus/test";
1516

17+
import { RotatingFileSink } from "./logging.ts";
1618
import {
1719
causeErrorTag,
1820
compactTraceAttributes,
@@ -376,6 +378,60 @@ describe("observability", () => {
376378
),
377379
);
378380

381+
it.effect("drops records after a failed write instead of retrying them on every push", () => {
382+
const warnings: Array<unknown> = [];
383+
const captureWarnings = Logger.make(({ logLevel, message }) => {
384+
if (logLevel === "Warn") warnings.push(message);
385+
});
386+
387+
return Effect.gen(function* () {
388+
const fileSystem = yield* FileSystem.FileSystem;
389+
const path = yield* Path.Path;
390+
const tempDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-trace-sink-" });
391+
const tracePath = path.join(tempDir, "shared.trace.ndjson");
392+
// A directory at the trace path fails every append, like a full disk.
393+
yield* fileSystem.makeDirectory(tracePath);
394+
const write = vi.spyOn(RotatingFileSink.prototype, "write");
395+
yield* Effect.addFinalizer(() => Effect.sync(() => write.mockRestore()));
396+
397+
const sink = yield* makeTraceSink({
398+
filePath: tracePath,
399+
maxBytes: 1024 * 1024,
400+
maxFiles: 2,
401+
batchWindowMs: 10_000,
402+
});
403+
404+
for (let index = 0; index < 1_024; index += 1) {
405+
sink.push(makeRecord("lost", String(index)));
406+
}
407+
yield* sink.flush;
408+
409+
// One write per full batch of 256, never a growing backlog.
410+
assert.deepStrictEqual(
411+
write.mock.calls.map(([chunk]) => String(chunk).split("\n").length - 1),
412+
[256, 256, 256, 256],
413+
);
414+
expect(warnings).toEqual([
415+
[expect.any(String), { filePath: tracePath, droppedCount: 1_024 }],
416+
]);
417+
418+
// Once the disk recovers, new records are written again.
419+
yield* fileSystem.remove(tracePath, { recursive: true });
420+
sink.push(makeRecord("recovered"));
421+
yield* sink.flush;
422+
423+
const records = yield* readTraceRecords(tracePath);
424+
assert.deepStrictEqual(
425+
records.map((record) => record.name),
426+
["recovered"],
427+
);
428+
assert.equal(warnings.length, 1);
429+
}).pipe(
430+
Effect.scoped,
431+
Effect.provide(Logger.layer([captureWarnings], { mergeWithExisting: false })),
432+
);
433+
});
434+
379435
it.effect("writes nested spans to disk and captures log messages as span events", () =>
380436
Effect.scoped(
381437
Effect.gen(function* () {

‎packages/shared/src/observability.ts‎

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -371,6 +371,8 @@ export const makeTraceSink = Effect.fn("makeTraceSink")(function* (options: Trac
371371
});
372372

373373
let buffer: Array<string> = [];
374+
// Records lost to failed writes since the last flush reported them.
375+
let droppedCount = 0;
374376
let pendingFlushStats: TraceSinkFlushStats = {
375377
logicalWriteBytes: 0,
376378
count: 0,
@@ -407,7 +409,10 @@ export const makeTraceSink = Effect.fn("makeTraceSink")(function* (options: Trac
407409
try {
408410
sink.write(chunk);
409411
} catch {
410-
buffer.unshift(...records.slice(persistedCount));
412+
// A failing disk (ENOSPC, EACCES, EIO) drops the rest of the batch.
413+
// Retrying it would grow the backlog, and every later push would
414+
// retry all of it.
415+
droppedCount += records.length - persistedCount;
411416
return;
412417
}
413418
pendingFlushStats = {
@@ -419,21 +424,26 @@ export const makeTraceSink = Effect.fn("makeTraceSink")(function* (options: Trac
419424
}
420425
};
421426

422-
const flush = Effect.sync(() => {
427+
const flush = Effect.gen(function* () {
423428
flushUnsafe();
424429
const stats = pendingFlushStats;
430+
const dropped = droppedCount;
425431
pendingFlushStats = {
426432
logicalWriteBytes: 0,
427433
count: 0,
428434
durationMs: 0,
429435
};
430-
return stats;
431-
}).pipe(
432-
Effect.flatMap((stats) =>
433-
stats.count > 0 && options.onFlush ? options.onFlush(stats).pipe(Effect.ignore) : Effect.void,
434-
),
435-
Effect.withTracerEnabled(false),
436-
);
436+
droppedCount = 0;
437+
if (stats.count > 0 && options.onFlush) {
438+
yield* options.onFlush(stats).pipe(Effect.ignore);
439+
}
440+
if (dropped > 0) {
441+
yield* Effect.logWarning("Dropped trace records after a failed write", {
442+
filePath: options.filePath,
443+
droppedCount: dropped,
444+
});
445+
}
446+
}).pipe(Effect.withTracerEnabled(false));
437447

438448
yield* Effect.addFinalizer(() => flush.pipe(Effect.ignore));
439449
yield* Effect.forkScoped(

0 commit comments

Comments
 (0)