Skip to content

Commit e2d9d9a

Browse files
only21milclaudemaria-rcks
authored
fix(server): provider logs of finished threads obey retention limits (#23274)
Co-authored-by: Claude Opus 5.5 <[email protected]> Co-authored-by: maria-rcks <[email protected]>
1 parent e9485df commit e2d9d9a

2 files changed

Lines changed: 51 additions & 0 deletions

File tree

‎apps/server/src/provider/EventNdjsonLogger.test.ts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -420,6 +420,45 @@ it.layer(NodeServices.layer)("EventNdjsonLogger", (it) => {
420420
}),
421421
);
422422

423+
it.effect("retires a thread log once a retention interval passes without writes", () =>
424+
Effect.gen(function* () {
425+
const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-"));
426+
const basePath = NodePath.join(tempDir, "events.log");
427+
const finishedPath = ownedLogPath(basePath, "finished");
428+
const activePath = ownedLogPath(basePath, "active");
429+
430+
try {
431+
const store = yield* makeEventNdjsonLogStore(basePath, {
432+
batchWindowMs: 0,
433+
});
434+
const logger = store.logger("native");
435+
436+
yield* logger.write({ id: "finished-first" }, ThreadId.make("finished"));
437+
yield* TestClock.adjust("5 minutes");
438+
yield* logger.write({ id: "active-first" }, ThreadId.make("active"));
439+
assert.equal(NodeFS.existsSync(finishedPath), true);
440+
NodeFS.truncateSync(finishedPath, 512 * 1024 * 1024 + 1);
441+
442+
yield* TestClock.adjust("5 minutes");
443+
yield* logger.write({ id: "active-second" }, ThreadId.make("active"));
444+
assert.equal(NodeFS.existsSync(finishedPath), false);
445+
assert.equal(NodeFS.existsSync(activePath), true);
446+
447+
yield* logger.write({ id: "finished-again" }, ThreadId.make("finished"));
448+
yield* store.close();
449+
assert.deepEqual(
450+
NodeFS.readFileSync(finishedPath, "utf8")
451+
.trim()
452+
.split("\n")
453+
.map((line) => parseLogLine(line).payload),
454+
['{"id":"finished-again"}'],
455+
);
456+
} finally {
457+
NodeFS.rmSync(tempDir, { recursive: true, force: true });
458+
}
459+
}),
460+
);
461+
423462
it.effect("attributes batches that were written before a later chunk fails", () =>
424463
Effect.gen(function* () {
425464
const records: ReadonlyArray<PendingRecord> = [

‎apps/server/src/provider/EventNdjsonLogger.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ interface StoreState {
114114
readonly pending: Array<PendingRecord>;
115115
readonly pendingBytes: number;
116116
readonly sinks: ReadonlyMap<string, AsyncFileSink>;
117+
readonly writtenSinceRetention: ReadonlySet<string>;
117118
readonly flushScheduled: boolean;
118119
readonly closed: boolean;
119120
readonly lastRetentionAt: number;
@@ -483,6 +484,7 @@ const drainPending = Effect.fnUntraced(function* (input: {
483484
}
484485

485486
const sinks = new Map(input.state.sinks);
487+
const writtenSegments = new Set(input.state.writtenSinceRetention);
486488
const failures: Array<FileOperationFailure> = [];
487489
const attributionByStream = new Map<
488490
EventNdjsonStream,
@@ -508,6 +510,7 @@ const drainPending = Effect.fnUntraced(function* (input: {
508510
sink = created.success;
509511
sinks.set(threadSegment, sink);
510512
}
513+
writtenSegments.add(threadSegment);
511514

512515
const written = yield* writeBatchedMessages(
513516
sink,
@@ -534,6 +537,13 @@ const drainPending = Effect.fnUntraced(function* (input: {
534537

535538
const retentionDue =
536539
input.now - input.state.lastRetentionAt >= input.options.retentionCheckIntervalMs;
540+
if (retentionDue) {
541+
// A sink idle for a whole retention interval reopens from disk on its next
542+
// write, so retention may remove its files without leaving a stale size.
543+
for (const threadSegment of sinks.keys()) {
544+
if (!writtenSegments.has(threadSegment)) sinks.delete(threadSegment);
545+
}
546+
}
537547
const retention = retentionDue
538548
? yield* Effect.promise(() =>
539549
enforceRetention({
@@ -563,6 +573,7 @@ const drainPending = Effect.fnUntraced(function* (input: {
563573
pending: [],
564574
pendingBytes: 0,
565575
sinks,
576+
writtenSinceRetention: retentionDue ? new Set() : writtenSegments,
566577
flushScheduled: input.timerFired ? false : input.state.flushScheduled,
567578
closed: input.close,
568579
lastRetentionAt: retentionDue ? input.now : input.state.lastRetentionAt,
@@ -629,6 +640,7 @@ export const makeEventNdjsonLogStore = Effect.fnUntraced(function* (
629640
pending: [],
630641
pendingBytes: 0,
631642
sinks: new Map(),
643+
writtenSinceRetention: new Set(),
632644
flushScheduled: false,
633645
closed: false,
634646
lastRetentionAt: initializedAt,

0 commit comments

Comments
 (0)