Skip to content

Commit 6fd1017

Browse files
committed
perf(logging): move local trace and provider log writes off the event loop
1 parent 8b460a3 commit 6fd1017

7 files changed

Lines changed: 497 additions & 160 deletions

File tree

‎apps/server/src/provider/EventNdjsonLogger.test.ts‎

Lines changed: 93 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,13 @@ import * as NodeOS from "node:os";
44
import * as NodePath from "node:path";
55

66
import { ThreadId } from "@t3tools/contracts";
7-
import { assert, describe, it } from "@effect/vitest";
7+
import * as NodeServices from "@effect/platform-node/NodeServices";
8+
import { assert, it } from "@effect/vitest";
9+
import { RotatingFileSinkError } from "@t3tools/shared/logging";
810
import * as Clock from "effect/Clock";
911
import * as Effect from "effect/Effect";
12+
import * as Deferred from "effect/Deferred";
13+
import * as FileSystem from "effect/FileSystem";
1014
import * as Fiber from "effect/Fiber";
1115
import * as Logger from "effect/Logger";
1216
import * as Schema from "effect/Schema";
@@ -49,7 +53,7 @@ function parseLogLine(line: string) {
4953
};
5054
}
5155

52-
describe("EventNdjsonLogger", () => {
56+
it.layer(NodeServices.layer)("EventNdjsonLogger", (it) => {
5357
it.effect("summarizes circular events without exposing their contents in diagnostics", () => {
5458
const messages: Array<unknown> = [];
5559
const logCapture = Logger.make<unknown, void>(({ message }) => {
@@ -256,8 +260,8 @@ describe("EventNdjsonLogger", () => {
256260

257261
assert.equal(NodeFS.existsSync(threadPath), false);
258262
yield* TestClock.adjust(1_000);
259-
assert.equal(NodeFS.existsSync(threadPath), true);
260263
yield* store.close();
264+
assert.equal(NodeFS.existsSync(threadPath), true);
261265
} finally {
262266
NodeFS.rmSync(tempDir, { recursive: true, force: true });
263267
}
@@ -281,10 +285,10 @@ describe("EventNdjsonLogger", () => {
281285
yield* logger.write({ id: "accepted" }, ThreadId.make("thread-interrupted"));
282286

283287
yield* TestClock.adjust(1_000);
288+
yield* store.close();
284289

285290
assert.equal(NodeFS.existsSync(threadPath), true);
286291
assert.include(NodeFS.readFileSync(threadPath, "utf8"), '{"id":"accepted"}');
287-
yield* store.close();
288292
} finally {
289293
NodeFS.rmSync(tempDir, { recursive: true, force: true });
290294
}
@@ -788,39 +792,99 @@ describe("EventNdjsonLogger", () => {
788792
}),
789793
);
790794

791-
it("attributes batches that were written before a later chunk fails", () => {
792-
const records: ReadonlyArray<PendingRecord> = [
793-
{
794-
stream: "native",
795-
threadSegment: "thread",
796-
line: "first",
797-
bytes: 5,
798-
},
799-
{
800-
stream: "canonical",
801-
threadSegment: "thread",
802-
line: "second",
803-
bytes: 6,
804-
},
805-
];
806-
const attributed: Array<PendingRecord> = [];
807-
let writes = 0;
808-
809-
assert.throws(() =>
810-
writeBatchedMessages(
795+
it.effect("attributes batches that were written before a later chunk fails", () =>
796+
Effect.gen(function* () {
797+
const records: ReadonlyArray<PendingRecord> = [
798+
{
799+
stream: "native",
800+
threadSegment: "thread",
801+
line: "first",
802+
bytes: 5,
803+
},
804+
{
805+
stream: "canonical",
806+
threadSegment: "thread",
807+
line: "second",
808+
bytes: 6,
809+
},
810+
];
811+
const attributed: Array<PendingRecord> = [];
812+
let writes = 0;
813+
814+
const result = yield* writeBatchedMessages(
811815
{
812816
write: () => {
813817
writes += 1;
814-
if (writes === 2) throw new Error("simulated disk exhaustion");
818+
return writes === 2
819+
? Effect.fail(
820+
new RotatingFileSinkError({
821+
operation: "write",
822+
filePath: "test.log",
823+
cause: "simulated disk exhaustion",
824+
}),
825+
)
826+
: Effect.void;
815827
},
816828
},
817829
records,
818830
5,
819831
(written) => attributed.push(...written),
820-
),
821-
);
822-
assert.deepEqual(attributed, [records[0]]);
823-
});
832+
).pipe(Effect.result);
833+
assert.equal(result._tag, "Failure");
834+
assert.deepEqual(attributed, [records[0]]);
835+
}),
836+
);
837+
838+
it.effect(
839+
"keeps unrelated work responsive and drains before close while a disk write is held",
840+
() =>
841+
Effect.gen(function* () {
842+
const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-"));
843+
const basePath = NodePath.join(tempDir, "events.log");
844+
const fs = yield* FileSystem.FileSystem;
845+
const started = yield* Deferred.make<void>();
846+
const release = yield* Deferred.make<void>();
847+
let writes = 0;
848+
try {
849+
const store = yield* makeEventNdjsonLogStore(basePath, { batchWindowMs: 0 }).pipe(
850+
Effect.provideService(FileSystem.FileSystem, {
851+
...fs,
852+
writeFile: (filePath, content, options) =>
853+
Effect.gen(function* () {
854+
if (++writes === 1) {
855+
yield* Deferred.succeed(started, undefined);
856+
yield* Deferred.await(release);
857+
}
858+
yield* fs.writeFile(filePath, content, options);
859+
}),
860+
}),
861+
);
862+
const first = yield* store
863+
.logger("native")
864+
.write({ id: "first" }, ThreadId.make("held"))
865+
.pipe(Effect.forkChild);
866+
yield* Deferred.await(started);
867+
const closed = yield* Deferred.make<void>();
868+
const close = yield* store
869+
.close()
870+
.pipe(Effect.andThen(Deferred.succeed(closed, undefined)), Effect.forkChild);
871+
assert.equal(yield* Deferred.isDone(closed), false);
872+
// A separate fiber runs while the file operation remains pending.
873+
const heartbeat = yield* Deferred.make<void>();
874+
yield* Effect.forkChild(Deferred.succeed(heartbeat, undefined));
875+
yield* Deferred.await(heartbeat);
876+
yield* Deferred.succeed(release, undefined);
877+
yield* Fiber.join(first);
878+
yield* Fiber.join(close);
879+
assert.include(
880+
NodeFS.readFileSync(ownedLogPath(basePath, "held"), "utf8"),
881+
'{"id":"first"}',
882+
);
883+
} finally {
884+
NodeFS.rmSync(tempDir, { recursive: true, force: true });
885+
}
886+
}),
887+
);
824888

825889
it.effect("reports logical provider log writes to resource attribution", () =>
826890
Effect.gen(function* () {

0 commit comments

Comments
 (0)