Skip to content

Commit 7d8faad

Browse files
t3dotggJustYanniccclaude
committed
perf(server): prune thread event history older than a day
Projections already hold thread state, so compaction now deletes thread events older than 24 hours except the anchors other readers need, and records a per-thread replay floor in the same transaction. Thread and shell resumes from below a floor get a snapshot instead of an incomplete replay. Rebuild refuses once history was pruned. The background pass rests between pages so its deletes do not stall foreground writes behind WAL checkpoints. Co-authored-by: Yannic Charlon <[email protected]> Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 3eb8366 commit 7d8faad

13 files changed

Lines changed: 632 additions & 56 deletions

‎apps/server/src/cli/doctor.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ export const runDoctor = Effect.fn("runDoctor")(function* (
110110
.pipe(
111111
Effect.tap((result) =>
112112
Console.log(
113-
`Done: ${formatBytes(result.beforeBytes)} -> ${formatBytes(result.afterBytes)}, ${result.deletedEventCount} superseded events removed.`,
113+
`Done: ${formatBytes(result.beforeBytes)} -> ${formatBytes(result.afterBytes)}, ${result.deletedEventCount} old history events removed.`,
114114
),
115115
),
116116
),

‎apps/server/src/orchestration-v2/FoundationPersistence.test.ts‎

Lines changed: 261 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,12 +40,15 @@ import * as Ref from "effect/Ref";
4040
import * as Scheduler from "effect/Scheduler";
4141
import * as Schema from "effect/Schema";
4242
import * as Stream from "effect/Stream";
43+
import * as TestClock from "effect/testing/TestClock";
4344
import * as Tracer from "effect/Tracer";
4445
import * as SqlClient from "effect/sql/SqlClient";
4546
import * as Statement from "effect/sql/Statement";
4647

4748
import { LIVE_STREAM_MAX_ITEMS, LiveStreamBufferError } from "./LiveStreamBudget.ts";
49+
import * as OrchestrationEventStore from "../persistence/OrchestrationEventStore.ts";
4850
import * as SqlitePersistence from "../persistence/Sqlite.ts";
51+
import { subscribeOrchestrationV2Thread } from "../ws.ts";
4952
import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
5053
import * as CommandReceiptStore from "./CommandReceiptStore.ts";
5154
import * as EffectOutbox from "./EffectOutbox.ts";
@@ -55,6 +58,7 @@ import * as EventStore from "./EventStore.ts";
5558
import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
5659
import * as ProjectionMaintenance from "./ProjectionMaintenance.ts";
5760
import * as ProjectionStore from "./ProjectionStore.ts";
61+
import * as ThreadManagement from "./ThreadManagementService.ts";
5862
import * as ProjectStore from "./ProjectStore.ts";
5963
import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts";
6064
import * as TurnItemPositionStore from "./TurnItemPositionStore.ts";
@@ -219,6 +223,263 @@ it.effect("rebuilds event history one bounded page at a time", () =>
219223
}).pipe(Effect.provide(layerTest)),
220224
);
221225

226+
// Own database: advancing the clock would also expire other tests' history.
227+
it.effect(
228+
"retention prunes expired history to anchors and old cursors resume from a snapshot",
229+
() =>
230+
Effect.gen(function* () {
231+
const eventSink = yield* EventSink.EventSinkV2;
232+
const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2;
233+
const projections = yield* ProjectionStore.ProjectionStoreV2;
234+
const sql = yield* SqlClient.SqlClient;
235+
const createdAt = yield* DateTime.now;
236+
const threadId = ThreadId.make("thread:retention");
237+
const thread = makeThread(threadId, createdAt);
238+
const run: OrchestrationV2Run = {
239+
id: RunId.make("run:retention"),
240+
threadId,
241+
ordinal: 1,
242+
providerInstanceId,
243+
modelSelection,
244+
providerThreadId: null,
245+
userMessageId: MessageId.make("message:retention-1"),
246+
rootNodeId: null,
247+
activeAttemptId: RunAttemptId.make("run-attempt:retention"),
248+
status: "running",
249+
queuePosition: null,
250+
requestedAt: createdAt,
251+
startedAt: createdAt,
252+
completedAt: null,
253+
checkpointId: null,
254+
contextHandoffId: null,
255+
};
256+
const runEvent = (suffix: string): OrchestrationV2DomainEvent => ({
257+
id: EventId.make(`event:retention:${suffix}`),
258+
type: "run.updated",
259+
threadId,
260+
runId: run.id,
261+
occurredAt: createdAt,
262+
payload: run,
263+
});
264+
const messageEvent = (
265+
suffix: string,
266+
occurredAt: DateTime.Utc = createdAt,
267+
): OrchestrationV2DomainEvent => ({
268+
id: EventId.make(`event:retention:${suffix}`),
269+
type: "message.updated",
270+
threadId,
271+
occurredAt,
272+
payload: {
273+
createdBy: "user",
274+
creationSource: "web",
275+
id: MessageId.make(`message:retention-${suffix}`),
276+
threadId,
277+
runId: null,
278+
nodeId: null,
279+
role: "user",
280+
text: suffix,
281+
attachments: [],
282+
streaming: false,
283+
createdAt: occurredAt,
284+
updatedAt: occurredAt,
285+
},
286+
});
287+
288+
// Raw rows keep these fixtures out of projections: retention never decodes them.
289+
const insertRaw = (streamId: string, version: number, payload: object) => sql`
290+
INSERT INTO orchestration_events (
291+
event_id, aggregate_kind, stream_id, stream_version, event_type,
292+
occurred_at, actor_kind, payload_json, metadata_json, application_event_version
293+
) VALUES (
294+
${`event:${streamId}:${version}`}, 'thread', ${streamId}, ${version}, 'turn-item.updated',
295+
${DateTime.formatIso(createdAt)}, 'server', ${JSON.stringify(payload)}, '{}', 2
296+
)
297+
`;
298+
// Slimmed payloads restore their prompt from this row.
299+
yield* insertRaw("thread:retention-prompt", 0, { id: "item", prompt: "full prompt" });
300+
yield* insertRaw("thread:retention-prompt", 1, { id: "item", prompt: "", promptSequence: 1 });
301+
yield* insertRaw("thread:retention-prompt", 2, { id: "item", prompt: "", promptSequence: 1 });
302+
// The transcript import still reads its imported event IDs.
303+
yield* insertRaw("thread:retention-importing", 0, { id: "imported-1" });
304+
yield* insertRaw("thread:retention-importing", 1, { id: "imported-2" });
305+
yield* sql`
306+
INSERT INTO orchestration_v2_legacy_imports (
307+
thread_id, source_updated_at, shell_imported_at, transcript_imported_at,
308+
imported_message_count, last_error
309+
) VALUES (
310+
'thread:retention-importing', ${DateTime.formatIso(createdAt)}, ${DateTime.formatIso(createdAt)}, NULL, 0, NULL
311+
)
312+
`;
313+
const deletedThread = makeThread(ThreadId.make("thread:retention-deleted"), createdAt);
314+
const deletedCreated = threadCreatedEvent({
315+
id: "event:retention:deleted-created",
316+
thread: deletedThread,
317+
now: createdAt,
318+
});
319+
yield* eventSink.write({
320+
events: [
321+
deletedCreated,
322+
{ ...messageEvent("deleted-thread-message"), threadId: deletedThread.id },
323+
],
324+
});
325+
326+
yield* eventSink.write({
327+
events: [
328+
threadCreatedEvent({ id: "event:retention:created", thread, now: createdAt }),
329+
runEvent("first-running"),
330+
messageEvent("expired-1"),
331+
runEvent("later-running"),
332+
messageEvent("expired-2"),
333+
],
334+
});
335+
// A retry of this command still reads its result events.
336+
yield* eventSink.commitCommand({
337+
commandId: CommandId.make("command:retention"),
338+
threadId,
339+
commandType: "thread.send",
340+
acceptedAt: createdAt,
341+
events: [messageEvent("receipt")],
342+
effects: [],
343+
});
344+
yield* TestClock.adjust(Duration.hours(25));
345+
const recentAt = yield* DateTime.now;
346+
yield* eventSink.write({ events: [messageEvent("recent", recentAt)] });
347+
yield* eventSink.write({
348+
events: [
349+
{
350+
id: EventId.make("event:retention:deleted"),
351+
type: "thread.deleted",
352+
threadId: deletedThread.id,
353+
providerInstanceId,
354+
occurredAt: recentAt,
355+
payload: { ...deletedThread, deletedAt: recentAt },
356+
},
357+
],
358+
});
359+
const sequenceOf = (suffix: string) =>
360+
sql<{ readonly sequence: number }>`
361+
SELECT sequence FROM orchestration_events
362+
WHERE event_id = ${`event:retention:${suffix}`}
363+
`.pipe(Effect.map((rows) => rows[0]!.sequence));
364+
const oldCursor = yield* sequenceOf("first-running");
365+
const lastExpired = yield* sequenceOf("expired-2");
366+
const beforeRetention = yield* projections.getThreadProjection(threadId);
367+
368+
assert.equal((yield* maintenance.compactEventStore).deletedEventCount, 4);
369+
const survivors = yield* sql<{ readonly event_id: string }>`
370+
SELECT event_id FROM orchestration_events
371+
WHERE stream_id IN ('thread:retention-prompt', 'thread:retention-importing')
372+
ORDER BY sequence
373+
`;
374+
assert.deepEqual(
375+
survivors.map((row) => row.event_id),
376+
[
377+
"event:thread:retention-prompt:0",
378+
"event:thread:retention-prompt:2",
379+
"event:thread:retention-importing:0",
380+
"event:thread:retention-importing:1",
381+
],
382+
);
383+
const remaining = yield* sql<{ readonly event_id: string }>`
384+
SELECT event_id FROM orchestration_events WHERE stream_id = ${threadId} ORDER BY sequence
385+
`;
386+
assert.deepEqual(
387+
remaining.map((row) => row.event_id),
388+
[
389+
"event:retention:created",
390+
"event:retention:first-running",
391+
"event:retention:later-running",
392+
"event:retention:receipt",
393+
"event:retention:recent",
394+
],
395+
);
396+
const floors = yield* sql<{ readonly replay_floor: number }>`
397+
SELECT replay_floor FROM orchestration_v2_event_retention WHERE thread_id = ${threadId}
398+
`;
399+
assert.deepEqual(
400+
floors.map((row) => row.replay_floor),
401+
[lastExpired],
402+
);
403+
assert.deepEqual(yield* projections.getThreadProjection(threadId), beforeRetention);
404+
assert.isTrue((yield* maintenance.verify).valid);
405+
assert.equal((yield* Effect.exit(maintenance.rebuild))._tag, "Failure");
406+
assert.deepEqual(yield* projections.getThreadProjection(threadId), beforeRetention);
407+
408+
const resume = (afterSequence: number, resumedThreadId = threadId) =>
409+
subscribeOrchestrationV2Thread({
410+
threadId: resumedThreadId,
411+
afterSequence,
412+
requestCompletionMarker: true,
413+
}).pipe(
414+
Effect.flatMap((stream) =>
415+
stream.pipe(
416+
Stream.takeUntil((item) => item.kind === "synchronized"),
417+
Stream.map((item) => (item.kind === "event" ? item.sequence : item.kind)),
418+
Stream.runCollect,
419+
),
420+
),
421+
Effect.provideService(ThreadManagement.ThreadManagementService, {
422+
ensureLegacyTranscript: () => Effect.void,
423+
getThreadShell: projections.getThreadShell,
424+
getThreadSnapshot: projections.getThreadSnapshot,
425+
streamStoredEventsFrom: () => Stream.never,
426+
} as unknown as ThreadManagement.ThreadManagementService["Service"]),
427+
Effect.provide(OrchestrationEventStore.layer),
428+
);
429+
// Below the floor the replay would skip deleted events, so the client gets a snapshot.
430+
assert.deepEqual(Array.from(yield* resume(oldCursor)), ["snapshot", "synchronized"]);
431+
assert.deepEqual(Array.from(yield* resume(lastExpired)), [
432+
yield* sequenceOf("receipt"),
433+
yield* sequenceOf("recent"),
434+
"synchronized",
435+
]);
436+
// A deleted thread has no snapshot; the client learns of the deletion from its tombstone.
437+
assert.deepEqual(
438+
Array.from(yield* resume(yield* sequenceOf("deleted-created"), deletedThread.id)),
439+
[yield* sequenceOf("deleted"), "synchronized"],
440+
);
441+
}).pipe(Effect.provide(layerTest)),
442+
);
443+
444+
// A newer CLI's doctor can open a database whose server never ran migration 70.
445+
it.effect("retention stays off without the replay floor table", () =>
446+
Effect.gen(function* () {
447+
const eventSink = yield* EventSink.EventSinkV2;
448+
const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2;
449+
const sql = yield* SqlClient.SqlClient;
450+
const now = yield* DateTime.now;
451+
const thread = makeThread(ThreadId.make("thread:retention-pre-70"), now);
452+
yield* eventSink.write({
453+
events: [
454+
threadCreatedEvent({ id: "event:pre-70:created", thread, now }),
455+
...["a", "b"].map((suffix): OrchestrationV2DomainEvent => ({
456+
id: EventId.make(`event:pre-70:${suffix}`),
457+
type: "message.updated",
458+
threadId: thread.id,
459+
occurredAt: now,
460+
payload: {
461+
createdBy: "user",
462+
creationSource: "web",
463+
id: MessageId.make(`message:pre-70-${suffix}`),
464+
threadId: thread.id,
465+
runId: null,
466+
nodeId: null,
467+
role: "user",
468+
text: suffix,
469+
attachments: [],
470+
streaming: false,
471+
createdAt: now,
472+
updatedAt: now,
473+
},
474+
})),
475+
],
476+
});
477+
yield* sql`DROP TABLE orchestration_v2_event_retention`;
478+
yield* TestClock.adjust(Duration.hours(25));
479+
assert.equal((yield* maintenance.compactEventStore).deletedEventCount, 0);
480+
}).pipe(Effect.provide(layerTest)),
481+
);
482+
222483
it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
223484
it.effect("projects oversized tool bodies before both replay and live RPC retention", () =>
224485
Effect.scoped(

0 commit comments

Comments
 (0)