Skip to content

Commit cf8be3b

Browse files
fix(server): compact superseded history in the background
Superseded orchestration events (whole-thread state, message and node updates) accumulated for the lifetime of a database; one install reached 114 GB. compactEventStore existed but nothing in the running server called it, and SQLite never returned the freed pages to the filesystem. The new DatabaseStorage service runs compactEventStore six-hourly after startup and hands freed pages back every minute with incremental_vacuum, one page per step so client requests interleave. New databases are created with auto_vacuum = INCREMENTAL; existing files keep their mode until an offline VACUUM converts them. thread.pull-request-link-updated joins the compaction rules, keyed by the link's host/repository/number. No replay boundary is published. Whole-thread events keep only the newest per thread. Keyed entities (messages, nodes, PR links) keep both their oldest and newest event: client reducers append an entity where they first see it, so a replay from any cursor either carries that first event or finds the entity already present, and ordering survives. thread.deleted is never compacted because clients switch on the event type. The scan uses 50-row pages so discovery on the shared connection stays short. Co-Authored-By: Claude Fable 5.1 <[email protected]>
1 parent 8414d16 commit cf8be3b

9 files changed

Lines changed: 377 additions & 16 deletions

File tree

‎apps/server/src/mcp/ThreadMetadataMcpService.ts‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,10 @@ function resultFromThread(input: {
111111
readonly action: ThreadMetadataMcpAction;
112112
readonly commandId: CommandId;
113113
readonly sequence: number;
114-
readonly thread: OrchestrationV2AppThread;
114+
readonly thread: Pick<
115+
OrchestrationV2AppThread,
116+
"id" | "title" | "titleRegeneration" | "linkedPullRequest" | "updatedAt"
117+
>;
115118
}): ThreadMetadataMcpUpdateResult {
116119
return {
117120
threadId: input.thread.id,
@@ -199,7 +202,15 @@ const make = Effect.gen(function* () {
199202
const metadataEvent = dispatched.storedEvents.find(
200203
(stored) => stored.event.type === "thread.metadata-updated",
201204
);
202-
if (metadataEvent === undefined || metadataEvent.event.type !== "thread.metadata-updated") {
205+
// A retried commandId replays the stored events of the first dispatch;
206+
// compaction may have deleted them by then, so answer from the projection.
207+
const thread =
208+
metadataEvent?.event.type === "thread.metadata-updated"
209+
? metadataEvent.event.payload
210+
: yield* threadManagement
211+
.getThreadShell(threadId)
212+
.pipe(Effect.mapError((error) => failure("orchestration_error", errorMessage(error))));
213+
if (thread === null) {
203214
return yield* failure(
204215
"orchestration_error",
205216
`Thread ${threadId} metadata update completed without a resultant state.`,
@@ -209,7 +220,7 @@ const make = Effect.gen(function* () {
209220
action: input.action,
210221
commandId: updateCommandId,
211222
sequence: dispatched.sequence,
212-
thread: metadataEvent.event.payload,
223+
thread,
213224
});
214225
});
215226

‎apps/server/src/orchestration-v2/FoundationPersistence.test.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -941,8 +941,10 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
941941
`;
942942

943943
const summary = yield* maintenance.compactEventStore;
944-
// Superseded state spans several discovery pages. Both turn-item updates stay.
945-
assert.isAtLeast(summary.deletedEventCount, 507);
944+
// Superseded state spans several discovery pages. Both turn-item updates
945+
// stay, and so do the first message and node events (replay order). The
946+
// rename stays too: visits are read state and only supersede each other.
947+
assert.isAtLeast(summary.deletedEventCount, 504);
946948
assert.isAtLeast(summary.deletedReceiptCount, 1);
947949

948950
const remaining = yield* sql<{ readonly event_id: string }>`
@@ -956,6 +958,9 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
956958
remaining.map((row) => row.event_id),
957959
[
958960
"event:foundation-compact:create",
961+
"event:foundation-compact:meta",
962+
"event:foundation-compact:message-1",
963+
"event:foundation-compact:node-1",
959964
"event:foundation-compact:item-1",
960965
"event:foundation-compact:visit-2",
961966
"event:foundation-compact:message-2",

‎apps/server/src/orchestration-v2/ProjectionMaintenance.ts‎

Lines changed: 52 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -205,11 +205,11 @@ export const layer: Layer.Layer<
205205
// Thread-state events whose payload is the complete thread: only the
206206
// newest per thread can influence a replay or an afterSequence catch-up.
207207
// thread.created stays out — verify derives the expected thread set from
208-
// it, and it anchors replay ordering.
208+
// it, and it anchors replay ordering. thread.deleted stays out too: clients
209+
// and waitForThread switch on the event type, not just the payload.
209210
const SUPERSEDABLE_THREAD_EVENT_TYPES = [
210211
"thread.archived",
211212
"thread.unarchived",
212-
"thread.deleted",
213213
"thread.settled",
214214
"thread.unsettled",
215215
"thread.snoozed",
@@ -225,16 +225,26 @@ export const layer: Layer.Layer<
225225
"thread.interaction-mode-updated",
226226
"thread.model-selection-updated",
227227
"thread.provider-switched",
228-
"thread.visited",
229-
"thread.marked-unread",
230228
];
229+
// Read-state events carry the whole thread too, but client reducers apply
230+
// them without bumping updatedAt. They supersede only each other: a newer
231+
// visit must not delete the rename that carries the activity timestamp,
232+
// while any newer activity-bearing event supersedes both groups.
233+
const READ_STATE_THREAD_EVENT_TYPES = ["thread.visited", "thread.marked-unread"];
231234

232235
const supersedableThreadEventTypes = new Set(SUPERSEDABLE_THREAD_EVENT_TYPES);
236+
const readStateThreadEventTypes = new Set(READ_STATE_THREAD_EVENT_TYPES);
233237
// Keep subagent.updated and turn-item.updated rows: slimmed payloads reference their prompts.
234-
const COMPACTION_PAGE_SIZE = 500;
238+
// Discovery runs on the shared synchronous connection; json_extract over
239+
// large payloads made 500-row pages block the event loop for ~90ms.
240+
const COMPACTION_PAGE_SIZE = 50;
235241

236242
/**
237-
* Scan newest first to retain the newest state for each entity.
243+
* Scan newest first to retain the newest state for each entity. Keyed
244+
* entities (messages, nodes, PR links) also keep their oldest event: client
245+
* reducers append an entity where they first see it, so a replay from any
246+
* cursor must either carry that first event or find the entity already
247+
* present. Only the events in between are deleted.
238248
* Page every event, including non-candidates: filtering before LIMIT could
239249
* still scan the entire history when superseded events are sparse.
240250
* turn-item.updated stays intact because replay assigns positions on first write.
@@ -251,7 +261,10 @@ export const layer: Layer.Layer<
251261
let throughSequence = bounds[0]?.event_sequence ?? 0;
252262
let throughReceiptRowId = bounds[0]?.receipt_row_id ?? 0;
253263
const retainedThreadIds = new Set<string>();
254-
const retainedEntityKeys = new Set<string>();
264+
const retainedReadStateThreadIds = new Set<string>();
265+
// Per key: the lowest sequence seen so far after the newest. It is
266+
// deleted once an even older event turns up and kept otherwise.
267+
const oldestCandidateByEntityKey = new Map<string, number | null>();
255268
let deletedEventCount = 0;
256269
let deletedReceiptCount = 0;
257270

@@ -275,6 +288,11 @@ export const layer: Layer.Layer<
275288
WHEN event.application_event_version = 2
276289
AND event.event_type IN ('message.updated', 'node.updated')
277290
THEN json_extract(event.payload_json, '$.id')
291+
WHEN event.application_event_version = 2
292+
AND event.event_type = 'thread.pull-request-link-updated'
293+
THEN json_array(json_extract(event.payload_json, '$.host'),
294+
json_extract(event.payload_json, '$.repository'),
295+
json_extract(event.payload_json, '$.number'))
278296
ELSE NULL
279297
END AS entity_id,
280298
CASE
@@ -302,10 +320,34 @@ export const layer: Layer.Layer<
302320
) {
303321
if (retainedThreadIds.has(row.stream_id)) obsolete.push(row.sequence);
304322
else retainedThreadIds.add(row.stream_id);
305-
} else if (row.event_type === "message.updated" || row.event_type === "node.updated") {
323+
} else if (
324+
row.aggregate_kind === "thread" &&
325+
readStateThreadEventTypes.has(row.event_type)
326+
) {
327+
if (
328+
retainedThreadIds.has(row.stream_id) ||
329+
retainedReadStateThreadIds.has(row.stream_id)
330+
) {
331+
obsolete.push(row.sequence);
332+
} else retainedReadStateThreadIds.add(row.stream_id);
333+
} else if (
334+
row.event_type === "thread.pull-request-link-updated" &&
335+
retainedThreadIds.has(row.stream_id)
336+
) {
337+
// A newer whole-thread payload already carries this link's state.
338+
obsolete.push(row.sequence);
339+
} else if (
340+
row.event_type === "thread.pull-request-link-updated" ||
341+
row.event_type === "message.updated" ||
342+
row.event_type === "node.updated"
343+
) {
306344
const key = encodeEntityKey([row.event_type, row.stream_id, row.entity_id]);
307-
if (retainedEntityKeys.has(key)) obsolete.push(row.sequence);
308-
else retainedEntityKeys.add(key);
345+
const candidate = oldestCandidateByEntityKey.get(key);
346+
if (candidate === undefined) oldestCandidateByEntityKey.set(key, null);
347+
else {
348+
if (candidate !== null) obsolete.push(candidate);
349+
oldestCandidateByEntityKey.set(key, row.sequence);
350+
}
309351
}
310352
}
311353
}

‎apps/server/src/orchestration-v2/runtimeLayer.test.ts‎

Lines changed: 78 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4950,11 +4950,88 @@ it.layer(Layer.fresh(layerSharedApplicationDataPlaneTest))("compact PR events",
49504950
(yield* orchestrator.getThreadShell(threadId))?.pullRequests,
49514951
after?.pullRequests,
49524952
);
4953+
// Later syncs of the same link supersede the ones between the first
4954+
// (kept for replay order) and the newest.
4955+
for (const [commandId, title] of [
4956+
["compact-pr-sync-again", "Still open PR"],
4957+
["compact-pr-sync-merged", "Merged PR"],
4958+
] as const) {
4959+
yield* orchestrator.dispatch({
4960+
type: "thread.pull-request-link.sync",
4961+
commandId: CommandId.make(commandId),
4962+
threadId,
4963+
...key,
4964+
number: 2,
4965+
snapshot: {
4966+
state: title === "Merged PR" ? "merged" : "open",
4967+
title,
4968+
headBranch: "feature/pr",
4969+
baseBranch: "main",
4970+
isDraft: false,
4971+
updatedAt: null,
4972+
syncedAt: "2026-10-13T13:00:00.000Z",
4973+
},
4974+
stack: null,
4975+
});
4976+
}
4977+
const latest = yield* orchestrator.getThreadShell(threadId);
49534978
yield* maintenance.compactEventStore;
4979+
const linkEvents = yield* sql<{ command_id: string }>`
4980+
SELECT command_id FROM orchestration_events
4981+
WHERE stream_id = ${threadId} AND event_type = 'thread.pull-request-link-updated'
4982+
ORDER BY sequence
4983+
`;
4984+
assert.deepEqual(
4985+
linkEvents.map((row) => row.command_id),
4986+
["compact-pr-sync", "compact-pr-sync-merged"],
4987+
);
49544988
assert.isTrue((yield* maintenance.rebuild).valid);
49554989
assert.deepEqual(
49564990
(yield* orchestrator.getThreadShell(threadId))?.pullRequests,
4957-
after?.pullRequests,
4991+
latest?.pullRequests,
4992+
);
4993+
4994+
// A later visit is read state only: clients keep updatedAt from the
4995+
// rename, so compaction must keep the rename. Two visits supersede each
4996+
// other, and a later rename supersedes a visit.
4997+
const threadEvents = () => sql<{ event_type: string }>`
4998+
SELECT event_type FROM orchestration_events
4999+
WHERE stream_id = ${threadId}
5000+
AND event_type IN ('thread.metadata-updated', 'thread.visited')
5001+
ORDER BY sequence
5002+
`;
5003+
yield* orchestrator.dispatch({
5004+
type: "thread.metadata.update",
5005+
commandId: CommandId.make("compact-pr-rename"),
5006+
threadId,
5007+
title: "Renamed before visit",
5008+
});
5009+
for (const [commandId, visitedAt] of [
5010+
["compact-pr-visit", "2026-10-13T14:00:00.000Z"],
5011+
["compact-pr-visit-again", "2026-10-13T15:00:00.000Z"],
5012+
] as const) {
5013+
yield* orchestrator.dispatch({
5014+
type: "thread.visit",
5015+
commandId: CommandId.make(commandId),
5016+
threadId,
5017+
visitedAt,
5018+
});
5019+
}
5020+
yield* maintenance.compactEventStore;
5021+
assert.deepEqual(
5022+
(yield* threadEvents()).map((row) => row.event_type),
5023+
["thread.metadata-updated", "thread.visited"],
5024+
);
5025+
yield* orchestrator.dispatch({
5026+
type: "thread.metadata.update",
5027+
commandId: CommandId.make("compact-pr-rename-again"),
5028+
threadId,
5029+
title: "Renamed after visit",
5030+
});
5031+
yield* maintenance.compactEventStore;
5032+
assert.deepEqual(
5033+
(yield* threadEvents()).map((row) => row.event_type),
5034+
["thread.metadata-updated"],
49585035
);
49595036
}),
49605037
);
Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
import { assert, it } from "@effect/vitest";
2+
import * as Effect from "effect/Effect";
3+
import * as Layer from "effect/Layer";
4+
import * as SqlClient from "effect/sql/SqlClient";
5+
6+
import * as EventStore from "../orchestration-v2/EventStore.ts";
7+
import * as ProjectionMaintenance from "../orchestration-v2/ProjectionMaintenance.ts";
8+
import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts";
9+
import * as DatabaseStorage from "./DatabaseStorage.ts";
10+
import * as Sqlite from "./Sqlite.ts";
11+
12+
const freePages = Effect.map(
13+
Effect.flatMap(
14+
SqlClient.SqlClient,
15+
(sql) => sql<{ readonly freelist_count: number }>`PRAGMA freelist_count`,
16+
),
17+
(rows) => rows[0]?.freelist_count ?? 0,
18+
);
19+
20+
it.effect("a new database returns freed pages online without losing data", () =>
21+
Effect.gen(function* () {
22+
const sql = yield* SqlClient.SqlClient;
23+
const storage = yield* DatabaseStorage.DatabaseStorage;
24+
assert.deepEqual(yield* sql`PRAGMA auto_vacuum`, [{ auto_vacuum: 2 }]);
25+
yield* sql`CREATE TABLE retained (value TEXT)`;
26+
yield* sql`INSERT INTO retained VALUES ('keep')`;
27+
yield* sql`CREATE TABLE discarded (value BLOB)`;
28+
yield* sql`INSERT INTO discarded VALUES (zeroblob(1048576))`;
29+
yield* sql`DROP TABLE discarded`;
30+
assert.isAbove(yield* freePages, 0);
31+
yield* storage.reclaim;
32+
assert.equal(yield* freePages, 0);
33+
assert.deepEqual(yield* sql`SELECT value FROM retained`, [{ value: "keep" }]);
34+
}).pipe(
35+
Effect.provide(
36+
DatabaseStorage.layer.pipe(
37+
Layer.provide(ProjectionMaintenance.layer),
38+
Layer.provide(Layer.merge(EventStore.layer, ProjectionStore.layer)),
39+
Layer.provideMerge(Sqlite.layerMemory),
40+
),
41+
),
42+
),
43+
);
Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
import * as Context from "effect/Context";
2+
import * as Effect from "effect/Effect";
3+
import * as Layer from "effect/Layer";
4+
import * as Schedule from "effect/Schedule";
5+
import * as SqlClient from "effect/sql/SqlClient";
6+
7+
import * as ProjectionMaintenance from "../orchestration-v2/ProjectionMaintenance.ts";
8+
import { forkParked } from "../serverActivation.ts";
9+
10+
/** Pages returned per reclaim pass; one minute of churn, not a backlog. */
11+
const RECLAIM_PAGES_PER_PASS = 1024;
12+
13+
export class DatabaseStorage extends Context.Service<
14+
DatabaseStorage,
15+
{
16+
/** Return free pages to the filesystem on databases with incremental auto_vacuum. */
17+
readonly reclaim: Effect.Effect<void>;
18+
}
19+
>()("t3/persistence/DatabaseStorage") {}
20+
21+
/**
22+
* Deletes superseded history every few hours and hands freed pages back to
23+
* the filesystem every minute. Deleted events are always superseded by a
24+
* retained event with a higher sequence, so a client replaying from an older
25+
* cursor still reaches the same state; no replay boundary is published.
26+
*/
27+
export const layer = Layer.effect(
28+
DatabaseStorage,
29+
Effect.gen(function* () {
30+
const sql = yield* SqlClient.SqlClient;
31+
const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2;
32+
33+
const reclaim = Effect.gen(function* () {
34+
const mode = yield* sql<{ readonly auto_vacuum: number }>`PRAGMA auto_vacuum`;
35+
if (mode[0]?.auto_vacuum !== 2) return;
36+
// The driver steps a statement without result columns once, and each
37+
// step of incremental_vacuum frees one page. Yield between pages so
38+
// client requests can run.
39+
for (let page = 0; page < RECLAIM_PAGES_PER_PASS; page++) {
40+
const free = yield* sql<{ readonly freelist_count: number }>`PRAGMA freelist_count`;
41+
if ((free[0]?.freelist_count ?? 0) === 0) break;
42+
yield* sql`PRAGMA incremental_vacuum(1)`;
43+
yield* Effect.yieldNow;
44+
}
45+
}).pipe(
46+
Effect.catch((cause) => Effect.logWarning("Database space reclamation failed", { cause })),
47+
);
48+
49+
const compact = maintenance.compactEventStore.pipe(
50+
Effect.tap((result) =>
51+
result.deletedEventCount > 0 || result.deletedReceiptCount > 0
52+
? Effect.logInfo("Compacted orchestration history", result)
53+
: Effect.void,
54+
),
55+
Effect.catch((cause) =>
56+
Effect.logWarning("Orchestration history compaction failed", { cause }),
57+
),
58+
);
59+
60+
yield* forkParked(
61+
Effect.sleep("1 minute").pipe(
62+
Effect.andThen(compact.pipe(Effect.repeat(Schedule.spaced("6 hours")))),
63+
),
64+
);
65+
yield* forkParked(
66+
Effect.sleep("1 minute").pipe(
67+
Effect.andThen(reclaim.pipe(Effect.repeat(Schedule.spaced("1 minute")))),
68+
),
69+
);
70+
71+
return DatabaseStorage.of({ reclaim });
72+
}),
73+
);

‎apps/server/src/persistence/Sqlite.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,12 @@ const layerSetup = Layer.effectDiscard(
1818
// CLI and server write from separate processes; wait rather than fail with SQLITE_BUSY.
1919
yield* sql`PRAGMA busy_timeout = 5000;`;
2020
yield* sql`PRAGMA foreign_keys = ON;`;
21+
// Only an empty file can switch modes here; an existing database needs the
22+
// offline VACUUM that `t3 doctor` runs, which rebuilds it with pointer maps.
23+
const tables = yield* sql<{
24+
readonly count: number;
25+
}>`SELECT COUNT(*) AS count FROM sqlite_schema WHERE type = 'table'`;
26+
if (tables[0]?.count === 0) yield* sql`PRAGMA auto_vacuum = INCREMENTAL;`;
2127
yield* sql`PRAGMA journal_mode = WAL;`;
2228
// FULL fsyncs the WAL on every commit, on the main thread. Every provider
2329
// event is its own commit, so a slow or busy disk stalls the event loop

0 commit comments

Comments
 (0)