Skip to content

Commit 42289be

Browse files
maria-rcksQard
andcommitted
fix(server): preserve reconnect replay progress
Co-authored-by: Stephen Belanger <[email protected]>
1 parent 54c5f25 commit 42289be

6 files changed

Lines changed: 290 additions & 49 deletions

File tree

‎apps/server/src/persistence/OrchestrationEventStore.sequence.test.ts‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,72 @@ const seedEvents = Effect.fn("test.seedSequenceEvents")(function* (
6868
`;
6969
});
7070

71+
it.effect("checks creation ranges without restoring or decoding payloads", () =>
72+
Effect.gen(function* () {
73+
const store = yield* OrchestrationEventStore.OrchestrationEventStore;
74+
const threadId = ThreadId.make("creation-existence");
75+
const inserted = yield* seedEvents([
76+
{ ...eventRow(1, "thread", 2, threadId), event_type: "thread.created", payload_json: "{" },
77+
eventRow(2, "thread", 2, threadId),
78+
{ ...eventRow(3, "thread", 2, "other"), event_type: "thread.created", payload_json: "{" },
79+
{ ...eventRow(4, "thread", 1, threadId), event_type: "thread.created", payload_json: "{" },
80+
]);
81+
const range = {
82+
threadId,
83+
afterSequence: 0,
84+
throughSequence: inserted[3]!.sequence,
85+
eventType: "thread.created" as const,
86+
};
87+
assert.isTrue(yield* store.hasAgentEvent(range));
88+
assert.isFalse(yield* store.hasAgentEvent({ ...range, afterSequence: inserted[0]!.sequence }));
89+
assert.isFalse(yield* store.hasAgentEvent({ ...range, throughSequence: 0 }));
90+
assert.isTrue(yield* store.hasAgentEvent({ ...range, throughSequence: inserted[0]!.sequence }));
91+
}).pipe(Effect.provide(layerEventStore)),
92+
);
93+
94+
it.effect("fetches the next byte-bounded replay page after delivering the previous page", () =>
95+
Effect.gen(function* () {
96+
const store = yield* OrchestrationEventStore.OrchestrationEventStore;
97+
const sql = yield* SqlClient.SqlClient;
98+
const threadId = ThreadId.make("byte-paged-replay");
99+
const rows = [1, 2].map((ordinal) => ({
100+
...eventRow(ordinal, "thread", 2, threadId),
101+
payload_json: encodeJson({
102+
providerSessionId: `session:${ordinal}:${"x".repeat(180_000)}`,
103+
detachedAt: occurredAt,
104+
}),
105+
}));
106+
const inserted = yield* seedEvents(rows);
107+
const updatedPayload = {
108+
providerSessionId: ProviderSessionId.make("session:updated"),
109+
detachedAt: occurredAt,
110+
};
111+
const items = yield* store
112+
.readAgentEvents({
113+
threadId,
114+
throughSequence: inserted[1]!.sequence,
115+
maxPageBytes: 256 * 1024,
116+
})
117+
.pipe(
118+
Stream.tap((item) =>
119+
item.sequence === inserted[0]!.sequence
120+
? sql`UPDATE orchestration_events SET payload_json = ${encodeJson(updatedPayload)}
121+
WHERE sequence = ${inserted[1]!.sequence}`
122+
: Effect.void,
123+
),
124+
Stream.runCollect,
125+
);
126+
assert.deepEqual(
127+
items.map((item) => item.sequence),
128+
inserted.map((item) => item.sequence),
129+
);
130+
assert.deepEqual(items[1]!.event.payload, {
131+
...updatedPayload,
132+
detachedAt: DateTime.makeUnsafe(occurredAt),
133+
});
134+
}).pipe(Effect.provide(layerEventStore)),
135+
);
136+
71137
it.effect("keeps application and scoped agent high-water marks separate from legacy history", () =>
72138
Effect.gen(function* () {
73139
const store = yield* OrchestrationEventStore.OrchestrationEventStore;

‎apps/server/src/persistence/OrchestrationEventStore.test.ts‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ layer("OrchestrationEventStore", (it) => {
113113
const id = NodeId.make("node:subagent-prompt");
114114
const driver = ProviderDriverKind.make("claudeAgent");
115115
const providerInstanceId = ProviderInstanceId.make("claudeAgent");
116-
const prompt = "Audit the repository. ".repeat(200);
116+
const prompt = 'Audit "repo"\n漢字 '.repeat(9_000);
117117
const subagent = (index: number, progress: string, nextPrompt = prompt) =>
118118
({
119119
id: EventId.make(`event:subagent-prompt:subagent:${index}`),
@@ -185,14 +185,14 @@ layer("OrchestrationEventStore", (it) => {
185185
subagent(3, "Retrying", "Audit again."),
186186
item(3, "Retrying", "Audit again."),
187187
];
188-
yield* store.appendAgentEvents({ events });
188+
const appended = yield* store.appendAgentEvents({ events });
189189

190190
const stored = yield* sql<{ readonly payload_json: string }>`
191191
SELECT payload_json FROM orchestration_events
192192
WHERE stream_id = ${threadId} ORDER BY sequence ASC
193193
`;
194194
assert.deepEqual(
195-
stored.map((row) => row.payload_json.includes(prompt)),
195+
stored.map((row) => row.payload_json.includes(JSON.stringify(prompt).slice(1, -1))),
196196
[true, true, false, false, false, false, false, false],
197197
);
198198
const replayed = yield* store
@@ -214,6 +214,28 @@ layer("OrchestrationEventStore", (it) => {
214214
throughSequence: yield* store.latestApplicationSequence,
215215
});
216216
assert.isAbove(applicationStats.rawPayloadBytes, prompt.length * 6);
217+
const paged = yield* store
218+
.readAgentEvents({ threadId, afterSequence, maxPageBytes: 256 * 1024 })
219+
.pipe(Stream.runCollect);
220+
assert.deepEqual(
221+
paged.map((entry) => entry.event),
222+
events,
223+
);
224+
// The first two updates here reference prompts in earlier anchor rows.
225+
// Sizing must stop before touching a later page's malformed payload.
226+
yield* sql`UPDATE orchestration_events SET payload_json = '{' WHERE sequence = ${appended[4]!.sequence}`;
227+
const firstPage = yield* store
228+
.readAgentEvents({
229+
threadId,
230+
afterSequence: appended[1]!.sequence,
231+
throughSequence: appended[4]!.sequence,
232+
maxPageBytes: 256 * 1024,
233+
})
234+
.pipe(Stream.take(1), Stream.runCollect);
235+
assert.deepEqual(
236+
firstPage.map((entry) => entry.event),
237+
[events[2]!],
238+
);
217239
}),
218240
);
219241

‎apps/server/src/persistence/OrchestrationEventStore.ts‎

Lines changed: 69 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,18 @@ export class OrchestrationEventStore extends Context.Service<
6060
readonly eventType?: OrchestrationV2DomainEvent["type"];
6161
readonly limit?: number;
6262
readonly skipUnknownEventTypes?: boolean;
63+
/** Bound fetched JSON per page, always allowing one indivisible event. */
64+
readonly maxPageBytes?: number;
6365
}) => Stream.Stream<OrchestrationV2StoredEvent, OrchestrationEventStoreError>;
6466

67+
/** Check a scoped replay range without reading or restoring event payloads. */
68+
readonly hasAgentEvent: (input: {
69+
readonly threadId: ThreadId;
70+
readonly afterSequence: number;
71+
readonly throughSequence: number;
72+
readonly eventType: OrchestrationV2DomainEvent["type"];
73+
}) => Effect.Effect<boolean, OrchestrationEventStoreError>;
74+
6575
readonly getAgentReplayStats: (input: {
6676
readonly threadId: ThreadId;
6777
readonly afterSequence: number;
@@ -392,9 +402,9 @@ const makeEventStore = Effect.gen(function* () {
392402
readonly eventType?: OrchestrationV2DomainEvent["type"];
393403
readonly onlyAgentEvents?: boolean;
394404
readonly limit: number;
395-
}) =>
396-
sql<ApplicationEventRow>`
397-
SELECT
405+
readonly maxPageBytes?: number;
406+
}) => {
407+
const columns = sql`
398408
sequence,
399409
event_id,
400410
command_id,
@@ -407,6 +417,8 @@ const makeEventStore = Effect.gen(function* () {
407417
application_event_version,
408418
causation_event_id,
409419
correlation_id
420+
`;
421+
const from = sql`
410422
FROM orchestration_events
411423
${
412424
// Otherwise the sequence index wins and command lookups scan every V2 event.
@@ -431,9 +443,41 @@ const makeEventStore = Effect.gen(function* () {
431443
...(input.commandId === undefined ? [] : [sql`command_id = ${input.commandId}`]),
432444
...(input.eventType === undefined ? [] : [sql`event_type = ${input.eventType}`]),
433445
])}
434-
ORDER BY sequence ASC
435-
LIMIT ${input.limit}
446+
`;
447+
if (input.maxPageBytes === undefined) {
448+
return sql<ApplicationEventRow>`
449+
SELECT ${columns} ${from} ORDER BY sequence ASC LIMIT ${input.limit}
450+
`.pipe(NodeSqliteClient.readOnly);
451+
}
452+
// Materialize only candidate ids. Walk their restored sizes until the page
453+
// fills (at most one overflow row), rather than restoring all 500 bodies.
454+
// Prompt references must be sized after restoration, including JSON escaping.
455+
return sql<ApplicationEventRow>`
456+
WITH RECURSIVE candidates AS MATERIALIZED (
457+
SELECT sequence, ROW_NUMBER() OVER (ORDER BY sequence) AS ordinal
458+
${from} ORDER BY sequence ASC LIMIT ${input.limit}
459+
), replay_page(cursor, replay_bytes, row_count) AS (
460+
SELECT orchestration_events.sequence,
461+
octet_length(${restoredPayloadJson}) + octet_length(metadata_json), 1
462+
FROM orchestration_events
463+
JOIN candidates ON candidates.sequence = orchestration_events.sequence
464+
WHERE candidates.ordinal = 1
465+
UNION ALL
466+
SELECT orchestration_events.sequence,
467+
page.replay_bytes + octet_length(${restoredPayloadJson}) + octet_length(metadata_json),
468+
page.row_count + 1
469+
FROM replay_page AS page
470+
JOIN candidates ON candidates.ordinal = page.row_count + 1
471+
JOIN orchestration_events ON orchestration_events.sequence = candidates.sequence
472+
WHERE page.replay_bytes < ${input.maxPageBytes}
473+
)
474+
SELECT ${columns}
475+
FROM orchestration_events
476+
JOIN replay_page ON replay_page.cursor = orchestration_events.sequence
477+
WHERE replay_page.row_count = 1 OR replay_page.replay_bytes <= ${input.maxPageBytes}
478+
ORDER BY sequence
436479
`.pipe(NodeSqliteClient.readOnly);
480+
};
437481

438482
const appendAgentEvents: OrchestrationEventStore["Service"]["appendAgentEvents"] = (input) =>
439483
Effect.forEach(
@@ -556,6 +600,7 @@ const makeEventStore = Effect.gen(function* () {
556600
...(input?.eventType === undefined ? {} : { eventType: input.eventType }),
557601
onlyAgentEvents: true,
558602
limit: pageLimit,
603+
...(input?.maxPageBytes === undefined ? {} : { maxPageBytes: input.maxPageBytes }),
559604
}).pipe(
560605
Effect.mapError(toPersistenceSqlError("OrchestrationEventStore.readAgentEvents:query")),
561606
Effect.map((rows) => {
@@ -564,7 +609,7 @@ const makeEventStore = Effect.gen(function* () {
564609
return [
565610
rows,
566611
last === undefined ||
567-
rows.length < pageLimit ||
612+
(input?.maxPageBytes === undefined && rows.length < pageLimit) ||
568613
nextRemaining <= 0 ||
569614
(input?.throughSequence !== undefined && last.sequence >= input.throughSequence)
570615
? Option.none()
@@ -585,6 +630,23 @@ const makeEventStore = Effect.gen(function* () {
585630
);
586631
};
587632

633+
const hasAgentEvent: OrchestrationEventStore["Service"]["hasAgentEvent"] = (input) =>
634+
sql<{ readonly found: number }>`
635+
SELECT EXISTS (
636+
SELECT 1 FROM orchestration_events
637+
WHERE aggregate_kind = 'thread'
638+
AND stream_id = ${input.threadId}
639+
AND application_event_version = 2
640+
AND sequence > ${input.afterSequence}
641+
AND sequence <= ${input.throughSequence}
642+
AND event_type = ${input.eventType}
643+
) AS found
644+
`.pipe(
645+
NodeSqliteClient.readOnly,
646+
Effect.mapError(toPersistenceSqlError("OrchestrationEventStore.hasAgentEvent:query")),
647+
Effect.map((rows) => rows[0]?.found === 1),
648+
);
649+
588650
const getAgentReplayStats: OrchestrationEventStore["Service"]["getAgentReplayStats"] = (input) =>
589651
sql<{
590652
readonly eventCount: number;
@@ -743,6 +805,7 @@ const makeEventStore = Effect.gen(function* () {
743805
appendProjectEvent,
744806
appendAgentEvents,
745807
readAgentEvents,
808+
hasAgentEvent,
746809
getAgentReplayStats,
747810
getReplayStats,
748811
latestAgentSequence,

‎apps/server/src/ws.test.ts‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -306,6 +306,65 @@ it.live.each(["thread", "shell"] as const)(
306306
}).pipe(Effect.scoped, Effect.provide(ReplayTestLayer)),
307307
);
308308

309+
it.effect.each([
310+
{ count: 260, padding: 0, unknownPrefix: 0 },
311+
{ count: 3, padding: 600_000, unknownPrefix: 0 },
312+
{ count: 3, padding: 0, unknownPrefix: 140 },
313+
])(
314+
"resumes catch-up of $count events ($padding bytes, $unknownPrefix unknown rows) after interruption",
315+
({ count, padding, unknownPrefix }) =>
316+
Effect.gen(function* () {
317+
const sql = yield* SqlClient.SqlClient;
318+
const threadId = ThreadId.make("thread:incremental-replay");
319+
const now = "2026-10-28T00:00:00.000Z";
320+
const rows = Array.from({ length: count + unknownPrefix }, (_, version) => ({
321+
event_id: `incremental-replay:${version}`,
322+
aggregate_kind: "thread",
323+
stream_id: threadId,
324+
stream_version: version,
325+
event_type: version < unknownPrefix ? "future.event" : "provider-session.detached",
326+
occurred_at: now,
327+
actor_kind: "server",
328+
payload_json: JSON.stringify({
329+
providerSessionId: `session:${version}:${"x".repeat(padding)}`,
330+
detachedAt: now,
331+
}),
332+
metadata_json: "{}",
333+
application_event_version: 2,
334+
}));
335+
const inserted = yield* sql<{ sequence: number }>`
336+
INSERT INTO orchestration_events ${sql.insert(rows)} RETURNING sequence
337+
`;
338+
const subscribe = (afterSequence: number) =>
339+
subscribeOrchestrationV2Thread({
340+
threadId,
341+
afterSequence,
342+
requestCompletionMarker: true,
343+
}).pipe(
344+
Effect.provideService(ThreadManagement.ThreadManagementService, {
345+
ensureLegacyTranscript: () => Effect.void,
346+
getThreadSnapshot: () =>
347+
Effect.die("Ordinary catch-up must not restart with a snapshot"),
348+
streamStoredEventsFrom: () => Stream.empty,
349+
} as unknown as ThreadManagement.ThreadManagementService["Service"]),
350+
);
351+
const first = yield* subscribe(0).pipe(
352+
Effect.flatMap((stream) =>
353+
stream.pipe(Stream.take(Math.min(10, count - 1)), Stream.runCollect),
354+
),
355+
);
356+
const last = first.at(-1);
357+
assert.equal(first.length, Math.min(10, count - 1));
358+
assert.equal(last?.kind, "event");
359+
if (last?.kind !== "event") return;
360+
const resumed = yield* subscribe(last.sequence).pipe(Effect.flatMap(Stream.runCollect));
361+
assert.deepEqual(
362+
[...first, ...resumed].map((item) => (item.kind === "event" ? item.sequence : item.kind)),
363+
[...inserted.slice(unknownPrefix).map((row) => row.sequence), "synchronized"],
364+
);
365+
}).pipe(Effect.scoped, Effect.provide(ReplayTestLayer)),
366+
);
367+
309368
it.effect("coalesces streaming replay before synchronization and the live tail", () =>
310369
Effect.gen(function* () {
311370
const store = yield* OrchestrationEventStore.OrchestrationEventStore;

0 commit comments

Comments
 (0)