Skip to content

Commit a56ae77

Browse files
fix(server): keep tool payloads out of completion queues
Filter terminal-worker subscriptions before live buffering and historical payload decoding. Preserve matching run updates without retaining unrelated tool output while queue promotion waits on providers or thread locks.
1 parent ebfd663 commit a56ae77

6 files changed

Lines changed: 169 additions & 33 deletions

File tree

‎apps/server/src/orchestration-v2/EventSink.ts‎

Lines changed: 50 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,8 @@ export interface EventSinkV2Shape {
159159
readonly stream: (input?: {
160160
readonly threadId?: ThreadId;
161161
readonly afterSequence?: number;
162+
/** Filter before queuing live events so a busy worker retains only the events it handles. */
163+
readonly eventType?: OrchestrationV2DomainEvent["type"];
162164
/** Bound RPC subscribers; internal workers must not drop their subscription under load. */
163165
readonly bounded?: boolean;
164166
}) => Stream.Stream<OrchestrationV2StoredEvent, EventSinkV2Error>;
@@ -196,6 +198,20 @@ const baseLayer: Layer.Layer<
196198
const projectionStore = yield* ProjectionStoreV2;
197199
const turnItemPositions = yield* TurnItemPositionStoreV2;
198200
const liveEvents = yield* PubSub.unbounded<OrchestrationV2StoredEvent>();
201+
const liveEventsByType = new Map<
202+
OrchestrationV2DomainEvent["type"],
203+
PubSub.PubSub<OrchestrationV2StoredEvent>
204+
>();
205+
const publishLiveEvents = (events: ReadonlyArray<OrchestrationV2StoredEvent>) =>
206+
Effect.gen(function* () {
207+
yield* PubSub.publishAll(liveEvents, events);
208+
for (const [type, pubsub] of liveEventsByType) {
209+
yield* PubSub.publishAll(
210+
pubsub,
211+
events.filter((stored) => stored.event.type === type),
212+
);
213+
}
214+
});
199215

200216
// A user can answer after terminal normalization reads the pending request.
201217
// Recheck inside the write transaction so stale cleanup cannot erase answers.
@@ -324,7 +340,7 @@ const baseLayer: Layer.Layer<
324340
yield* effectOutbox.notifyAvailable(input.effects.length);
325341
}
326342
yield* eventStore.publishCommitted(storedEvents);
327-
yield* PubSub.publishAll(liveEvents, storedEvents);
343+
yield* publishLiveEvents(storedEvents);
328344
return storedEvents;
329345
});
330346

@@ -378,7 +394,7 @@ const baseLayer: Layer.Layer<
378394
);
379395
if (result.committed) {
380396
yield* eventStore.publishCommitted(result.storedEvents);
381-
yield* PubSub.publishAll(liveEvents, result.storedEvents);
397+
yield* publishLiveEvents(result.storedEvents);
382398
}
383399
return result;
384400
},
@@ -439,7 +455,7 @@ const baseLayer: Layer.Layer<
439455
);
440456
if (result.committed) {
441457
yield* eventStore.publishCommitted(result.storedEvents);
442-
yield* PubSub.publishAll(liveEvents, result.storedEvents);
458+
yield* publishLiveEvents(result.storedEvents);
443459
}
444460
return result;
445461
});
@@ -517,7 +533,7 @@ const baseLayer: Layer.Layer<
517533
}
518534
if (result.committed) {
519535
yield* eventStore.publishCommitted(result.storedEvents);
520-
yield* PubSub.publishAll(liveEvents, result.storedEvents);
536+
yield* publishLiveEvents(result.storedEvents);
521537
}
522538
return {
523539
receipt: result.receipt,
@@ -556,6 +572,7 @@ const baseLayer: Layer.Layer<
556572
readonly afterSequence: number;
557573
readonly throughSequence: number;
558574
readonly threadId?: ThreadId;
575+
readonly eventType?: OrchestrationV2DomainEvent["type"];
559576
}): Stream.Stream<OrchestrationV2StoredEvent, unknown> => {
560577
const pageSize = 256;
561578
const loop = (afterSequence: number): Stream.Stream<OrchestrationV2StoredEvent, unknown> =>
@@ -565,6 +582,7 @@ const baseLayer: Layer.Layer<
565582
afterSequence,
566583
throughSequence: input.throughSequence,
567584
...(input.threadId === undefined ? {} : { threadId: input.threadId }),
585+
...(input.eventType === undefined ? {} : { eventType: input.eventType }),
568586
limit: pageSize,
569587
})
570588
.pipe(
@@ -587,31 +605,44 @@ const baseLayer: Layer.Layer<
587605

588606
const stream = (input?: Parameters<EventSinkV2Shape["stream"]>[0]) => {
589607
const afterSequence = input?.afterSequence ?? 0;
590-
const matchesThread = (stored: OrchestrationV2StoredEvent) =>
591-
input?.threadId === undefined || stored.event.threadId === input.threadId;
608+
const matches = (stored: OrchestrationV2StoredEvent) =>
609+
(input?.threadId === undefined || stored.event.threadId === input.threadId) &&
610+
(input?.eventType === undefined || stored.event.type === input.eventType);
592611
const replay = (throughSequence: number) =>
593612
catchUp({
594613
afterSequence,
595614
throughSequence,
596615
...(input?.threadId === undefined ? {} : { threadId: input.threadId }),
597-
});
598-
if (input?.bounded === true) {
599-
return replayAndBufferProjectedLiveEvents({
600-
subscribe: PubSub.subscribe(liveEvents),
601-
latestSequence: eventStore.latestSequence(),
602-
afterSequence,
603-
filter: matchesThread,
604-
replay,
605-
project: (stored) => ({ ...stored, event: projectDomainEventForWire(stored.event) }),
606-
});
607-
}
616+
...(input?.eventType === undefined ? {} : { eventType: input.eventType }),
617+
}).pipe(Stream.filter(matches));
608618
return Stream.unwrap(
609619
Effect.gen(function* () {
610-
const subscription = yield* PubSub.subscribe(liveEvents);
620+
let pubsub = liveEvents;
621+
if (input?.eventType !== undefined) {
622+
const existing = liveEventsByType.get(input.eventType);
623+
if (existing !== undefined) {
624+
pubsub = existing;
625+
} else {
626+
const created = yield* PubSub.unbounded<OrchestrationV2StoredEvent>();
627+
pubsub = liveEventsByType.get(input.eventType) ?? created;
628+
liveEventsByType.set(input.eventType, pubsub);
629+
}
630+
}
631+
if (input?.bounded === true) {
632+
return replayAndBufferProjectedLiveEvents({
633+
subscribe: PubSub.subscribe(pubsub),
634+
latestSequence: eventStore.latestSequence(),
635+
afterSequence,
636+
filter: matches,
637+
replay,
638+
project: (stored) => ({ ...stored, event: projectDomainEventForWire(stored.event) }),
639+
});
640+
}
641+
const subscription = yield* PubSub.subscribe(pubsub);
611642
const highWater = yield* eventStore.latestSequence();
612643
const live = Stream.fromSubscription(subscription).pipe(
613644
Stream.filter((stored) => stored.sequence > Math.max(highWater, afterSequence)),
614-
Stream.filter(matchesThread),
645+
Stream.filter(matches),
615646
);
616647
return Stream.concat(replay(highWater), live);
617648
}),

‎apps/server/src/orchestration-v2/EventStore.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ export interface EventStoreV2Shape {
5656
readonly afterSequence?: number;
5757
readonly throughSequence?: number;
5858
readonly threadId?: ThreadId;
59+
readonly eventType?: OrchestrationV2DomainEvent["type"];
5960
readonly limit?: number;
6061
}) => Stream.Stream<OrchestrationV2StoredEvent, EventStoreV2Error>;
6162
readonly readByCommandId: (input: {
@@ -86,6 +87,7 @@ const baseLayer: Layer.Layer<EventStoreV2, never, OrchestrationEventStore> = Lay
8687
? {}
8788
: { throughSequence: input.throughSequence }),
8889
...(input?.threadId === undefined ? {} : { threadId: input.threadId }),
90+
...(input?.eventType === undefined ? {} : { eventType: input.eventType }),
8991
...(input?.limit === undefined ? {} : { limit: input.limit }),
9092
})
9193
.pipe(

‎apps/server/src/orchestration-v2/FoundationPersistence.test.ts‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -550,6 +550,101 @@ it.layer(TestLayer)("orchestration V2 foundation persistence", (it) => {
550550
),
551551
);
552552

553+
it.effect("filters worker replay and live queues without losing matching events", () =>
554+
Effect.scoped(
555+
Effect.gen(function* () {
556+
const sink = yield* EventSinkV2;
557+
const sql = yield* SqlClient.SqlClient;
558+
const now = yield* DateTime.now;
559+
const thread = makeThread(ThreadId.make("thread:filtered-worker"), now);
560+
const other = makeThread(ThreadId.make("thread:filtered-worker-other"), now);
561+
const run: OrchestrationV2Run = {
562+
id: RunId.make("run:filtered-worker"),
563+
threadId: thread.id,
564+
ordinal: 1,
565+
providerInstanceId,
566+
modelSelection,
567+
providerThreadId: null,
568+
userMessageId: MessageId.make("message:filtered-worker"),
569+
rootNodeId: null,
570+
activeAttemptId: null,
571+
status: "completed",
572+
queuePosition: null,
573+
requestedAt: now,
574+
startedAt: now,
575+
completedAt: now,
576+
checkpointId: null,
577+
contextHandoffId: null,
578+
};
579+
const runEvent = (id: string, payload: OrchestrationV2Run): OrchestrationV2DomainEvent => ({
580+
id: EventId.make(id),
581+
type: "run.updated",
582+
threadId: payload.threadId,
583+
runId: payload.id,
584+
occurredAt: now,
585+
payload,
586+
});
587+
const history = yield* sink.write({
588+
events: [
589+
threadCreatedEvent({ id: "event:filtered-worker:created", thread, now }),
590+
threadCreatedEvent({ id: "event:filtered-worker:other", thread: other, now }),
591+
runEvent("event:filtered-worker:history", run),
592+
runEvent("event:filtered-worker:other-history", {
593+
...run,
594+
id: RunId.make("run:filtered-worker-other"),
595+
threadId: other.id,
596+
}),
597+
],
598+
});
599+
// Unrelated payloads must be skipped in SQL, before decoding or
600+
// allocating their bodies, even when a retained row is unreadable.
601+
const original = yield* sql<{ readonly payload_json: string }>`
602+
SELECT payload_json FROM orchestration_events
603+
WHERE sequence = ${history[0]!.sequence}
604+
`;
605+
yield* Effect.acquireRelease(
606+
sql`
607+
UPDATE orchestration_events SET payload_json = 'unreadable unrelated payload'
608+
WHERE sequence = ${history[0]!.sequence}
609+
`,
610+
() =>
611+
sql`
612+
UPDATE orchestration_events SET payload_json = ${original[0]!.payload_json}
613+
WHERE sequence = ${history[0]!.sequence}
614+
`.pipe(Effect.orDie),
615+
);
616+
const pull = yield* Stream.toPull(
617+
sink.stream({ threadId: thread.id, eventType: "run.updated" }),
618+
);
619+
assert.deepEqual(
620+
(yield* pull).map((stored) => stored.sequence),
621+
[history[2]!.sequence],
622+
);
623+
// The worker is occupied with the previous batch while the thread
624+
// publishes output. Its live queue must receive just run updates.
625+
yield* sink.write({
626+
events: Array.from({ length: LIVE_STREAM_MAX_ITEMS + 1 }, (_, index) => ({
627+
id: EventId.make(`event:filtered-worker:output:${index}`),
628+
type: "thread.metadata-updated" as const,
629+
threadId: thread.id,
630+
occurredAt: now,
631+
payload: { ...thread, title: `Output ${index}` },
632+
})),
633+
});
634+
const live = yield* sink.write({
635+
events: [
636+
runEvent("event:filtered-worker:live:1", { ...run, status: "interrupted" }),
637+
runEvent("event:filtered-worker:live:2", { ...run, status: "failed" }),
638+
],
639+
});
640+
assert.deepEqual(
641+
(yield* pull).map((stored) => stored.sequence),
642+
live.map((stored) => stored.sequence),
643+
);
644+
}),
645+
),
646+
);
647+
553648
it.effect("paginates catch-up beyond the event-store read limit", () =>
554649
Effect.gen(function* () {
555650
const eventSink = yield* EventSinkV2;

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -8549,20 +8549,24 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
85498549
// below. Replaying the full event table on every server start delays live
85508550
// queue promotion in proportion to the lifetime size of the database.
85518551
const terminalEventsAfterSequence = yield* eventSink.latestSequence().pipe(Effect.orDie);
8552-
yield* eventSink.stream({ afterSequence: terminalEventsAfterSequence }).pipe(
8553-
Stream.filter(
8554-
(stored) =>
8555-
stored.event.type === "run.updated" &&
8556-
!String(stored.commandId).startsWith("command:runtime-reconcile:") &&
8557-
(stored.event.payload.status === "completed" ||
8558-
stored.event.payload.status === "interrupted" ||
8559-
stored.event.payload.status === "failed" ||
8560-
stored.event.payload.status === "cancelled" ||
8561-
stored.event.payload.status === "rolled_back"),
8562-
),
8563-
Stream.runForEach(handleTerminalRun),
8564-
Effect.forkDetach,
8565-
);
8552+
// Queue promotion can wait on a provider or a thread lock. Subscribe to run
8553+
// updates before buffering so that wait never retains unrelated tool bodies.
8554+
yield* eventSink
8555+
.stream({ afterSequence: terminalEventsAfterSequence, eventType: "run.updated" })
8556+
.pipe(
8557+
Stream.filter(
8558+
(stored) =>
8559+
stored.event.type === "run.updated" &&
8560+
!String(stored.commandId).startsWith("command:runtime-reconcile:") &&
8561+
(stored.event.payload.status === "completed" ||
8562+
stored.event.payload.status === "interrupted" ||
8563+
stored.event.payload.status === "failed" ||
8564+
stored.event.payload.status === "cancelled" ||
8565+
stored.event.payload.status === "rolled_back"),
8566+
),
8567+
Stream.runForEach(handleTerminalRun),
8568+
Effect.forkDetach,
8569+
);
85668570

85678571
// The high-water subscription deliberately skips history, so recover the
85688572
// two terminal side effects from current projections instead: one queued

‎apps/server/src/persistence/Layers/OrchestrationEventStore.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -372,6 +372,7 @@ const makeEventStore = Effect.gen(function* () {
372372
readonly throughSequence?: number;
373373
readonly threadId?: ThreadId;
374374
readonly commandId?: CommandId;
375+
readonly eventType?: OrchestrationV2DomainEvent["type"];
375376
readonly onlyAgentEvents?: boolean;
376377
readonly limit: number;
377378
}) =>
@@ -408,6 +409,7 @@ const makeEventStore = Effect.gen(function* () {
408409
AND ${sql.and([
409410
...(input.threadId === undefined ? [] : [sql`stream_id = ${input.threadId}`]),
410411
...(input.commandId === undefined ? [] : [sql`command_id = ${input.commandId}`]),
412+
...(input.eventType === undefined ? [] : [sql`event_type = ${input.eventType}`]),
411413
])}
412414
ORDER BY sequence ASC
413415
LIMIT ${input.limit}
@@ -492,6 +494,7 @@ const makeEventStore = Effect.gen(function* () {
492494
: { throughSequence: input.throughSequence }),
493495
...(input?.threadId === undefined ? {} : { threadId: input.threadId }),
494496
...(input?.commandId === undefined ? {} : { commandId: input.commandId }),
497+
...(input?.eventType === undefined ? {} : { eventType: input.eventType }),
495498
onlyAgentEvents: true,
496499
limit: pageLimit,
497500
}).pipe(

‎apps/server/src/persistence/Services/OrchestrationEventStore.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ export interface OrchestrationEventStoreShape {
7878
readonly throughSequence?: number;
7979
readonly threadId?: ThreadId;
8080
readonly commandId?: CommandId;
81+
readonly eventType?: OrchestrationV2DomainEvent["type"];
8182
readonly limit?: number;
8283
}) => Stream.Stream<OrchestrationV2StoredEvent, OrchestrationEventStoreError>;
8384

0 commit comments

Comments
 (0)