Skip to content

Commit 31f6861

Browse files
perf(server): coalesce superseded streaming text updates
1 parent 22dc1da commit 31f6861

2 files changed

Lines changed: 364 additions & 22 deletions

File tree

‎apps/server/src/orchestration-v2/ThreadLiveEventCoalescer.test.ts‎

Lines changed: 332 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import {
55
RunId,
66
TurnItemId,
77
type OrchestrationV2ThreadStreamItem,
8+
type OrchestrationV2TurnItem,
89
} from "@t3tools/contracts";
910
import { it } from "@effect/vitest";
1011
import * as DateTime from "effect/DateTime";
@@ -13,7 +14,7 @@ import * as Stream from "effect/Stream";
1314
import * as TestClock from "effect/testing/TestClock";
1415
import { describe, expect } from "vite-plus/test";
1516
import {
16-
coalesceLiveToolUpdatedEvents,
17+
coalesceLiveUpdatedEvents,
1718
coalesceThreadLiveStream,
1819
makeThreadLiveEventCoalescer,
1920
} from "./ThreadLiveEventCoalescer.ts";
@@ -94,6 +95,66 @@ function makeMessage(sequence: number, text = "Still working"): ThreadEvent {
9495
};
9596
}
9697

98+
function makeStreamingMessage(
99+
sequence: number,
100+
options: {
101+
readonly id?: string;
102+
readonly text?: string;
103+
readonly streaming?: boolean;
104+
readonly role?: "assistant" | "user";
105+
} = {},
106+
): ThreadEvent {
107+
const message = makeMessage(sequence, options.text ?? `text-${sequence}`);
108+
if (message.event.type !== "message.updated") throw new Error("Expected message");
109+
return {
110+
...message,
111+
event: {
112+
...message.event,
113+
payload: {
114+
...message.event.payload,
115+
id: MessageId.make(options.id ?? "message"),
116+
streaming: options.streaming ?? true,
117+
role: options.role ?? "assistant",
118+
},
119+
},
120+
};
121+
}
122+
123+
function makeTextItem(
124+
sequence: number,
125+
options: {
126+
readonly id?: string;
127+
readonly text?: string;
128+
readonly type?: "assistant_message" | "reasoning";
129+
readonly status?: OrchestrationV2TurnItem["status"];
130+
readonly streaming?: boolean;
131+
} = {},
132+
): ThreadEvent {
133+
const tool = makeToolActivity(sequence);
134+
if (tool.event.type !== "turn-item.updated" || tool.event.payload.type !== "command_execution")
135+
throw new Error("Expected command item");
136+
const { input: _input, output: _output, ...base } = tool.event.payload;
137+
return {
138+
...tool,
139+
event: {
140+
...tool.event,
141+
payload: {
142+
...base,
143+
id: TurnItemId.make(options.id ?? "item"),
144+
type: options.type ?? "assistant_message",
145+
messageId: MessageId.make("message"),
146+
text: options.text ?? `text-${sequence}`,
147+
status: options.status ?? "running",
148+
streaming: options.streaming ?? true,
149+
},
150+
},
151+
};
152+
}
153+
154+
function sequences(events: ReadonlyArray<{ readonly kind: string; readonly sequence?: number }>) {
155+
return events.map((event) => (event.kind === "event" ? event.sequence : event.kind));
156+
}
157+
97158
describe("ThreadLiveEventCoalescer", () => {
98159
it("coalesces only calls with a stable toolCallId", () => {
99160
const events = [
@@ -102,7 +163,7 @@ describe("ThreadLiveEventCoalescer", () => {
102163
makeToolActivity(3, { toolCallId: "call-a" }),
103164
];
104165

105-
expect(coalesceLiveToolUpdatedEvents(events).map((event) => event.sequence)).toEqual([2, 3]);
166+
expect(coalesceLiveUpdatedEvents(events).map((event) => event.sequence)).toEqual([2, 3]);
106167
});
107168

108169
it("does not coalesce stable tool calls across turns", () => {
@@ -111,7 +172,7 @@ describe("ThreadLiveEventCoalescer", () => {
111172
makeToolActivity(2, { turnId: RunId.make("turn-new") }),
112173
];
113174

114-
expect(coalesceLiveToolUpdatedEvents(events).map((event) => event.sequence)).toEqual([1, 2]);
175+
expect(coalesceLiveUpdatedEvents(events).map((event) => event.sequence)).toEqual([1, 2]);
115176
});
116177

117178
it("flushes a stable update run before a completion boundary", () => {
@@ -122,9 +183,276 @@ describe("ThreadLiveEventCoalescer", () => {
122183
makeToolActivity(4),
123184
];
124185

125-
expect(coalesceLiveToolUpdatedEvents(events).map((event) => event.sequence)).toEqual([2, 3, 4]);
186+
expect(coalesceLiveUpdatedEvents(events).map((event) => event.sequence)).toEqual([2, 3, 4]);
126187
});
127188

189+
it("coalesces paired assistant messages/items and interleaved reasoning/tools by identity", () => {
190+
const events = [
191+
makeStreamingMessage(1, { id: "shared" }),
192+
makeTextItem(2, { id: "shared" }),
193+
makeTextItem(3, { id: "reasoning", type: "reasoning" }),
194+
makeToolActivity(4, { toolCallId: "tool" }),
195+
makeTextItem(5, { id: "unique" }),
196+
makeStreamingMessage(6, { id: "shared" }),
197+
makeTextItem(7, { id: "shared" }),
198+
makeTextItem(8, { id: "reasoning", type: "reasoning" }),
199+
makeToolActivity(9, { toolCallId: "tool" }),
200+
];
201+
const result = coalesceLiveUpdatedEvents(events);
202+
expect(sequences(result)).toEqual([5, 6, 7, 8, 9]);
203+
expect(result).toEqual(events.slice(4));
204+
});
205+
206+
it("preserves message, text and tool completions and non-streaming boundaries", () => {
207+
for (const boundary of [
208+
makeStreamingMessage(3, { streaming: false }),
209+
makeStreamingMessage(3, { role: "user" }),
210+
makeTextItem(3, { streaming: false }),
211+
...(["completed", "failed", "cancelled"] as const).map((status) =>
212+
makeTextItem(3, { status }),
213+
),
214+
makeToolActivity(3, { kind: "tool.completed" }),
215+
makeMessage(3),
216+
]) {
217+
const events = [
218+
makeStreamingMessage(1),
219+
makeStreamingMessage(2),
220+
boundary,
221+
makeStreamingMessage(4),
222+
makeStreamingMessage(5),
223+
];
224+
expect(coalesceLiveUpdatedEvents(events)).toEqual([events[1], boundary, events[4]]);
225+
}
226+
});
227+
228+
it("does not mix message/item identities, item types, threads or runs", () => {
229+
const first = makeStreamingMessage(1, { id: "same" });
230+
const events = [
231+
first,
232+
makeTextItem(2, { id: "same" }),
233+
makeTextItem(3, { id: "same", type: "reasoning" }),
234+
{ ...first, sequence: 4, event: { ...first.event, runId: RunId.make("other-run") } },
235+
{ ...first, sequence: 5, event: { ...first.event, threadId: ThreadId.make("other-thread") } },
236+
];
237+
expect(coalesceLiveUpdatedEvents(events)).toEqual(events);
238+
});
239+
240+
it("preserves assistant-message identity changes as insertion-order boundaries", () => {
241+
const events = [
242+
makeStreamingMessage(1, { id: "message-a" }),
243+
makeTextItem(2, { id: "item-a" }),
244+
makeStreamingMessage(3, { id: "message-b" }),
245+
makeTextItem(4, { id: "item-b" }),
246+
makeStreamingMessage(5, { id: "message-a" }),
247+
makeTextItem(6, { id: "item-a" }),
248+
];
249+
expect(coalesceLiveUpdatedEvents(events)).toEqual(events);
250+
});
251+
252+
it.effect("flushes text within 50ms of the first update under sustained offers", () =>
253+
Effect.scoped(
254+
Effect.gen(function* () {
255+
const coalescer = yield* makeThreadLiveEventCoalescer();
256+
yield* coalescer.offerAll([makeStreamingMessage(1), makeTextItem(2)]);
257+
yield* TestClock.adjust("20 millis");
258+
yield* coalescer.offerAll([makeStreamingMessage(3), makeTextItem(4)]);
259+
yield* TestClock.adjust("20 millis");
260+
yield* coalescer.offerAll([makeStreamingMessage(5), makeTextItem(6)]);
261+
yield* TestClock.adjust("10 millis");
262+
const pull = yield* Stream.toPull(coalescer.stream);
263+
expect(sequences(yield* pull)).toEqual([5, 6]);
264+
yield* coalescer.offerAll([makeStreamingMessage(7), makeTextItem(8)]);
265+
yield* TestClock.adjust("50 millis");
266+
expect(sequences(yield* pull)).toEqual([7, 8]);
267+
yield* coalescer.end;
268+
yield* pull.pipe(Effect.result);
269+
expect(yield* coalescer.usage).toEqual({ retainedItems: 0, retainedSerializedBytes: 0 });
270+
}),
271+
),
272+
);
273+
274+
it.effect("flushes before synchronization without coalescing across it", () =>
275+
Effect.scoped(
276+
Effect.gen(function* () {
277+
const coalescer = yield* makeThreadLiveEventCoalescer();
278+
const events = [
279+
makeStreamingMessage(1),
280+
makeTextItem(2),
281+
makeStreamingMessage(3),
282+
makeTextItem(4),
283+
];
284+
yield* coalescer.offerAll([
285+
...events,
286+
{ kind: "synchronized" },
287+
makeStreamingMessage(5),
288+
makeStreamingMessage(6),
289+
]);
290+
yield* coalescer.end;
291+
const result = yield* Stream.runCollect(coalescer.stream);
292+
expect(sequences(result)).toEqual([3, 4, "synchronized", 6]);
293+
expect(yield* coalescer.usage).toEqual({ retainedItems: 0, retainedSerializedBytes: 0 });
294+
}),
295+
),
296+
);
297+
298+
it.effect("retains the exact serialized budget for surviving and unacknowledged text", () =>
299+
Effect.scoped(
300+
Effect.gen(function* () {
301+
const coalescer = yield* makeThreadLiveEventCoalescer();
302+
const events = [1, 2, 3].map((sequence) =>
303+
makeStreamingMessage(sequence, { text: "é".repeat(sequence * 100) }),
304+
);
305+
yield* coalescer.offerAll(events);
306+
expect(yield* coalescer.usage).toEqual({
307+
retainedItems: 3,
308+
retainedSerializedBytes: events.reduce(
309+
(sum, event) => sum + Buffer.byteLength(encodeEvent(event)),
310+
0,
311+
),
312+
});
313+
yield* TestClock.adjust("50 millis");
314+
const expectedUsage = {
315+
retainedItems: 1,
316+
retainedSerializedBytes: Buffer.byteLength(encodeEvent(events[2])),
317+
};
318+
expect(yield* coalescer.usage).toEqual(expectedUsage);
319+
const pull = yield* Stream.toPull(coalescer.stream);
320+
expect(sequences(yield* pull)).toEqual([3]);
321+
expect(yield* coalescer.usage).toEqual(expectedUsage);
322+
yield* coalescer.end;
323+
yield* pull.pipe(Effect.result);
324+
expect(yield* coalescer.usage).toEqual({ retainedItems: 0, retainedSerializedBytes: 0 });
325+
}),
326+
),
327+
);
328+
329+
it.effect("does not drop unique text items at the pending batch cap", () =>
330+
Effect.scoped(
331+
Effect.gen(function* () {
332+
const coalescer = yield* makeThreadLiveEventCoalescer();
333+
const events = Array.from({ length: 513 }, (_, index) =>
334+
makeTextItem(index + 1, { id: `item-${index}` }),
335+
);
336+
yield* coalescer.offerAll(events);
337+
yield* coalescer.end;
338+
expect(yield* Stream.runCollect(coalescer.stream)).toEqual(events);
339+
}),
340+
),
341+
);
342+
343+
it("can resume after every surviving cursor without losing the final state", () => {
344+
const events = [
345+
makeStreamingMessage(1),
346+
makeTextItem(2),
347+
makeTextItem(3, { id: "unique", type: "reasoning" }),
348+
makeStreamingMessage(4),
349+
makeTextItem(5),
350+
makeStreamingMessage(6, { streaming: false, text: "final answer" }),
351+
makeTextItem(7, { streaming: false, status: "completed", text: "final answer" }),
352+
];
353+
const latestStates = (input: ReadonlyArray<ThreadEvent>) =>
354+
Object.fromEntries(
355+
input.map(({ event }) => [
356+
`${event.type}:${"id" in event.payload ? event.payload.id : ""}`,
357+
event.payload,
358+
]),
359+
);
360+
const survivors = coalesceLiveUpdatedEvents(events);
361+
expect(sequences(survivors)).toEqual([3, 4, 5, 6, 7]);
362+
for (let index = 0; index < survivors.length; index += 1) {
363+
const received = survivors.slice(0, index + 1);
364+
const cursor = received.at(-1)!.sequence;
365+
const resumed = coalesceLiveUpdatedEvents(events.filter((event) => event.sequence > cursor));
366+
expect(latestStates([...received, ...resumed])).toEqual(latestStates(events));
367+
}
368+
});
369+
370+
it.effect("reduces growing paired/interleaved states queued across fixed windows", () =>
371+
Effect.scoped(
372+
Effect.gen(function* () {
373+
const coalescer = yield* makeThreadLiveEventCoalescer();
374+
const input: Array<ThreadEvent> = [];
375+
const expected: Array<ThreadEvent> = [];
376+
for (let window = 0; window < 20; window += 1) {
377+
for (let update = 0; update < 5; update += 1) {
378+
const text = "x".repeat((window * 5 + update + 1) * 100);
379+
const pair = [
380+
makeStreamingMessage(input.length + 1, { text }),
381+
makeTextItem(input.length + 2, { text }),
382+
makeTextItem(input.length + 3, { type: "reasoning", id: "reasoning", text }),
383+
];
384+
input.push(...pair);
385+
yield* coalescer.offerAll(pair);
386+
if (update === 4) expected.push(...pair);
387+
yield* TestClock.adjust("10 millis");
388+
}
389+
}
390+
const completions = [
391+
makeStreamingMessage(input.length + 1, { streaming: false, text: "final answer" }),
392+
makeTextItem(input.length + 2, {
393+
status: "completed",
394+
streaming: false,
395+
text: "final answer",
396+
}),
397+
];
398+
input.push(...completions);
399+
expected.push(...completions);
400+
yield* coalescer.offerAll(completions);
401+
yield* coalescer.end;
402+
const serializedBytes = (events: ReadonlyArray<ThreadEvent>) =>
403+
events.reduce((sum, event) => sum + Buffer.byteLength(encodeEvent(event)), 0);
404+
expect(yield* coalescer.usage).toEqual({
405+
retainedItems: 62,
406+
retainedSerializedBytes: serializedBytes(expected),
407+
});
408+
const result = yield* Stream.runCollect(coalescer.stream);
409+
expect(result).toEqual(expected);
410+
expect(input).toHaveLength(302);
411+
expect(serializedBytes(expected)).toBeLessThan(serializedBytes(input) * 0.22);
412+
expect(yield* coalescer.usage).toEqual({ retainedItems: 0, retainedSerializedBytes: 0 });
413+
}),
414+
),
415+
);
416+
417+
it.effect("preserves every pair when producer updates are already 50ms apart", () =>
418+
Effect.scoped(
419+
Effect.gen(function* () {
420+
const coalescer = yield* makeThreadLiveEventCoalescer();
421+
const input: Array<ThreadEvent> = [];
422+
for (let index = 0; index < 10; index += 1) {
423+
const text = "x".repeat((index + 1) * 100);
424+
const pair = [
425+
makeStreamingMessage(index * 2 + 1, { text }),
426+
makeTextItem(index * 2 + 2, { text }),
427+
];
428+
input.push(...pair);
429+
yield* coalescer.offerAll(pair);
430+
yield* TestClock.adjust("50 millis");
431+
}
432+
yield* coalescer.end;
433+
expect(yield* Stream.runCollect(coalescer.stream)).toEqual(input);
434+
}),
435+
),
436+
);
437+
438+
it.effect("fails rather than bypassing the byte budget for pending text", () =>
439+
Effect.scoped(
440+
Effect.gen(function* () {
441+
const first = makeStreamingMessage(1, { text: "x".repeat(1_000) });
442+
const coalescer = yield* makeThreadLiveEventCoalescer({
443+
maxSerializedBytes: Buffer.byteLength(encodeEvent(first)),
444+
});
445+
yield* coalescer.offer(first);
446+
const result = yield* coalescer
447+
.offer(makeStreamingMessage(2, { text: "x".repeat(2_000) }))
448+
.pipe(Effect.result);
449+
expect(result._tag).toBe("Failure");
450+
yield* coalescer.closed;
451+
expect(yield* coalescer.usage).toEqual({ retainedItems: 0, retainedSerializedBytes: 0 });
452+
}),
453+
),
454+
);
455+
128456
it.effect("keeps the flush timer alive when an offer's shorter scope closes", () =>
129457
Effect.scoped(
130458
Effect.gen(function* () {

0 commit comments

Comments
 (0)