Skip to content

Commit 3ca0235

Browse files
perf(client): avoid full ordering scans for streamed replacements
1 parent 22dc1da commit 3ca0235

4 files changed

Lines changed: 315 additions & 32 deletions

File tree

‎packages/client-runtime/src/state/orchestrationV2Projection.test.ts‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -555,3 +555,88 @@ it("does not scan every row against every run for a streaming item update", () =
555555
expect(next?.visibleTurnItems[0]).toBe(projection.visibleTurnItems[0]);
556556
expect(runReads).toBeLessThanOrEqual(100);
557557
});
558+
559+
it.each([100, 1000, 10000])(
560+
"does not scan ordering or positions for a stable replacement among %i rows",
561+
(size) => {
562+
let ordinalReads = 0;
563+
let positionReads = 0;
564+
const items = Array.from({ length: size }, (_, index) => ({
565+
...commandItem(`item-${index}`, "before", index),
566+
get ordinal() {
567+
ordinalReads++;
568+
return index;
569+
},
570+
}));
571+
const rows = items.map((item, index) => ({
572+
item,
573+
get position() {
574+
positionReads++;
575+
return index;
576+
},
577+
visibility: "local" as const,
578+
sourceThreadId: threadId,
579+
sourceItemId: item.id,
580+
}));
581+
const payload = commandItem(`item-${size - 1}`, "after", size - 1);
582+
const next = applyOrchestrationV2ProjectionEvent(
583+
{ ...emptyProjection, runs: [run], turnItems: items, visibleTurnItems: rows },
584+
{
585+
id: "stable-replacement",
586+
type: "turn-item.updated",
587+
threadId,
588+
occurredAt: now,
589+
payload,
590+
} as OrchestrationV2DomainEvent,
591+
);
592+
593+
expect(ordinalReads).toBeLessThanOrEqual(2);
594+
expect(positionReads).toBeLessThanOrEqual(2);
595+
expect(next?.visibleTurnItems.at(-1)).toMatchObject({ item: payload, position: size - 1 });
596+
expect(next?.visibleTurnItems[0]).toBe(rows[0]);
597+
expect(items.at(-1)).toMatchObject({ output: "before" });
598+
expect(rows.at(-1)?.item).toBe(items.at(-1));
599+
},
600+
);
601+
602+
it("rechecks visibility when an existing item changes runs without changing ordinal", () => {
603+
const first = commandItem("item-first", "first", 1);
604+
const second = commandItem("item-second", "second", 2);
605+
const rolledBackRun = { ...run, id: RunId.make("rolled-back"), status: "rolled_back" as const };
606+
const projection = {
607+
...emptyProjection,
608+
runs: [run, rolledBackRun],
609+
turnItems: [first, second],
610+
visibleTurnItems: [first, second].map((item, position) => ({
611+
item,
612+
position,
613+
visibility: "local" as const,
614+
sourceThreadId: threadId,
615+
sourceItemId: item.id,
616+
})),
617+
};
618+
const update = (payload: OrchestrationV2TurnItem) =>
619+
({
620+
id: "visibility-update",
621+
type: "turn-item.updated",
622+
threadId,
623+
occurredAt: now,
624+
payload,
625+
}) as OrchestrationV2DomainEvent;
626+
const hidden = applyOrchestrationV2ProjectionEvent(
627+
projection,
628+
update({
629+
...first,
630+
runId: rolledBackRun.id,
631+
}),
632+
);
633+
expect(hidden?.visibleTurnItems.map((row) => [row.item.id, row.position])).toEqual([
634+
[second.id, 0],
635+
]);
636+
const restored = applyOrchestrationV2ProjectionEvent(hidden, update(first));
637+
expect(restored?.visibleTurnItems.map((row) => [row.item.id, row.position])).toEqual([
638+
[first.id, 0],
639+
[second.id, 1],
640+
]);
641+
expect(projection.visibleTurnItems[0]?.item).toBe(first);
642+
});

‎packages/client-runtime/src/state/orchestrationV2Projection.ts‎

Lines changed: 42 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,36 @@ export type ApplyOrchestrationV2ProjectionEventOptions = {
1515
readonly latestLocalTurnOrdinal?: number | null;
1616
};
1717

18+
// Both sides are weak keys: comparison hints must not retain old streaming
19+
// arrays (and their text) for the lifetime of the thread.
20+
const arrayReplacements = new WeakMap<
21+
ReadonlyArray<unknown>,
22+
WeakMap<ReadonlyArray<unknown>, number>
23+
>();
24+
25+
export function getProjectionArrayReplacementIndex(
26+
previous: ReadonlyArray<unknown>,
27+
next: ReadonlyArray<unknown>,
28+
): number | undefined {
29+
return arrayReplacements.get(previous)?.get(next);
30+
}
31+
32+
function replaceProjectionArrayItem<T>(
33+
items: ReadonlyArray<T>,
34+
index: number,
35+
item: T,
36+
): ReadonlyArray<T> {
37+
const next = items.slice();
38+
next[index] = item;
39+
let replacements = arrayReplacements.get(items);
40+
if (replacements === undefined) {
41+
replacements = new WeakMap();
42+
arrayReplacements.set(items, replacements);
43+
}
44+
replacements.set(next, index);
45+
return next;
46+
}
47+
1848
function upsertEntity<T extends { readonly id: unknown }>(
1949
items: ReadonlyArray<T>,
2050
item: T,
@@ -173,6 +203,14 @@ function upsertVisibleTurnItem(
173203
) {
174204
return rows;
175205
}
206+
if (
207+
previous !== undefined &&
208+
previous.visibility === next.visibility &&
209+
previous.sourceThreadId === next.sourceThreadId &&
210+
previous.item.ordinal === item.ordinal
211+
) {
212+
return replaceProjectionArrayItem(rows, index, { ...next, position: previous.position });
213+
}
176214
if (
177215
index === -1 &&
178216
partialTimeline &&
@@ -326,9 +364,10 @@ export function applyOrchestrationV2ProjectionEvent(
326364
) {
327365
return projection;
328366
}
329-
const turnItems = [...projection.turnItems];
330-
if (itemIndex === -1) turnItems.push(event.payload);
331-
else turnItems[itemIndex] = event.payload;
367+
const turnItems =
368+
itemIndex === -1
369+
? [...projection.turnItems, event.payload]
370+
: replaceProjectionArrayItem(projection.turnItems, itemIndex, event.payload);
332371
const next = { ...base, turnItems };
333372
// Only interrupt requests can change another item's visibility. Streaming
334373
// text/tool updates must not recheck every row against every run.

‎packages/client-runtime/src/state/threadDetail.test.ts‎

Lines changed: 155 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import {
22
EnvironmentId,
3+
EventId,
34
MessageId,
45
RunId,
56
TurnItemId,
@@ -12,7 +13,9 @@ import * as Option from "effect/Option";
1213
import { AsyncResult, Atom, AtomRegistry } from "effect/reactivity";
1314

1415
import { v2Now, v2Projection, v2ThreadId } from "./orchestrationV2TestFixtures.ts";
16+
import { applyOrchestrationV2ProjectionEvent } from "./orchestrationV2Projection.ts";
1517
import { createEnvironmentThreadDetailAtoms, sameThreadStructure } from "./threadDetail.ts";
18+
import { mergeOlderHistoryIntoProjection } from "./threadHistoryMerge.ts";
1619
import { EMPTY_ENVIRONMENT_THREAD_STATE } from "./threadState.ts";
1720

1821
const later = DateTime.makeUnsafe("2026-06-20T00:00:01.000Z");
@@ -132,10 +135,14 @@ describe("quietThreadAtom", () => {
132135
registry.mount(quietAtom);
133136
return {
134137
read: () => registry.get(quietAtom),
135-
push: (projection: OrchestrationV2ThreadProjection) =>
138+
[Symbol.dispose]: () => registry.dispose(),
139+
push: (projection: OrchestrationV2ThreadProjection | null) =>
136140
registry.set(
137141
stateAtom,
138-
AsyncResult.success({ ...EMPTY_ENVIRONMENT_THREAD_STATE, data: Option.some(projection) }),
142+
AsyncResult.success({
143+
...EMPTY_ENVIRONMENT_THREAD_STATE,
144+
data: Option.fromNullishOr(projection),
145+
}),
139146
),
140147
};
141148
}
@@ -158,6 +165,22 @@ describe("quietThreadAtom", () => {
158165
expect(next?.projection.turnItems[0]).toMatchObject({ status: "completed", streaming: false });
159166
});
160167

168+
it("discards structure held before the thread resets", () => {
169+
const initial = withItem(assistant);
170+
using thread = setup(initial);
171+
const first = thread.read();
172+
thread.push(replaceItem(initial, { ...assistant, text: "Hello" }));
173+
expect(thread.read()).toBe(first);
174+
thread.push(null);
175+
expect(thread.read()).toBeNull();
176+
const reset = withItem({ ...assistant, title: "Restored", text: "New snapshot" });
177+
thread.push(reset);
178+
expect(thread.read()).not.toBe(first);
179+
expect(thread.read()?.projection.turnItems[0]).toMatchObject({ title: "Restored" });
180+
thread.push(replaceItem(reset, { ...assistant, title: "Restored", text: "More text" }));
181+
expect(thread.read()?.projection.turnItems[0]).toMatchObject({ title: "Restored" });
182+
});
183+
161184
it("leaves out the fields that streaming moves", () => {
162185
const thread = setup(withItem(assistant));
163186
const projection = thread.read()?.projection;
@@ -173,3 +196,133 @@ describe("quietThreadAtom", () => {
173196
});
174197
});
175198
});
199+
200+
it.each([100, 1000, 10000])(
201+
"only compares replaced entries in a reducer delta among %i items",
202+
(size) => {
203+
let slotReads = 0;
204+
const countReads = <T>(items: ReadonlyArray<T>): ReadonlyArray<T> =>
205+
new Proxy(items, {
206+
get(target, key, receiver) {
207+
if (typeof key === "string" && /^\d+$/.test(key)) slotReads++;
208+
return Reflect.get(target, key, receiver);
209+
},
210+
});
211+
const items = Array.from({ length: size }, (_, index) => ({
212+
...assistant,
213+
id: TurnItemId.make(`item-${index}`),
214+
ordinal: index,
215+
}));
216+
const before = {
217+
...v2Projection,
218+
turnItems: countReads(items),
219+
visibleTurnItems: countReads(
220+
items.map((item, position) => ({
221+
item,
222+
position,
223+
visibility: "local" as const,
224+
sourceThreadId: v2ThreadId,
225+
sourceItemId: item.id,
226+
})),
227+
),
228+
};
229+
const next = applyOrchestrationV2ProjectionEvent(before, {
230+
id: EventId.make("text-delta"),
231+
type: "turn-item.updated",
232+
threadId: v2ThreadId,
233+
occurredAt: later,
234+
payload: { ...items.at(-1)!, text: "Hello", updatedAt: later },
235+
});
236+
slotReads = 0;
237+
238+
expect(sameThreadStructure(before, next!)).toBe(true);
239+
expect(slotReads).toBeLessThanOrEqual(2);
240+
},
241+
);
242+
243+
function replaceItem(projection: OrchestrationV2ThreadProjection, item: OrchestrationV2TurnItem) {
244+
return applyOrchestrationV2ProjectionEvent(projection, {
245+
id: EventId.make(`event-${item.id}`),
246+
type: "turn-item.updated",
247+
threadId: v2ThreadId,
248+
occurredAt: item.updatedAt,
249+
payload: item,
250+
})!;
251+
}
252+
253+
describe("reducer structure comparisons", () => {
254+
it.each(["assistant_message", "reasoning"] as const)(
255+
"keeps %s text quiet but observes completion and run changes",
256+
(type) => {
257+
const item = { ...assistant, type };
258+
const initial = withItem(item);
259+
const streamed = replaceItem(initial, { ...item, text: "Hello", updatedAt: later });
260+
const completed = replaceItem(streamed, {
261+
...item,
262+
status: "completed",
263+
streaming: false,
264+
text: "Hello there",
265+
});
266+
const movedRun = replaceItem(completed, { ...item, runId: RunId.make("another-run") });
267+
268+
expect(sameThreadStructure(initial, streamed)).toBe(true);
269+
expect(sameThreadStructure(streamed, completed)).toBe(false);
270+
expect(sameThreadStructure(completed, movedRun)).toBe(false);
271+
expect(streamed.visibleTurnItems[0]?.item).toMatchObject({ text: "Hello" });
272+
expect(initial.turnItems[0]).toBe(item);
273+
expect(initial.visibleTurnItems[0]?.item).toMatchObject({ text: "Hel" });
274+
},
275+
);
276+
277+
it("compares all changes when the reader skips events in a batch", () => {
278+
const initial = mergeOlderHistoryIntoProjection(withItem(assistant), [
279+
{
280+
...withItem(assistant).visibleTurnItems[0]!,
281+
sourceItemId: TurnItemId.make("older"),
282+
item: { ...assistant, id: TurnItemId.make("older"), ordinal: 0 },
283+
},
284+
]);
285+
const older = initial.turnItems.find((item) => item.id === "older")!;
286+
const completed = replaceItem(initial, { ...older, status: "completed" });
287+
const streamed = replaceItem(completed, { ...assistant, text: "Hello", updatedAt: later });
288+
289+
expect(sameThreadStructure(completed, streamed)).toBe(true);
290+
expect(sameThreadStructure(initial, streamed)).toBe(false);
291+
expect(
292+
sameThreadStructure(initial, {
293+
...streamed,
294+
turnItems: initial.turnItems,
295+
visibleTurnItems: streamed.visibleTurnItems.slice(),
296+
}),
297+
).toBe(false);
298+
});
299+
300+
it("compares snapshot resets and history merges without a replacement hint", () => {
301+
const initial = withItem(assistant);
302+
const streamed = replaceItem(initial, { ...assistant, text: "Hello", updatedAt: later });
303+
const snapshot = {
304+
...streamed,
305+
turnItems: streamed.turnItems.map((item) => ({ ...item, title: "Restored" })),
306+
visibleTurnItems: streamed.visibleTurnItems.map((row) => ({ ...row })),
307+
};
308+
const older = { ...assistant, id: TurnItemId.make("older"), ordinal: 0 };
309+
const history = mergeOlderHistoryIntoProjection(streamed, [
310+
{
311+
position: 0,
312+
visibility: "local",
313+
sourceThreadId: v2ThreadId,
314+
sourceItemId: older.id,
315+
item: older,
316+
},
317+
]);
318+
const next = replaceItem(history, { ...assistant, text: "Hello there", updatedAt: later });
319+
320+
expect(sameThreadStructure(initial, snapshot)).toBe(false);
321+
expect(sameThreadStructure(streamed, history)).toBe(false);
322+
expect(sameThreadStructure(history, next)).toBe(true);
323+
expect(next.visibleTurnItems.map((row) => [row.item.id, row.position])).toEqual([
324+
[older.id, 0],
325+
[assistant.id, 1],
326+
]);
327+
});
328+
});

0 commit comments

Comments
 (0)