Skip to content

Commit 7de9535

Browse files
perf(client): avoid full ordering scans for streamed replacements (pingdotgg#24264)
1 parent 26c3625 commit 7de9535

4 files changed

Lines changed: 299 additions & 38 deletions

File tree

‎packages/client-runtime/src/state/orchestrationV2Projection.test.ts‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -555,3 +555,86 @@ it("does not scan every row against every run for a streaming item update", () =
555555
expect(next?.visibleTurnItems[0]).toBe(projection.visibleTurnItems[0]);
556556
expect(runReads).toBeLessThanOrEqual(100);
557557
});
558+
559+
it("does not scan ordering or positions for a streamed replacement", () => {
560+
const size = 100;
561+
let ordinalReads = 0;
562+
let positionReads = 0;
563+
const items = Array.from({ length: size }, (_, index) => ({
564+
...commandItem(`item-${index}`, "before", index),
565+
get ordinal() {
566+
ordinalReads++;
567+
return index;
568+
},
569+
}));
570+
const rows = items.map((item, index) => ({
571+
item,
572+
get position() {
573+
positionReads++;
574+
return index;
575+
},
576+
visibility: "local" as const,
577+
sourceThreadId: threadId,
578+
sourceItemId: item.id,
579+
}));
580+
const payload = commandItem(`item-${size - 1}`, "after", size - 1);
581+
const next = applyOrchestrationV2ProjectionEvent(
582+
{ ...emptyProjection, runs: [run], turnItems: items, visibleTurnItems: rows },
583+
{
584+
id: "stable-replacement",
585+
type: "turn-item.updated",
586+
threadId,
587+
occurredAt: now,
588+
payload,
589+
} as OrchestrationV2DomainEvent,
590+
);
591+
592+
expect(ordinalReads).toBeLessThanOrEqual(2);
593+
expect(positionReads).toBeLessThanOrEqual(2);
594+
expect(next?.visibleTurnItems.at(-1)).toMatchObject({ item: payload, position: size - 1 });
595+
expect(next?.visibleTurnItems[0]).toBe(rows[0]);
596+
expect(items.at(-1)).toMatchObject({ output: "before" });
597+
expect(rows.at(-1)?.item).toBe(items.at(-1));
598+
});
599+
600+
it("rechecks visibility when an existing item changes runs without changing ordinal", () => {
601+
const first = commandItem("item-first", "first", 1);
602+
const second = commandItem("item-second", "second", 2);
603+
const rolledBackRun = { ...run, id: RunId.make("rolled-back"), status: "rolled_back" as const };
604+
const projection = {
605+
...emptyProjection,
606+
runs: [run, rolledBackRun],
607+
turnItems: [first, second],
608+
visibleTurnItems: [first, second].map((item, position) => ({
609+
item,
610+
position,
611+
visibility: "local" as const,
612+
sourceThreadId: threadId,
613+
sourceItemId: item.id,
614+
})),
615+
};
616+
const update = (payload: OrchestrationV2TurnItem) =>
617+
({
618+
id: "visibility-update",
619+
type: "turn-item.updated",
620+
threadId,
621+
occurredAt: now,
622+
payload,
623+
}) as OrchestrationV2DomainEvent;
624+
const hidden = applyOrchestrationV2ProjectionEvent(
625+
projection,
626+
update({
627+
...first,
628+
runId: rolledBackRun.id,
629+
}),
630+
);
631+
expect(hidden?.visibleTurnItems.map((row) => [row.item.id, row.position])).toEqual([
632+
[second.id, 0],
633+
]);
634+
const restored = applyOrchestrationV2ProjectionEvent(hidden, update(first));
635+
expect(restored?.visibleTurnItems.map((row) => [row.item.id, row.position])).toEqual([
636+
[first.id, 0],
637+
[second.id, 1],
638+
]);
639+
expect(projection.visibleTurnItems[0]?.item).toBe(first);
640+
});

‎packages/client-runtime/src/state/orchestrationV2Projection.ts‎

Lines changed: 41 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,36 @@ export type ApplyOrchestrationV2ProjectionEventOptions = {
1515
readonly latestLocalTurnOrdinal?: number | null;
1616
};
1717

18+
// Only this exact pair differs at one index. Both keys are weak so hints
19+
// neither retain streaming history nor apply to replacement snapshots.
20+
const arrayReplacements = new WeakMap<
21+
ReadonlyArray<unknown>,
22+
WeakMap<ReadonlyArray<unknown>, number>
23+
>();
24+
25+
export function getProjectionArrayReplacementIndex(
26+
previous: ReadonlyArray<unknown>,
27+
next: ReadonlyArray<unknown>,
28+
): number | undefined {
29+
return arrayReplacements.get(previous)?.get(next);
30+
}
31+
32+
function replaceProjectionArrayItem<T>(
33+
items: ReadonlyArray<T>,
34+
index: number,
35+
item: T,
36+
): ReadonlyArray<T> {
37+
const next = items.slice();
38+
next[index] = item;
39+
let replacements = arrayReplacements.get(items);
40+
if (replacements === undefined) {
41+
replacements = new WeakMap();
42+
arrayReplacements.set(items, replacements);
43+
}
44+
replacements.set(next, index);
45+
return next;
46+
}
47+
1848
function upsertEntity<T extends { readonly id: unknown }>(
1949
items: ReadonlyArray<T>,
2050
item: T,
@@ -167,11 +197,12 @@ function upsertVisibleTurnItem(
167197
if (
168198
previous !== undefined &&
169199
previous.visibility === next.visibility &&
170-
previous.sourceThreadId === next.sourceThreadId &&
171-
previous.sourceItemId === next.sourceItemId &&
172-
previous.item === item
200+
previous.sourceThreadId === next.sourceThreadId
173201
) {
174-
return rows;
202+
if (previous.item === item) return rows;
203+
if (previous.item.ordinal === item.ordinal) {
204+
return replaceProjectionArrayItem(rows, index, { ...next, position: previous.position });
205+
}
175206
}
176207
if (
177208
index === -1 &&
@@ -181,8 +212,8 @@ function upsertVisibleTurnItem(
181212
return rows;
182213
}
183214
// The item goes before the first later local row, as if its old row were
184-
// removed first. One pass builds and renumbers the result: this runs for
185-
// every streamed chunk of a thread that can hold thousands of rows.
215+
// removed first. Inserts, reorders, and inherited-to-local changes need
216+
// this ordering pass; replacements above preserve every other row.
186217
let insertion = rows.length;
187218
for (let candidate = 0; candidate < rows.length; candidate += 1) {
188219
const row = rows[candidate]!;
@@ -326,9 +357,10 @@ export function applyOrchestrationV2ProjectionEvent(
326357
) {
327358
return projection;
328359
}
329-
const turnItems = [...projection.turnItems];
330-
if (itemIndex === -1) turnItems.push(event.payload);
331-
else turnItems[itemIndex] = event.payload;
360+
const turnItems =
361+
itemIndex === -1
362+
? [...projection.turnItems, event.payload]
363+
: replaceProjectionArrayItem(projection.turnItems, itemIndex, event.payload);
332364
const next = { ...base, turnItems };
333365
// Only interrupt requests can change another item's visibility. Streaming
334366
// text/tool updates must not recheck every row against every run.

‎packages/client-runtime/src/state/threadDetail.test.ts‎

Lines changed: 142 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import {
22
EnvironmentId,
3+
EventId,
34
MessageId,
45
RunId,
56
TurnItemId,
@@ -12,7 +13,9 @@ import * as Option from "effect/Option";
1213
import { AsyncResult, Atom, AtomRegistry } from "effect/reactivity";
1314

1415
import { v2Now, v2Projection, v2ThreadId } from "./orchestrationV2TestFixtures.ts";
16+
import { applyOrchestrationV2ProjectionEvent } from "./orchestrationV2Projection.ts";
1517
import { createEnvironmentThreadDetailAtoms, sameThreadStructure } from "./threadDetail.ts";
18+
import { mergeOlderHistoryIntoProjection } from "./threadHistoryMerge.ts";
1619
import { EMPTY_ENVIRONMENT_THREAD_STATE } from "./threadState.ts";
1720

1821
const later = DateTime.makeUnsafe("2026-06-20T00:00:01.000Z");
@@ -132,10 +135,14 @@ describe("quietThreadAtom", () => {
132135
registry.mount(quietAtom);
133136
return {
134137
read: () => registry.get(quietAtom),
135-
push: (projection: OrchestrationV2ThreadProjection) =>
138+
[Symbol.dispose]: () => registry.dispose(),
139+
push: (projection: OrchestrationV2ThreadProjection | null) =>
136140
registry.set(
137141
stateAtom,
138-
AsyncResult.success({ ...EMPTY_ENVIRONMENT_THREAD_STATE, data: Option.some(projection) }),
142+
AsyncResult.success({
143+
...EMPTY_ENVIRONMENT_THREAD_STATE,
144+
data: Option.fromNullishOr(projection),
145+
}),
139146
),
140147
};
141148
}
@@ -158,6 +165,22 @@ describe("quietThreadAtom", () => {
158165
expect(next?.projection.turnItems[0]).toMatchObject({ status: "completed", streaming: false });
159166
});
160167

168+
it("discards structure held before the thread resets", () => {
169+
const initial = withItem(assistant);
170+
using thread = setup(initial);
171+
const first = thread.read();
172+
thread.push(replaceItem(initial, { ...assistant, text: "Hello" }));
173+
expect(thread.read()).toBe(first);
174+
thread.push(null);
175+
expect(thread.read()).toBeNull();
176+
const reset = withItem({ ...assistant, title: "Restored", text: "New snapshot" });
177+
thread.push(reset);
178+
expect(thread.read()).not.toBe(first);
179+
expect(thread.read()?.projection.turnItems[0]).toMatchObject({ title: "Restored" });
180+
thread.push(replaceItem(reset, { ...assistant, title: "Restored", text: "More text" }));
181+
expect(thread.read()?.projection.turnItems[0]).toMatchObject({ title: "Restored" });
182+
});
183+
161184
it("leaves out the fields that streaming moves", () => {
162185
const thread = setup(withItem(assistant));
163186
const projection = thread.read()?.projection;
@@ -173,3 +196,120 @@ describe("quietThreadAtom", () => {
173196
});
174197
});
175198
});
199+
200+
it("only compares replaced entries in a streamed reducer delta", () => {
201+
const size = 100;
202+
let slotReads = 0;
203+
const countReads = <T>(items: ReadonlyArray<T>): ReadonlyArray<T> =>
204+
new Proxy(items, {
205+
get(target, key, receiver) {
206+
if (typeof key === "string" && /^\d+$/.test(key)) slotReads++;
207+
return Reflect.get(target, key, receiver);
208+
},
209+
});
210+
const items = Array.from({ length: size }, (_, index) => ({
211+
...assistant,
212+
id: TurnItemId.make(`item-${index}`),
213+
ordinal: index,
214+
}));
215+
const before = {
216+
...v2Projection,
217+
turnItems: countReads(items),
218+
visibleTurnItems: countReads(
219+
items.map((item, position) => ({
220+
item,
221+
position,
222+
visibility: "local" as const,
223+
sourceThreadId: v2ThreadId,
224+
sourceItemId: item.id,
225+
})),
226+
),
227+
};
228+
const next = applyOrchestrationV2ProjectionEvent(before, {
229+
id: EventId.make("text-delta"),
230+
type: "turn-item.updated",
231+
threadId: v2ThreadId,
232+
occurredAt: later,
233+
payload: { ...items.at(-1)!, text: "Hello", updatedAt: later },
234+
});
235+
slotReads = 0;
236+
237+
expect(sameThreadStructure(before, next!)).toBe(true);
238+
expect(slotReads).toBeLessThanOrEqual(2);
239+
});
240+
241+
function replaceItem(projection: OrchestrationV2ThreadProjection, item: OrchestrationV2TurnItem) {
242+
return applyOrchestrationV2ProjectionEvent(projection, {
243+
id: EventId.make(`event-${item.id}`),
244+
type: "turn-item.updated",
245+
threadId: v2ThreadId,
246+
occurredAt: item.updatedAt,
247+
payload: item,
248+
})!;
249+
}
250+
251+
it("matches full structure comparisons across streamed, batched, and reordered updates", () => {
252+
const message = { ...assistant, ordinal: 2 };
253+
const reasoning = {
254+
...assistant,
255+
id: TurnItemId.make("reasoning"),
256+
type: "reasoning" as const,
257+
ordinal: 1,
258+
};
259+
const initial = mergeOlderHistoryIntoProjection(withItem(message), [
260+
withItem(reasoning).visibleTurnItems[0]!,
261+
]);
262+
const updates = [
263+
{ ...message, text: "Hello", updatedAt: later },
264+
{ ...reasoning, text: "Thinking", updatedAt: later },
265+
{ ...reasoning, text: "Done", status: "completed" as const, streaming: false },
266+
{ ...message, text: "Hello there", updatedAt: later },
267+
{ ...message, text: "Hello there", ordinal: 0 },
268+
{ ...message, text: "More text", ordinal: 0 },
269+
{ ...message, text: "More text", ordinal: 2 },
270+
{ ...message, text: "More text", runId: RunId.make("another-run") },
271+
];
272+
const projections = [initial];
273+
const withoutHints = (projection: OrchestrationV2ThreadProjection) => ({
274+
...projection,
275+
turnItems: projection.turnItems.slice(),
276+
visibleTurnItems: projection.visibleTurnItems.slice(),
277+
});
278+
const consecutive: boolean[] = [];
279+
for (const item of updates) {
280+
const previous = projections.at(-1)!;
281+
const next = replaceItem(previous, item);
282+
consecutive.push(sameThreadStructure(previous, next));
283+
// All earlier snapshots include readers that skip one or several updates in a batch.
284+
for (const before of projections) {
285+
expect(sameThreadStructure(before, next)).toBe(
286+
sameThreadStructure(withoutHints(before), withoutHints(next)),
287+
);
288+
}
289+
projections.push(next);
290+
}
291+
292+
// A resumed snapshot can reuse item ids while changing a different row.
293+
const snapshot = withoutHints(projections[1]!);
294+
const resumed = replaceItem(snapshot, { ...reasoning, title: "Restored reasoning" });
295+
const sibling = replaceItem(initial, { ...message, text: "Another stream" });
296+
const reconnected = replaceItem(resumed, { ...message, text: "Resumed stream" });
297+
for (const before of [initial, sibling, projections[1]!, resumed]) {
298+
expect(sameThreadStructure(before, reconnected)).toBe(
299+
sameThreadStructure(withoutHints(before), withoutHints(reconnected)),
300+
);
301+
}
302+
expect(sameThreadStructure(sibling, reconnected)).toBe(false);
303+
expect(sameThreadStructure(resumed, reconnected)).toBe(true);
304+
305+
expect(consecutive).toEqual([true, true, false, true, false, true, false, false]);
306+
expect(sameThreadStructure(initial, projections[4]!)).toBe(false);
307+
expect(projections[5]!.visibleTurnItems.map((row) => [row.item.id, row.position])).toEqual([
308+
[assistant.id, 0],
309+
[reasoning.id, 1],
310+
]);
311+
expect(projections[7]!.visibleTurnItems.map((row) => [row.item.id, row.position])).toEqual([
312+
[reasoning.id, 0],
313+
[assistant.id, 1],
314+
]);
315+
});

0 commit comments

Comments
 (0)