Skip to content

Commit e19fe08

Browse files
t3dotggclaude
andauthored
perf(server): thread opens and live events send each message body once (pingdotgg#24647)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent d228f1f commit e19fe08

16 files changed

Lines changed: 943 additions & 29 deletions

‎apps/server/src/orchestration-v2/ThreadStream.ts‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import type {
44
} from "@t3tools/contracts";
55

66
import { omitLocalVisibleTurnItems } from "@t3tools/shared/orchestrationV2BoundedSnapshot";
7+
import { dedupeSnapshotBodies } from "@t3tools/shared/orchestrationV2DedupedBodies";
78

89
import {
910
buildBoundedThreadProjection,
@@ -42,13 +43,16 @@ type ThreadSnapshotStreamItem = Extract<
4243
export function boundedSnapshotResponseFields(input: {
4344
readonly bounded: BoundedProjectionResult;
4445
readonly compactTurnItems: boolean;
46+
readonly dedupeBodies: boolean;
4547
}) {
4648
const { bounded } = input;
47-
const compactProjection = input.compactTurnItems
48-
? omitLocalVisibleTurnItems(bounded.projection)
49-
: null;
49+
// Dedupe keeps shared items one object, which the compaction's identity check needs.
50+
const projection = input.dedupeBodies
51+
? dedupeSnapshotBodies(bounded.projection)
52+
: bounded.projection;
53+
const compactProjection = input.compactTurnItems ? omitLocalVisibleTurnItems(projection) : null;
5054
return {
51-
projection: compactProjection ?? bounded.projection,
55+
projection: compactProjection ?? projection,
5256
historyCursor: bounded.historyCursor,
5357
hasMoreHistory: bounded.hasMoreHistory,
5458
latestLocalTurnOrdinal: bounded.latestLocalTurnOrdinal,
@@ -60,6 +64,7 @@ export function buildBoundedThreadStreamSnapshot(input: {
6064
readonly snapshotSequence: number;
6165
readonly projection: OrchestrationV2ThreadProjection;
6266
readonly compactTurnItems?: boolean;
67+
readonly dedupeBodies?: boolean;
6368
}): ThreadSnapshotStreamItem {
6469
const bounded = buildBoundedThreadProjection({
6570
snapshotSequence: input.snapshotSequence,
@@ -71,6 +76,7 @@ export function buildBoundedThreadStreamSnapshot(input: {
7176
...boundedSnapshotResponseFields({
7277
bounded,
7378
compactTurnItems: input.compactTurnItems === true,
79+
dedupeBodies: input.dedupeBodies === true,
7480
}),
7581
};
7682
}

‎apps/server/src/orchestration-v2/WireProjection.test.ts‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,15 +9,19 @@ import {
99
ProviderInstanceId,
1010
ThreadId,
1111
TurnItemId,
12+
type OrchestrationV2DomainEvent,
1213
type OrchestrationV2ThreadProjection,
14+
type OrchestrationV2ThreadStreamItem,
1315
OrchestrationV2TurnItem,
1416
OrchestrationV2TurnItemJson,
1517
} from "@t3tools/contracts";
1618
import { describe, expect, it } from "@effect/vitest";
1719
import * as DateTime from "effect/DateTime";
1820
import * as Schema from "effect/Schema";
1921

22+
import { coalesceLiveUpdatedEvents } from "./ThreadLiveEventCoalescer.ts";
2023
import {
24+
makeLiveBodyDeduper,
2125
projectTurnItemForWire,
2226
projectTurnItemForDetail,
2327
projectDomainEventForWire,
@@ -302,3 +306,74 @@ describe("orchestration V2 wire projection", () => {
302306
expect(projectDomainEventForWire(projected)).toEqual(projected);
303307
});
304308
});
309+
310+
describe("live body deduper", () => {
311+
const at = base.updatedAt;
312+
const messageId = MessageId.make("message-live");
313+
const event = (
314+
sequence: number,
315+
rest: Pick<OrchestrationV2DomainEvent, "type" | "payload">,
316+
): Extract<OrchestrationV2ThreadStreamItem, { kind: "event" }> => ({
317+
kind: "event",
318+
sequence,
319+
event: {
320+
id: EventId.make(`event-${sequence}`),
321+
threadId: base.threadId,
322+
occurredAt: at,
323+
...rest,
324+
} as OrchestrationV2DomainEvent,
325+
});
326+
const message = (sequence: number, text: string) =>
327+
event(sequence, {
328+
type: "message.updated",
329+
payload: {
330+
id: messageId,
331+
threadId: base.threadId,
332+
runId: null,
333+
nodeId: null,
334+
role: "assistant",
335+
text,
336+
attachments: [],
337+
streaming: true,
338+
createdBy: "agent",
339+
creationSource: "provider",
340+
createdAt: at,
341+
updatedAt: at,
342+
},
343+
});
344+
const reply = (sequence: number, text: string) =>
345+
event(sequence, {
346+
type: "turn-item.updated",
347+
payload: {
348+
...base,
349+
status: "running",
350+
type: "assistant_message",
351+
messageId,
352+
text,
353+
streaming: true,
354+
},
355+
});
356+
const payloadOf = (item: OrchestrationV2ThreadStreamItem) =>
357+
item.kind === "event" ? item.event.payload : undefined;
358+
359+
it("blanks a reply that repeats the message delivered before it, after coalescing", () => {
360+
const delivered = coalesceLiveUpdatedEvents([
361+
message(1, "a"),
362+
reply(2, "a"),
363+
message(3, "ab"),
364+
reply(4, "ab"),
365+
]).map(makeLiveBodyDeduper());
366+
367+
expect(
368+
delivered.map((item) => [item.kind === "event" && item.sequence, payloadOf(item)]),
369+
).toEqual([
370+
[3, message(3, "ab").event.payload],
371+
[4, { ...reply(4, "ab").event.payload, text: "", textOmitted: true }],
372+
]);
373+
});
374+
375+
it("ships a reply whole when its message arrives after it", () => {
376+
const first = reply(1, "a");
377+
expect([first, message(2, "a")].map(makeLiveBodyDeduper())[0]).toBe(first);
378+
});
379+
});

‎apps/server/src/orchestration-v2/WireProjection.ts‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,19 @@
11
import type {
2+
CheckpointId,
3+
MessageId,
4+
NodeId,
5+
OrchestrationV2Checkpoint,
26
OrchestrationV2DomainEvent,
37
OrchestrationV2ContextHandoff,
8+
OrchestrationV2Subagent,
49
OrchestrationV2ThreadProjection,
10+
OrchestrationV2ThreadStreamItem,
511
OrchestrationV2TurnItem,
612
} from "@t3tools/contracts";
13+
import {
14+
dedupeSubagentBodies,
15+
dedupeTurnItemBodies,
16+
} from "@t3tools/shared/orchestrationV2DedupedBodies";
717
import {
818
compactDynamicToolOutput,
919
omitToolOutputImageData,
@@ -245,3 +255,65 @@ export function projectDomainEventForWire(
245255
? { ...event, payload: projectContextHandoffForWire(event.payload) }
246256
: event;
247257
}
258+
259+
const LIVE_DEDUPE_MEMORY = 32;
260+
261+
function remember<K, V>(memory: Map<K, V>, key: K, value: V): void {
262+
memory.delete(key);
263+
memory.set(key, value);
264+
if (memory.size > LIVE_DEDUPE_MEMORY) memory.delete(memory.keys().next().value as K);
265+
}
266+
267+
/**
268+
* One per subscription, applied after coalescing so it only remembers records the client
269+
* received, in the snapshot or in events. A record event comes before the turn item that repeats
270+
* its body (adapters and the orchestrator emit them in that order), so the item can ship blank
271+
* and marked; so can a repeated subagent record. Emitting in the other order, or forgetting a
272+
* record, only costs bytes: the repeat ships whole.
273+
*/
274+
export function makeLiveBodyDeduper(): (
275+
item: OrchestrationV2ThreadStreamItem,
276+
) => OrchestrationV2ThreadStreamItem {
277+
const messageText = new Map<MessageId, string>();
278+
const checkpointFiles = new Map<CheckpointId, OrchestrationV2Checkpoint["files"]>();
279+
const subagents = new Map<NodeId, Pick<OrchestrationV2Subagent, "prompt" | "result">>();
280+
const source = {
281+
messageText: (id: MessageId) => messageText.get(id),
282+
checkpointFiles: (id: CheckpointId) => checkpointFiles.get(id),
283+
subagent: (id: NodeId) => subagents.get(id),
284+
};
285+
const rememberSubagent = ({ id, prompt, result }: OrchestrationV2Subagent) =>
286+
remember(subagents, id, { prompt, result });
287+
return (item) => {
288+
if (item.kind === "snapshot") {
289+
// The client now holds exactly these records; their bodies are never blanked.
290+
const { messages, checkpoints, subagents: records } = item.projection;
291+
for (const message of messages) remember(messageText, message.id, message.text);
292+
for (const checkpoint of checkpoints)
293+
remember(checkpointFiles, checkpoint.id, checkpoint.files);
294+
for (const subagent of records) rememberSubagent(subagent);
295+
return item;
296+
}
297+
if (item.kind !== "event") return item;
298+
const { event } = item;
299+
switch (event.type) {
300+
case "message.updated":
301+
remember(messageText, event.payload.id, event.payload.text);
302+
return item;
303+
case "checkpoint.captured":
304+
remember(checkpointFiles, event.payload.id, event.payload.files);
305+
return item;
306+
case "subagent.updated": {
307+
const payload = dedupeSubagentBodies(event.payload, subagents.get(event.payload.id));
308+
rememberSubagent(event.payload);
309+
return payload === event.payload ? item : { ...item, event: { ...event, payload } };
310+
}
311+
case "turn-item.updated": {
312+
const payload = dedupeTurnItemBodies(event.payload, source);
313+
return payload === event.payload ? item : { ...item, event: { ...event, payload } };
314+
}
315+
default:
316+
return item;
317+
}
318+
};
319+
}

0 commit comments

Comments
 (0)