Skip to content

Commit eba1c31

Browse files
hieu-leeclaude
andcommitted
perf(server): switching providers no longer rescans the whole thread for handoff context
Handoff history selection serialized every candidate message (twice) to measure it, even after the budget was spent, and the provider-switch paths loaded and decoded every turn item (tool calls, reasoning) only to discard all but seven types. selectHistory now sizes messages arithmetically without building strings, skips bodies that cannot fit and stops once the budget is exhausted; selection output is unchanged. Switch paths read only the historical item types, and recovery looks up earlier runs through a Set instead of scanning runs per item. Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
1 parent 4f0b74e commit eba1c31

5 files changed

Lines changed: 176 additions & 55 deletions

File tree

‎apps/server/src/orchestration-v2/ContextHandoffBudget.test.ts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import {
2323
historyResponseItems,
2424
selectHistory,
2525
historicalMessage,
26+
jsonEscapedByteLength,
2627
} from "@t3tools/provider-core/server/handoffBudget";
2728
import { projectContextHandoffForWire } from "./WireProjection.ts";
2829
import { deliverContextHandoffs } from "./ContextHandoffDelivery.ts";
@@ -193,6 +194,44 @@ describe("handoff budget", () => {
193194
}
194195
});
195196

197+
it("counts escaped JSON bytes exactly, including lone surrogates and control characters", () => {
198+
const samples = [
199+
"",
200+
"plain ascii",
201+
'\u0000\u0008\t\n\u000b\f\r\u001f"\\',
202+
"界é€𝄞🧪",
203+
"\ud800",
204+
"\udc00x",
205+
"a\ud83d",
206+
"\ud83d\ude00\ud83d",
207+
"\u007f\u0080\u07ff\u0800\uffff",
208+
];
209+
for (const sample of samples) {
210+
assert.strictEqual(
211+
jsonEscapedByteLength(sample),
212+
Buffer.byteLength(JSON.stringify(sample)) - 2,
213+
);
214+
}
215+
});
216+
217+
it("selects exactly what the serialized envelope allows at every budget edge", () => {
218+
const candidates = [
219+
message("edge:0", "user", "界🧪\n".repeat(7)),
220+
message("edge:1", "assistant", 'quote " and \\ slash'),
221+
message("edge:2", "user", "x".repeat(300)),
222+
];
223+
for (let budget = 1_024; budget <= 2_200; budget++) {
224+
const selected = selectHistory({ messages: candidates, coverage: "Recover", budget });
225+
assert.isAtMost(historyCost(selected.messages, selected.context), budget);
226+
const smaller = selectHistory({
227+
messages: candidates,
228+
coverage: "Recover",
229+
budget: budget - 1,
230+
});
231+
assert.isAtMost(smaller.messages.length, selected.messages.length);
232+
}
233+
});
234+
196235
it("fits the final envelope at intermediate selected/omitted digit boundaries", () => {
197236
const candidates = Array.from({ length: 20 }, (_, index) =>
198237
message(`boundary:${index}`, "user", "Short request"),

‎apps/server/src/orchestration-v2/ContextHandoffService.ts‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -524,15 +524,11 @@ const makeContextHandoffService = Effect.fn("orchestrationV2.ContextHandoffServi
524524
const coverage = handoffCoverage(input);
525525
const selected = selectHistory({
526526
messages: input.items.flatMap((item) => {
527-
const message = historicalMessage(item);
528-
return message === null
529-
? []
530-
: [
531-
{
532-
...message,
533-
...(item.runId === null ? {} : { runStatus: runStatuses.get(item.runId) }),
534-
},
535-
];
527+
const message = historicalMessage(
528+
item,
529+
item.runId === null ? undefined : runStatuses.get(item.runId),
530+
);
531+
return message === null ? [] : [message];
536532
}),
537533
coverage,
538534
budget: tokenCap,

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@ import {
119119
visibleConversationItems,
120120
visibleDeltaRunOrdinals,
121121
} from "./ContextHandoffService.ts";
122+
import { HISTORICAL_ITEM_TYPES } from "@t3tools/provider-core/server/handoffBudget";
122123
import { notificationTurnItem } from "@t3tools/provider-core/server/notification";
123124
import {
124125
cancelledBeforeStart,
@@ -1332,13 +1333,16 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
13321333
Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })),
13331334
);
13341335

1335-
const readHandoffItems = (threadId: ThreadId, runIds?: ReadonlyArray<RunId | null>) =>
1336+
const readHandoffItems = (
1337+
threadId: ThreadId,
1338+
runIds?: ReadonlyArray<RunId | null>,
1339+
turnItemTypes?: ReadonlyArray<OrchestrationV2TurnItem["type"]>,
1340+
) =>
13361341
projectionStore
1337-
.getThreadRecords(
1338-
threadId,
1339-
["turnItems"],
1340-
runIds === undefined ? undefined : { turnItemRunIds: runIds },
1341-
)
1342+
.getThreadRecords(threadId, ["turnItems"], {
1343+
...(runIds === undefined ? {} : { turnItemRunIds: runIds }),
1344+
...(turnItemTypes === undefined ? {} : { turnItemTypes }),
1345+
})
13421346
.pipe(
13431347
Effect.map((records) => records.turnItems),
13441348
Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })),
@@ -2150,6 +2154,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
21502154
...(yield* readHandoffItems(
21512155
threadId,
21522156
coveredRuns.map((run) => run.id),
2157+
HISTORICAL_ITEM_TYPES,
21532158
)),
21542159
],
21552160
createdAt: now,
@@ -5442,6 +5447,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
54425447
: input.projection.runs
54435448
.filter((run) => run.ordinal >= contextBoundary)
54445449
.map((run) => run.id),
5450+
HISTORICAL_ITEM_TYPES,
54455451
),
54465452
runs: input.projection.runs,
54475453
createdAt: now,
@@ -7303,6 +7309,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
73037309
...(yield* readHandoffItems(
73047310
command.threadId,
73057311
providerSwitchCoveredRuns.map((run) => run.id),
7312+
HISTORICAL_ITEM_TYPES,
73067313
)),
73077314
];
73087315
const providerSwitchTransferId =

‎apps/server/src/orchestration-v2/ProviderTurnStartService.ts‎

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,8 @@ import {
4242
handoffBudget,
4343
attachmentTokenAllowance,
4444
contextUsageForHandoff,
45-
historicalMessage,
45+
historicalText,
46+
isHistoricalItem,
4647
latestNativeContextUsage,
4748
} from "@t3tools/provider-core/server/handoffBudget";
4849
import { deliverContextHandoffs, withInlineContext } from "./ContextHandoffDelivery.ts";
@@ -970,16 +971,12 @@ export const layer: Layer.Layer<
970971
!legacyRecoveredRunIds.has(item.runId)));
971972
if (item.runId === run.id || (!acceptedNatively && !deliveredItemIds.has(item.id)))
972973
return sum;
973-
const historical = historicalMessage(item);
974+
const text = historicalText(item);
974975
const nativeAttachments =
975976
item.type === "user_message" && acceptedNatively
976977
? attachmentTokenAllowance(item.attachments)
977978
: 0;
978-
return (
979-
sum +
980-
(historical === null ? 0 : Buffer.byteLength(historical.text)) +
981-
nativeAttachments
982-
);
979+
return sum + (text === null ? 0 : Buffer.byteLength(text)) + nativeAttachments;
983980
}, 0)
984981
: 0;
985982
});
@@ -1039,7 +1036,7 @@ export const layer: Layer.Layer<
10391036
item.runId !== null &&
10401037
missedRunIds.has(item.runId) &&
10411038
!coveredItemIds.has(item.id) &&
1042-
historicalMessage(item) !== null,
1039+
isHistoricalItem(item),
10431040
);
10441041
const retryHandoff =
10451042
missedItems.length === 0
@@ -1074,6 +1071,11 @@ export const layer: Layer.Layer<
10741071
type: "provider_resume_fallback",
10751072
});
10761073
const createdAt = yield* DateTime.now;
1074+
const earlierRunIds = new Set(
1075+
projection.runs
1076+
.filter((source) => source.ordinal < run.ordinal)
1077+
.map((source) => source.id),
1078+
);
10771079
const handoff = yield* contextHandoffService.prepareProviderHandoff({
10781080
threadId: projection.thread.id,
10791081
targetRunId: run.id,
@@ -1086,11 +1088,7 @@ export const layer: Layer.Layer<
10861088
strategy: "full_thread_summary",
10871089
runs: projection.runs,
10881090
items: (yield* projectionStore.getTurnStartHistory(input.threadId)).filter(
1089-
(item) =>
1090-
item.runId === null ||
1091-
projection.runs.some(
1092-
(source) => source.id === item.runId && source.ordinal < run.ordinal,
1093-
),
1091+
(item) => item.runId === null || earlierRunIds.has(item.runId),
10941092
),
10951093
createdAt,
10961094
});

‎packages/provider-core/src/server/handoffBudget.ts‎

Lines changed: 108 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -124,51 +124,69 @@ export function handoffBudget(input: {
124124
);
125125
}
126126

127-
export function historicalMessage(
128-
item: OrchestrationV2TurnItem,
129-
): OrchestrationV2HistoricalMessage | null {
130-
let text: string;
127+
export const HISTORICAL_ITEM_TYPES = [
128+
"user_message",
129+
"assistant_message",
130+
"command_execution",
131+
"error",
132+
"run_interrupt_result",
133+
"file_change",
134+
"proposed_plan",
135+
] as const satisfies ReadonlyArray<OrchestrationV2TurnItem["type"]>;
136+
const historicalItemTypes: ReadonlySet<string> = new Set(HISTORICAL_ITEM_TYPES);
137+
138+
export const isHistoricalItem = (item: OrchestrationV2TurnItem) =>
139+
historicalItemTypes.has(item.type);
140+
141+
export function historicalText(item: OrchestrationV2TurnItem): string | null {
131142
switch (item.type) {
132143
case "user_message":
133144
case "assistant_message":
134-
text = item.text;
135-
break;
145+
return item.text;
136146
case "command_execution":
137-
text = [
147+
return [
138148
`Command: ${item.input}`,
139149
`Exit code: ${item.exitCode ?? "unknown"}`,
140150
item.output ?? "",
141151
].join("\n");
142-
break;
143152
case "error":
144-
text = item.failure.message;
145-
break;
153+
return item.failure.message;
146154
case "run_interrupt_result":
147-
text = item.message;
148-
break;
155+
return item.message;
149156
case "file_change":
150-
text = `File change: ${item.changes?.map((change) => change.path).join(", ") ?? item.fileName}`;
151-
break;
157+
return `File change: ${item.changes?.map((change) => change.path).join(", ") ?? item.fileName}`;
152158
case "proposed_plan":
153-
text = item.markdown;
154-
break;
159+
return item.markdown;
155160
default:
156161
return null;
157162
}
163+
}
164+
165+
export function historicalMessage(
166+
item: OrchestrationV2TurnItem,
167+
runStatus?: OrchestrationV2HistoricalMessage["runStatus"],
168+
): OrchestrationV2HistoricalMessage | null {
169+
const text = historicalText(item);
170+
if (text === null) return null;
158171
return {
159172
role: item.type === "user_message" ? "user" : "assistant",
160173
text,
161174
threadId: item.threadId,
162175
runId: item.runId,
163176
itemId: item.id,
164-
providerThreadId: item.providerThreadId,
177+
providerThreadId: item.providerThreadId ?? null,
165178
status: item.status,
166179
kind: item.type,
180+
...(runStatus === undefined ? {} : { runStatus }),
167181
};
168182
}
169183

184+
function historicalMessageHeader(message: OrchestrationV2HistoricalMessage): string {
185+
return `[Historical ${message.role}; ${message.kind}; thread=${message.threadId}; run=${message.runId ?? "imported"}; item=${message.itemId}; provider-thread=${message.providerThreadId ?? "none"}; status=${message.status}${message.runStatus === undefined ? "" : `; run-status=${message.runStatus}`}]`;
186+
}
187+
170188
function renderHistoricalMessage(message: OrchestrationV2HistoricalMessage): string {
171-
return `[Historical ${message.role}; ${message.kind}; thread=${message.threadId}; run=${message.runId ?? "imported"}; item=${message.itemId}; provider-thread=${message.providerThreadId ?? "none"}; status=${message.status}${message.runStatus === undefined ? "" : `; run-status=${message.runStatus}`}]\n${message.text}`;
189+
return `${historicalMessageHeader(message)}\n${message.text}`;
172190
}
173191

174192
export function historyResponseItems(
@@ -199,6 +217,67 @@ export function renderHistory(
199217

200218
const jsonByteLength = (value: unknown) => Buffer.byteLength(JSON.stringify(value));
201219

220+
/** UTF-8 bytes of `JSON.stringify(text)` without the quotes, counted without building the string. */
221+
export function jsonEscapedByteLength(text: string): number {
222+
let bytes = 0;
223+
for (let index = 0; index < text.length; index++) {
224+
const code = text.charCodeAt(index);
225+
if (code < 0x20) {
226+
bytes += code === 8 || code === 9 || code === 10 || code === 12 || code === 13 ? 2 : 6;
227+
} else if (code === 0x22 || code === 0x5c) {
228+
bytes += 2;
229+
} else if (code < 0x80) {
230+
bytes += 1;
231+
} else if (code < 0x800) {
232+
bytes += 2;
233+
} else if (code >= 0xd800 && code <= 0xdbff) {
234+
const next = text.charCodeAt(index + 1);
235+
if (next >= 0xdc00 && next <= 0xdfff) {
236+
bytes += 4;
237+
index++;
238+
} else {
239+
bytes += 6;
240+
}
241+
} else if (code >= 0xdc00 && code <= 0xdfff) {
242+
bytes += 6;
243+
} else {
244+
bytes += 3;
245+
}
246+
}
247+
return bytes;
248+
}
249+
250+
// JSON string escaping is concatenative, so a message's serialized size is its escaped
251+
// header and body plus a constant per-role envelope; selection never builds the strings.
252+
const envelopeBytes = (role: OrchestrationV2HistoricalMessage["role"]) => {
253+
const probe = {
254+
role,
255+
text: "",
256+
threadId: "",
257+
runId: null,
258+
itemId: "",
259+
providerThreadId: null,
260+
status: "completed",
261+
kind: "user_message",
262+
} as unknown as OrchestrationV2HistoricalMessage;
263+
const rendered = renderHistoricalMessage(probe);
264+
return jsonByteLength(historyResponseItems([probe], "")[1]) - jsonByteLength(rendered);
265+
};
266+
const RESPONSE_ENVELOPE_BYTES = {
267+
user: envelopeBytes("user"),
268+
assistant: envelopeBytes("assistant"),
269+
};
270+
const MIN_MESSAGE_COST = Math.min(RESPONSE_ENVELOPE_BYTES.user, RESPONSE_ENVELOPE_BYTES.assistant);
271+
272+
function messageCost(message: OrchestrationV2HistoricalMessage): number {
273+
const rendered =
274+
jsonEscapedByteLength(historicalMessageHeader(message)) +
275+
2 +
276+
jsonEscapedByteLength(message.text) +
277+
2;
278+
return Math.max(rendered + RESPONSE_ENVELOPE_BYTES[message.role] + 1, rendered + 4);
279+
}
280+
202281
export function historyCost(
203282
messages: ReadonlyArray<OrchestrationV2HistoricalMessage>,
204283
context: string,
@@ -231,23 +310,25 @@ export function selectHistory(input: {
231310
const tryAdd = (index: number) => {
232311
const message = messages[index];
233312
if (message === undefined || selected.has(index) || message.text.length > remaining) return;
234-
const cost = Math.max(
235-
Buffer.byteLength(JSON.stringify(historyResponseItems([message], "")[1])) + 1,
236-
Buffer.byteLength(JSON.stringify(renderHistoricalMessage(message))) + 4,
237-
);
313+
const cost = messageCost(message);
238314
if (cost > remaining) return;
239315
selected.add(index);
240316
remaining -= cost;
241317
};
242318
tryAdd(messages.findLastIndex((message) => message.role === "user"));
243319
tryAdd(messages.findLastIndex((message) => message.role === "assistant"));
244320
tryAdd(messages.findIndex((message) => message.role === "user"));
245-
for (let index = messages.length - 1; index >= 0; index--) tryAdd(index);
321+
for (let index = messages.length - 1; index >= 0 && remaining >= MIN_MESSAGE_COST; index--)
322+
tryAdd(index);
323+
const selectedMessages: Array<OrchestrationV2HistoricalMessage> = [];
324+
const omittedItemIds: Array<OrchestrationV2HistoricalMessage["itemId"]> = [];
325+
for (const [index, message] of messages.entries()) {
326+
if (selected.has(index)) selectedMessages.push(message);
327+
else omittedItemIds.push(message.itemId);
328+
}
246329
return {
247-
messages: messages.filter((_, index) => selected.has(index)),
248-
omittedItemIds: messages
249-
.filter((_, index) => !selected.has(index))
250-
.map((message) => message.itemId),
330+
messages: selectedMessages,
331+
omittedItemIds,
251332
context: contextFor(selected.size),
252333
omittedItems: (input.omittedItems ?? 0) + messages.length - selected.size,
253334
};

0 commit comments

Comments
 (0)