Skip to content

Commit da9b500

Browse files
hieu-leeclaude
andcommitted
perf(server): switching providers no longer rescans the whole thread for handoff context
Handoff history selection serialized every candidate message (twice) to measure it, even after the budget was spent, and the provider-switch paths loaded and decoded every turn item (tool calls, reasoning) only to discard all but seven types. selectHistory now sizes messages arithmetically without building strings, skips bodies that cannot fit and stops once the budget is exhausted; selection output is unchanged. Switch paths read only the historical item types, and recovery looks up earlier runs through a Set instead of scanning runs per item. Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
1 parent 616c426 commit da9b500

5 files changed

Lines changed: 178 additions & 57 deletions

File tree

‎apps/server/src/orchestration-v2/ContextHandoffBudget.test.ts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import {
2323
historyResponseItems,
2424
selectHistory,
2525
historicalMessage,
26+
jsonEscapedByteLength,
2627
} from "@t3tools/provider-core/server/handoffBudget";
2728
import { projectContextHandoffForWire } from "./WireProjection.ts";
2829
import { deliverContextHandoffs } from "./ContextHandoffDelivery.ts";
@@ -180,6 +181,44 @@ describe("handoff budget", () => {
180181
}
181182
});
182183

184+
it("counts escaped JSON bytes exactly, including lone surrogates and control characters", () => {
185+
const samples = [
186+
"",
187+
"plain ascii",
188+
'\u0000\u0008\t\n\u000b\f\r\u001f"\\',
189+
"界é€𝄞🧪",
190+
"\ud800",
191+
"\udc00x",
192+
"a\ud83d",
193+
"\ud83d\ude00\ud83d",
194+
"\u007f\u0080\u07ff\u0800\uffff",
195+
];
196+
for (const sample of samples) {
197+
assert.strictEqual(
198+
jsonEscapedByteLength(sample),
199+
Buffer.byteLength(JSON.stringify(sample)) - 2,
200+
);
201+
}
202+
});
203+
204+
it("selects exactly what the serialized envelope allows at every budget edge", () => {
205+
const candidates = [
206+
message("edge:0", "user", "界🧪\n".repeat(7)),
207+
message("edge:1", "assistant", 'quote " and \\ slash'),
208+
message("edge:2", "user", "x".repeat(300)),
209+
];
210+
for (let budget = 1_024; budget <= 2_200; budget++) {
211+
const selected = selectHistory({ messages: candidates, coverage: "Recover", budget });
212+
assert.isAtMost(historyCost(selected.messages, selected.context), budget);
213+
const smaller = selectHistory({
214+
messages: candidates,
215+
coverage: "Recover",
216+
budget: budget - 1,
217+
});
218+
assert.isAtMost(smaller.messages.length, selected.messages.length);
219+
}
220+
});
221+
183222
it("fits the final envelope at intermediate selected/omitted digit boundaries", () => {
184223
const candidates = Array.from({ length: 20 }, (_, index) =>
185224
message(`boundary:${index}`, "user", "Short request"),

‎apps/server/src/orchestration-v2/ContextHandoffService.ts‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -375,15 +375,11 @@ const makeContextHandoffService = Effect.fn("orchestrationV2.ContextHandoffServi
375375
const coverage = handoffCoverage(input);
376376
const selected = selectHistory({
377377
messages: input.items.flatMap((item) => {
378-
const message = historicalMessage(item);
379-
return message === null
380-
? []
381-
: [
382-
{
383-
...message,
384-
...(item.runId === null ? {} : { runStatus: runStatuses.get(item.runId) }),
385-
},
386-
];
378+
const message = historicalMessage(
379+
item,
380+
item.runId === null ? undefined : runStatuses.get(item.runId),
381+
);
382+
return message === null ? [] : [message];
387383
}),
388384
coverage,
389385
budget: tokenCap,

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ import {
104104
} from "./CommandPolicy.ts";
105105
import { CommandReceiptStoreV2 } from "./CommandReceiptStore.ts";
106106
import { ContextHandoffServiceV2 } from "./ContextHandoffService.ts";
107+
import { HISTORICAL_ITEM_TYPES } from "@t3tools/provider-core/server/handoffBudget";
107108
import { notificationTurnItem } from "@t3tools/provider-core/server/notification";
108109
import {
109110
cancelledBeforeStart,
@@ -1167,13 +1168,16 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
11671168
Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })),
11681169
);
11691170

1170-
const readHandoffItems = (threadId: ThreadId, runIds?: ReadonlyArray<RunId | null>) =>
1171+
const readHandoffItems = (
1172+
threadId: ThreadId,
1173+
runIds?: ReadonlyArray<RunId | null>,
1174+
turnItemTypes?: ReadonlyArray<OrchestrationV2TurnItem["type"]>,
1175+
) =>
11711176
projectionStore
1172-
.getThreadRecords(
1173-
threadId,
1174-
["turnItems"],
1175-
runIds === undefined ? undefined : { turnItemRunIds: runIds },
1176-
)
1177+
.getThreadRecords(threadId, ["turnItems"], {
1178+
...(runIds === undefined ? {} : { turnItemRunIds: runIds }),
1179+
...(turnItemTypes === undefined ? {} : { turnItemTypes }),
1180+
})
11771181
.pipe(
11781182
Effect.map((records) => records.turnItems),
11791183
Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })),
@@ -1820,6 +1824,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
18201824
...(yield* readHandoffItems(
18211825
threadId,
18221826
coveredRuns.map((run) => run.id),
1827+
HISTORICAL_ITEM_TYPES,
18231828
)),
18241829
],
18251830
createdAt: now,
@@ -4879,7 +4884,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
48794884
toProviderInstanceId: input.modelSelection.instanceId,
48804885
coveredRunOrdinals: { from: 1, to: targetRun.ordinal },
48814886
strategy: "full_thread_summary",
4882-
items: yield* readHandoffItems(input.command.threadId),
4887+
items: yield* readHandoffItems(input.command.threadId, undefined, HISTORICAL_ITEM_TYPES),
48834888
runs: input.projection.runs,
48844889
createdAt: now,
48854890
})
@@ -6590,6 +6595,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
65906595
...(yield* readHandoffItems(
65916596
command.threadId,
65926597
providerSwitchCoveredRuns.map((run) => run.id),
6598+
HISTORICAL_ITEM_TYPES,
65936599
)),
65946600
];
65956601
const providerSwitchTransferId =

‎apps/server/src/orchestration-v2/ProviderTurnStartService.ts‎

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,8 @@ import {
4040
handoffBudget,
4141
attachmentTokenAllowance,
4242
contextUsageForHandoff,
43-
historicalMessage,
43+
historicalText,
44+
isHistoricalItem,
4445
latestNativeContextUsage,
4546
} from "@t3tools/provider-core/server/handoffBudget";
4647
import { deliverContextHandoffs } from "./ContextHandoffDelivery.ts";
@@ -898,16 +899,12 @@ export const layer: Layer.Layer<
898899
!legacyRecoveredRunIds.has(item.runId)));
899900
if (item.runId === run.id || (!acceptedNatively && !deliveredItemIds.has(item.id)))
900901
return sum;
901-
const historical = historicalMessage(item);
902+
const text = historicalText(item);
902903
const nativeAttachments =
903904
item.type === "user_message" && acceptedNatively
904905
? attachmentTokenAllowance(item.attachments)
905906
: 0;
906-
return (
907-
sum +
908-
(historical === null ? 0 : Buffer.byteLength(historical.text)) +
909-
nativeAttachments
910-
);
907+
return sum + (text === null ? 0 : Buffer.byteLength(text)) + nativeAttachments;
911908
}, 0)
912909
: 0;
913910
});
@@ -967,7 +964,7 @@ export const layer: Layer.Layer<
967964
item.runId !== null &&
968965
missedRunIds.has(item.runId) &&
969966
!coveredItemIds.has(item.id) &&
970-
historicalMessage(item) !== null,
967+
isHistoricalItem(item),
971968
);
972969
const retryHandoff =
973970
missedItems.length === 0
@@ -1002,6 +999,11 @@ export const layer: Layer.Layer<
1002999
type: "provider_resume_fallback",
10031000
});
10041001
const createdAt = yield* DateTime.now;
1002+
const earlierRunIds = new Set(
1003+
projection.runs
1004+
.filter((source) => source.ordinal < run.ordinal)
1005+
.map((source) => source.id),
1006+
);
10051007
const handoff = yield* contextHandoffService.prepareProviderHandoff({
10061008
threadId: projection.thread.id,
10071009
targetRunId: run.id,
@@ -1014,11 +1016,7 @@ export const layer: Layer.Layer<
10141016
strategy: "full_thread_summary",
10151017
runs: projection.runs,
10161018
items: (yield* projectionStore.getTurnStartHistory(input.threadId)).filter(
1017-
(item) =>
1018-
item.runId === null ||
1019-
projection.runs.some(
1020-
(source) => source.id === item.runId && source.ordinal < run.ordinal,
1021-
),
1019+
(item) => item.runId === null || earlierRunIds.has(item.runId),
10221020
),
10231021
createdAt,
10241022
});

‎packages/provider-core/src/server/handoffBudget.ts‎

Lines changed: 110 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -124,51 +124,69 @@ export function handoffBudget(input: {
124124
);
125125
}
126126

127-
export function historicalMessage(
128-
item: OrchestrationV2TurnItem,
129-
): OrchestrationV2HistoricalMessage | null {
130-
let text: string;
127+
export const HISTORICAL_ITEM_TYPES = [
128+
"user_message",
129+
"assistant_message",
130+
"command_execution",
131+
"error",
132+
"run_interrupt_result",
133+
"file_change",
134+
"proposed_plan",
135+
] as const satisfies ReadonlyArray<OrchestrationV2TurnItem["type"]>;
136+
const historicalItemTypes: ReadonlySet<string> = new Set(HISTORICAL_ITEM_TYPES);
137+
138+
export const isHistoricalItem = (item: OrchestrationV2TurnItem) =>
139+
historicalItemTypes.has(item.type);
140+
141+
export function historicalText(item: OrchestrationV2TurnItem): string | null {
131142
switch (item.type) {
132143
case "user_message":
133144
case "assistant_message":
134-
text = item.text;
135-
break;
145+
return item.text;
136146
case "command_execution":
137-
text = [
147+
return [
138148
`Command: ${item.input}`,
139149
`Exit code: ${item.exitCode ?? "unknown"}`,
140150
item.output ?? "",
141151
].join("\n");
142-
break;
143152
case "error":
144-
text = item.failure.message;
145-
break;
153+
return item.failure.message;
146154
case "run_interrupt_result":
147-
text = item.message;
148-
break;
155+
return item.message;
149156
case "file_change":
150-
text = `File change: ${item.fileName}`;
151-
break;
157+
return `File change: ${item.fileName}`;
152158
case "proposed_plan":
153-
text = item.markdown;
154-
break;
159+
return item.markdown;
155160
default:
156161
return null;
157162
}
163+
}
164+
165+
export function historicalMessage(
166+
item: OrchestrationV2TurnItem,
167+
runStatus?: OrchestrationV2HistoricalMessage["runStatus"],
168+
): OrchestrationV2HistoricalMessage | null {
169+
const text = historicalText(item);
170+
if (text === null) return null;
158171
return {
159172
role: item.type === "user_message" ? "user" : "assistant",
160173
text,
161174
threadId: item.threadId,
162175
runId: item.runId,
163176
itemId: item.id,
164-
providerThreadId: item.providerThreadId,
177+
providerThreadId: item.providerThreadId ?? null,
165178
status: item.status,
166179
kind: item.type,
180+
...(runStatus === undefined ? {} : { runStatus }),
167181
};
168182
}
169183

184+
function historicalMessageHeader(message: OrchestrationV2HistoricalMessage): string {
185+
return `[Historical ${message.role}; ${message.kind}; thread=${message.threadId}; run=${message.runId ?? "imported"}; item=${message.itemId}; provider-thread=${message.providerThreadId ?? "none"}; status=${message.status}${message.runStatus === undefined ? "" : `; run-status=${message.runStatus}`}]`;
186+
}
187+
170188
function renderHistoricalMessage(message: OrchestrationV2HistoricalMessage): string {
171-
return `[Historical ${message.role}; ${message.kind}; thread=${message.threadId}; run=${message.runId ?? "imported"}; item=${message.itemId}; provider-thread=${message.providerThreadId ?? "none"}; status=${message.status}${message.runStatus === undefined ? "" : `; run-status=${message.runStatus}`}]\n${message.text}`;
189+
return `${historicalMessageHeader(message)}\n${message.text}`;
172190
}
173191

174192
export function historyResponseItems(
@@ -199,6 +217,67 @@ export function renderHistory(
199217

200218
const jsonByteLength = (value: unknown) => Buffer.byteLength(JSON.stringify(value));
201219

220+
/** UTF-8 bytes of `JSON.stringify(text)` without the quotes, counted without building the string. */
221+
export function jsonEscapedByteLength(text: string): number {
222+
let bytes = 0;
223+
for (let index = 0; index < text.length; index++) {
224+
const code = text.charCodeAt(index);
225+
if (code < 0x20) {
226+
bytes += code === 8 || code === 9 || code === 10 || code === 12 || code === 13 ? 2 : 6;
227+
} else if (code === 0x22 || code === 0x5c) {
228+
bytes += 2;
229+
} else if (code < 0x80) {
230+
bytes += 1;
231+
} else if (code < 0x800) {
232+
bytes += 2;
233+
} else if (code >= 0xd800 && code <= 0xdbff) {
234+
const next = text.charCodeAt(index + 1);
235+
if (next >= 0xdc00 && next <= 0xdfff) {
236+
bytes += 4;
237+
index++;
238+
} else {
239+
bytes += 6;
240+
}
241+
} else if (code >= 0xdc00 && code <= 0xdfff) {
242+
bytes += 6;
243+
} else {
244+
bytes += 3;
245+
}
246+
}
247+
return bytes;
248+
}
249+
250+
// JSON string escaping is concatenative, so a message's serialized size is its escaped
251+
// header and body plus a constant per-role envelope; selection never builds the strings.
252+
const envelopeBytes = (role: OrchestrationV2HistoricalMessage["role"]) => {
253+
const probe = {
254+
role,
255+
text: "",
256+
threadId: "",
257+
runId: null,
258+
itemId: "",
259+
providerThreadId: null,
260+
status: "completed",
261+
kind: "user_message",
262+
} as unknown as OrchestrationV2HistoricalMessage;
263+
const rendered = renderHistoricalMessage(probe);
264+
return jsonByteLength(historyResponseItems([probe], "")[1]) - jsonByteLength(rendered);
265+
};
266+
const RESPONSE_ENVELOPE_BYTES = {
267+
user: envelopeBytes("user"),
268+
assistant: envelopeBytes("assistant"),
269+
};
270+
const MIN_MESSAGE_COST = Math.min(RESPONSE_ENVELOPE_BYTES.user, RESPONSE_ENVELOPE_BYTES.assistant);
271+
272+
function messageCost(message: OrchestrationV2HistoricalMessage): number {
273+
const rendered =
274+
jsonEscapedByteLength(historicalMessageHeader(message)) +
275+
2 +
276+
jsonEscapedByteLength(message.text) +
277+
2;
278+
return Math.max(rendered + RESPONSE_ENVELOPE_BYTES[message.role] + 1, rendered + 4);
279+
}
280+
202281
export function historyCost(
203282
messages: ReadonlyArray<OrchestrationV2HistoricalMessage>,
204283
context: string,
@@ -230,24 +309,27 @@ export function selectHistory(input: {
230309
historyCost([], contextFor(messages.length, (input.omittedItems ?? 0) + messages.length));
231310
const tryAdd = (index: number) => {
232311
const message = messages[index];
233-
if (message === undefined || selected.has(index)) return;
234-
const cost = Math.max(
235-
Buffer.byteLength(JSON.stringify(historyResponseItems([message], "")[1])) + 1,
236-
Buffer.byteLength(JSON.stringify(renderHistoricalMessage(message))) + 4,
237-
);
312+
// Escaped bytes are never fewer than UTF-16 units, so oversized bodies skip the scan.
313+
if (message === undefined || selected.has(index) || message.text.length > remaining) return;
314+
const cost = messageCost(message);
238315
if (cost > remaining) return;
239316
selected.add(index);
240317
remaining -= cost;
241318
};
242319
tryAdd(messages.findLastIndex((message) => message.role === "user"));
243320
tryAdd(messages.findLastIndex((message) => message.role === "assistant"));
244321
tryAdd(messages.findIndex((message) => message.role === "user"));
245-
for (let index = messages.length - 1; index >= 0; index--) tryAdd(index);
322+
for (let index = messages.length - 1; index >= 0 && remaining >= MIN_MESSAGE_COST; index--)
323+
tryAdd(index);
324+
const selectedMessages: Array<OrchestrationV2HistoricalMessage> = [];
325+
const omittedItemIds: Array<OrchestrationV2HistoricalMessage["itemId"]> = [];
326+
for (const [index, message] of messages.entries()) {
327+
if (selected.has(index)) selectedMessages.push(message);
328+
else omittedItemIds.push(message.itemId);
329+
}
246330
return {
247-
messages: messages.filter((_, index) => selected.has(index)),
248-
omittedItemIds: messages
249-
.filter((_, index) => !selected.has(index))
250-
.map((message) => message.itemId),
331+
messages: selectedMessages,
332+
omittedItemIds,
251333
context: contextFor(selected.size),
252334
omittedItems: (input.omittedItems ?? 0) + messages.length - selected.size,
253335
};

0 commit comments

Comments
 (0)