Skip to content

Commit b53cd67

Browse files
fix(server): handoff budget estimates stored history in tokens, not bytes (#34926)
1 parent e2f1a21 commit b53cd67

3 files changed

Lines changed: 109 additions & 3 deletions

File tree

‎apps/server/src/orchestration-v2/ProviderTurnStartService.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ import {
4545
handoffTokenCapConfig,
4646
handoffBudget,
4747
attachmentTokenAllowance,
48+
estimateTextTokens,
4849
contextUsageForHandoff,
4950
historicalMessage,
5051
latestNativeContextUsage,
@@ -998,7 +999,7 @@ export const layer: Layer.Layer<
998999
: 0;
9991000
return (
10001001
sum +
1001-
(historical === null ? 0 : Buffer.byteLength(historical.text)) +
1002+
(historical === null ? 0 : estimateTextTokens(historical.text)) +
10021003
nativeAttachments
10031004
);
10041005
}, 0)

‎apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts‎

Lines changed: 101 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -510,7 +510,7 @@ describe("orchestration v2 provider switching", () => {
510510
capabilities: ClaudeProviderCapabilitiesV2,
511511
modelSelection: CLAUDE_MODEL_SELECTION,
512512
responseByRunOrdinal: scenario.includes("target-estimate")
513-
? { 2: "x".repeat(10_000) }
513+
? { 2: "x".repeat(64_000) }
514514
: {},
515515
capturedTurns,
516516
...(scenario.includes("retry") || scenario.includes("unsent") ? { failStartOnce } : {}),
@@ -1230,6 +1230,106 @@ describe("orchestration v2 provider switching", () => {
12301230
),
12311231
);
12321232

1233+
it.live("keeps a native session whose stored prose is far below an unreported window", () =>
1234+
Effect.scoped(
1235+
Effect.gen(function* () {
1236+
const cwd = yield* checkpointWorkspace("handoff-estimate-headroom");
1237+
const capturedTurns = yield* Ref.make<ReadonlyArray<CapturedTurn>>([]);
1238+
const generation = yield* Ref.make(0);
1239+
// About 112 KB of ordinary English is about 25k tokens: a quarter of a 128k window.
1240+
const prose = "The quick brown fox jumps over the lazy dog. ".repeat(2_500);
1241+
const layerRegistry = ProviderAdapterRegistry.layerFromAdapters([
1242+
makeTestAdapter({
1243+
instanceId: CODEX_MODEL_SELECTION.instanceId,
1244+
driver: CODEX_DRIVER,
1245+
capabilities: CodexProviderCapabilitiesV2,
1246+
modelSelection: CODEX_MODEL_SELECTION,
1247+
responseByRunOrdinal: { 2: "Source work" },
1248+
capturedTurns,
1249+
}),
1250+
// No window and no usage, like an ACP registry agent that sends no usage_update.
1251+
makeTestAdapter({
1252+
instanceId: CLAUDE_MODEL_SELECTION.instanceId,
1253+
driver: CLAUDE_DRIVER,
1254+
capabilities: ClaudeProviderCapabilitiesV2,
1255+
modelSelection: CLAUDE_MODEL_SELECTION,
1256+
responseByRunOrdinal: { 1: prose },
1257+
capturedTurns,
1258+
nativeThreadGeneration: generation,
1259+
}),
1260+
]);
1261+
yield* Effect.gen(function* () {
1262+
const orchestrator = yield* Orchestrator.OrchestratorV2;
1263+
const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
1264+
const dispatch = (ordinal: number, text: string, selection: ModelSelection) =>
1265+
orchestrator.dispatch({
1266+
type: "message.dispatch",
1267+
commandId: CommandId.make(`headroom:${ordinal}`),
1268+
threadId,
1269+
messageId: MessageId.make(`headroom:${ordinal}`),
1270+
createdBy: "user",
1271+
creationSource: "web",
1272+
text,
1273+
attachments: [],
1274+
modelSelection: selection,
1275+
dispatchMode: { type: "start_immediately" },
1276+
});
1277+
const wait = (ordinal: number) =>
1278+
orchestrator.streamStoredEvents.pipe(
1279+
Stream.filter(
1280+
({ event }) =>
1281+
event.type === "run.updated" &&
1282+
event.payload.ordinal === ordinal &&
1283+
(event.payload.status === "completed" || event.payload.status === "failed"),
1284+
),
1285+
Stream.runHead,
1286+
Effect.andThen(worker.drain()),
1287+
);
1288+
yield* orchestrator.dispatch({
1289+
type: "thread.create",
1290+
commandId: CommandId.make("headroom:create"),
1291+
threadId,
1292+
projectId,
1293+
createdBy: "user",
1294+
creationSource: "web",
1295+
title: "Handoff estimate headroom",
1296+
modelSelection: CLAUDE_MODEL_SELECTION,
1297+
runtimeMode: "full-access",
1298+
interactionMode: "default",
1299+
branch: null,
1300+
worktreePath: null,
1301+
});
1302+
yield* dispatch(1, "Write the long report", CLAUDE_MODEL_SELECTION);
1303+
yield* wait(1);
1304+
yield* dispatch(2, "Continue elsewhere", CODEX_MODEL_SELECTION);
1305+
yield* wait(2);
1306+
yield* dispatch(3, "Back to the first provider", CLAUDE_MODEL_SELECTION);
1307+
yield* wait(3);
1308+
assert.equal(
1309+
(yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)?.status,
1310+
"completed",
1311+
);
1312+
// A second generation means T3 discarded the native session as full.
1313+
assert.equal(yield* Ref.get(generation), 1);
1314+
}).pipe(
1315+
Effect.provide(
1316+
ProviderReplayHarness.layerWithRegistry(
1317+
{
1318+
name: "handoff-estimate-headroom",
1319+
runtimePolicyOverride: {
1320+
cwd,
1321+
approvalPolicy: "never",
1322+
sandboxPolicy: { type: "readOnly" },
1323+
},
1324+
},
1325+
layerRegistry,
1326+
),
1327+
),
1328+
);
1329+
}),
1330+
),
1331+
);
1332+
12331333
it.live.each(["native", "legacy"] as const)(
12341334
"starts a replacement native thread as new after the resume fallback with %s attempts",
12351335
(attempts) =>

‎packages/provider-core/src/server/handoffBudget.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,12 @@ export function attachmentTokenAllowance(attachments: ReadonlyArray<ChatAttachme
9595
);
9696
}
9797

98-
// One UTF-8 byte per token is a pessimistic estimate, not a guarantee for custom models.
98+
// Stored history is mostly prose and tool output, about four UTF-8 bytes per token.
99+
export function estimateTextTokens(text: string): number {
100+
return Math.ceil(Buffer.byteLength(text) / 4);
101+
}
102+
103+
// One UTF-8 byte per token keeps the new turn and imported history pessimistic.
99104
export function handoffBudget(input: {
100105
readonly tokenCap: number;
101106
readonly userText: string;

0 commit comments

Comments
 (0)