Skip to content

Commit 21d27b4

Browse files
committed
fix(usage): cap token estimates at the model context window
Port codex-router PR #140: a request the provider answered cannot have exceeded its context window, so the token estimate must never claim it did. estimateTokens gains an optional contextWindow cap. The request log caps the ESTIMATE field (attempt.inputTokenEstimate and the usage-fallback inputTokens) at the routed model's window while the combined inputTokens field keeps its max(reported, estimate) behavior; provider-reported positive counts are never reduced. Kiro auto and unknown adapters/models stay uncapped - a window is never invented. Window sources: KIRO_MODEL_CONTEXT_WINDOWS (src/providers/kiro-models.ts, mirroring adapters/kiro.ts kiroUpstreamContextWindow) and inferCursorContextWindow (src/adapters/cursor/discovery.ts), picked by route adapter in request-log.ts contextWindowForModel. Tests: extended tests/token-estimate.test.ts, added tests/request-log-estimate-cap.test.ts, updated the one request-log test that asserted an above-window estimate.
1 parent 8b1c620 commit 21d27b4

5 files changed

Lines changed: 200 additions & 7 deletions

File tree

‎src/lib/token-estimate.ts‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,15 +55,32 @@ function cjkRatio(text: string): number {
5555
return sampled === 0 ? 0 : cjk / sampled;
5656
}
5757

58+
/**
59+
* Cap an estimated token count at a model's context window (codex-router PR #140).
60+
*
61+
* A request the provider answered cannot have exceeded the window, so the estimate is never
62+
* allowed to claim it did: the estimate only ever substitutes a reported-zero (or missing)
63+
* input count, and a value above the window would be provably false. Positive or missing
64+
* reported counts are never rewritten here — this bounds the heuristic estimate itself.
65+
* Non-positive or non-integer windows (unknown model) leave the estimate untouched.
66+
*/
67+
export function capEstimateAtContextWindow(estimate: number, contextWindow: number | undefined): number {
68+
return typeof contextWindow === "number" && Number.isInteger(contextWindow) && contextWindow > 0
69+
? Math.min(estimate, contextWindow)
70+
: estimate;
71+
}
72+
5873
/**
5974
* Estimate the token count of a text blob. Pure and deterministic.
6075
* Returns 0 for empty/whitespace-free-empty input; otherwise ceil(length / ratio), min 1.
76+
* When `contextWindow` is a positive integer the estimate is capped at it: a request the
77+
* provider answered cannot have exceeded the window, so the estimate must not claim it did.
6178
*/
62-
export function estimateTokens(text: string, modelId?: string): number {
79+
export function estimateTokens(text: string, modelId?: string, contextWindow?: number): number {
6380
if (!text) return 0;
6481
const len = text.length;
6582
if (len === 0) return 0;
6683
let ratio = charsPerToken(modelId);
6784
if (cjkRatio(text) > CJK_RATIO_THRESHOLD) ratio = Math.min(ratio, CJK_CHARS_PER_TOKEN);
68-
return Math.max(1, Math.ceil(len / ratio));
85+
return capEstimateAtContextWindow(Math.max(1, Math.ceil(len / ratio)), contextWindow);
6986
}

‎src/server/request-log.ts‎

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,10 @@ import {
3838
} from "../usage/debug";
3939
import { matchesLogConversationId } from "./request-log-conversation";
4040
import { enforceAppOwnedMemoryBudget, type RetainedStoreSnapshot } from "../lib/app-owned-memory";
41+
import { capEstimateAtContextWindow } from "../lib/token-estimate";
42+
import { inferCursorContextWindow } from "../adapters/cursor/discovery";
43+
import { KIRO_MODEL_CONTEXT_WINDOWS, normalizeKiroModelId } from "../providers/kiro-models";
44+
import { modelRecordValue } from "../reasoning-effort";
4145

4246
export interface RequestLogContext {
4347
model: string;
@@ -839,6 +843,7 @@ export function addFinalRequestLog(
839843
logCtx.providerAdapter ?? logCtx.provider,
840844
logCtx.usage,
841845
logCtx.usageLogInputTokens,
846+
contextWindowForModel(logCtx.providerAdapter ?? logCtx.provider, logCtx.model),
842847
);
843848
const attempts = logCtx.attempts?.map(attempt => ({
844849
...attempt,
@@ -960,15 +965,40 @@ interface FinalizedUsageResult {
960965
totalTokens?: number;
961966
}
962967

968+
/**
969+
* Context window for the routed model, used to cap the token estimate (codex-router PR #140):
970+
* a request the provider answered cannot have exceeded the window, so the estimate must never
971+
* claim it did. The family is picked by the route ADAPTER, not the model id alone, because
972+
* claude-family ids are shared between Kiro and Cursor with different windows. Kiro "auto" is
973+
* a router with no fixed window and is never guessed; unknown adapters/models stay uncapped.
974+
*/
975+
function contextWindowForModel(adapter: string, modelId: string | undefined): number | undefined {
976+
if (!modelId) return undefined;
977+
if (adapter === "kiro" || adapter.startsWith("kiro-")) {
978+
const normalized = normalizeKiroModelId(modelId);
979+
if (normalized === "auto") return undefined;
980+
return modelRecordValue(KIRO_MODEL_CONTEXT_WINDOWS, modelId)
981+
?? modelRecordValue(KIRO_MODEL_CONTEXT_WINDOWS, normalized);
982+
}
983+
if (adapter === "cursor" || adapter.startsWith("cursor-")) {
984+
return inferCursorContextWindow(modelId);
985+
}
986+
return undefined;
987+
}
988+
963989
function finalizedUsage(
964990
adapter: string,
965991
usage: OcxUsage | undefined,
966992
inputTokenEstimate: number | undefined,
993+
contextWindow: number | undefined,
967994
): FinalizedUsageResult {
995+
// The ESTIMATE itself is capped at the model's context window (codex-router PR #140). The
996+
// combined value below keeps its max(inputTokens, estimate) behavior — a provider-reported
997+
// positive count is never reduced by this cap, only the estimate that could substitute it.
968998
const estimate = typeof inputTokenEstimate === "number"
969999
&& Number.isFinite(inputTokenEstimate)
9701000
&& inputTokenEstimate >= 0
971-
? inputTokenEstimate
1001+
? capEstimateAtContextWindow(inputTokenEstimate, contextWindow)
9721002
: undefined;
9731003
const finalUsage = usageForFinalLog(adapter, usage);
9741004
const usageFallback = !finalUsage && estimate !== undefined
@@ -1030,7 +1060,12 @@ export function noteAttemptSend(
10301060
if (typeof inputTokenEstimate === "number"
10311061
&& Number.isFinite(inputTokenEstimate)
10321062
&& inputTokenEstimate >= 0) {
1033-
attempt.inputTokenEstimate = inputTokenEstimate;
1063+
// Store the ESTIMATE field already capped at the model's window (codex-router PR #140):
1064+
// what gets persisted, and later merged into usage, never claims a count above the window.
1065+
attempt.inputTokenEstimate = capEstimateAtContextWindow(
1066+
inputTokenEstimate,
1067+
contextWindowForModel(attempt.adapter, attempt.model),
1068+
);
10341069
}
10351070
if (recovery && !attempt.recoveryKinds.includes(recovery)) {
10361071
attempt.recoveryKinds.push(recovery);
@@ -1047,6 +1082,7 @@ export function finishRequestAttempt(
10471082
attempt.adapter,
10481083
usage ?? attempt.usage,
10491084
attempt.inputTokenEstimate,
1085+
contextWindowForModel(attempt.adapter, attempt.model),
10501086
);
10511087
attempt.status = status;
10521088
attempt.durationMs = Math.max(0, durationMs);
Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
import { describe, expect, test } from "bun:test";
2+
import {
3+
beginRequestAttempt,
4+
finishRequestAttempt,
5+
noteAttemptSend,
6+
} from "../src/server/request-log";
7+
8+
/**
9+
* The token-estimate context-window cap (codex-router PR #140) in the request log:
10+
* the ESTIMATE field is bounded by the routed model's context window, while the combined
11+
* inputTokens field keeps its max(reported, estimate) behavior. Provider-reported positive
12+
* counts are never reduced by the cap.
13+
*/
14+
describe("request log token-estimate context-window cap (codex-router PR #140)", () => {
15+
const DEEPSEEK_WINDOW = 128_000; // KIRO_MODEL_CONTEXT_WINDOWS["deepseek-3.2"]
16+
const CURSOR_CLAUDE_WINDOW = 200_000; // inferCursorContextWindow("claude-4.6-opus-high")
17+
18+
test("the estimate is capped at the kiro model's context window", () => {
19+
const attempt = beginRequestAttempt(1, "kiro", "deepseek-3.2", "kiro");
20+
noteAttemptSend(attempt, 200_000);
21+
finishRequestAttempt(attempt, 200, 10);
22+
expect(attempt.inputTokenEstimate).toBe(DEEPSEEK_WINDOW);
23+
expect(attempt.usage).toEqual({ inputTokens: DEEPSEEK_WINDOW, outputTokens: 0, estimated: true });
24+
});
25+
26+
test("below-window estimates pass through unchanged", () => {
27+
const attempt = beginRequestAttempt(1, "kiro", "deepseek-3.2", "kiro");
28+
noteAttemptSend(attempt, 100_000);
29+
finishRequestAttempt(attempt, 200, 10);
30+
expect(attempt.inputTokenEstimate).toBe(100_000);
31+
expect(attempt.usage?.inputTokens).toBe(100_000);
32+
});
33+
34+
test("a positive provider-reported input count is never reduced by the cap", () => {
35+
const attempt = beginRequestAttempt(1, "kiro", "deepseek-3.2", "kiro");
36+
noteAttemptSend(attempt, 200_000); // estimate would exceed the window
37+
finishRequestAttempt(attempt, 200, 10, { inputTokens: 150_000, outputTokens: 500 });
38+
// combined = max(reported 150k, capped estimate 128k) = 150k: the reported count wins.
39+
expect(attempt.usage?.inputTokens).toBe(150_000);
40+
});
41+
42+
test("a missing report substitutes the capped estimate (the ESTIMATE field)", () => {
43+
const attempt = beginRequestAttempt(1, "kiro", "deepseek-3.2", "kiro");
44+
noteAttemptSend(attempt, 200_000);
45+
finishRequestAttempt(attempt, 200, 10);
46+
// No usage at all: the fallback inputTokens IS the capped estimate — never above the window.
47+
expect(attempt.usage?.inputTokens).toBe(DEEPSEEK_WINDOW);
48+
expect(attempt.usage?.estimated).toBe(true);
49+
expect(attempt.usageStatus).toBe("estimated");
50+
});
51+
52+
test("the ESTIMATE field vs the combined inputTokens field distinction is preserved", () => {
53+
// Reported zero: combined max(0, capped estimate) equals the capped estimate.
54+
const zero = beginRequestAttempt(1, "kiro", "deepseek-3.2", "kiro");
55+
noteAttemptSend(zero, 200_000);
56+
finishRequestAttempt(zero, 200, 10, { inputTokens: 0, outputTokens: 5 });
57+
expect(zero.usage?.inputTokens).toBe(DEEPSEEK_WINDOW);
58+
59+
// Reported positive above the capped estimate: combined keeps the reported count.
60+
const positive = beginRequestAttempt(1, "kiro", "deepseek-3.2", "kiro");
61+
noteAttemptSend(positive, 200_000);
62+
finishRequestAttempt(positive, 200, 10, { inputTokens: 200_000, outputTokens: 5 });
63+
expect(positive.usage?.inputTokens).toBe(200_000);
64+
65+
// Reported positive below the capped estimate: combined takes the capped estimate.
66+
const middle = beginRequestAttempt(1, "kiro", "deepseek-3.2", "kiro");
67+
noteAttemptSend(middle, 100_000);
68+
finishRequestAttempt(middle, 200, 10, { inputTokens: 50_000, outputTokens: 5 });
69+
expect(middle.usage?.inputTokens).toBe(100_000);
70+
});
71+
72+
test("cursor adapter uses the cursor window inference", () => {
73+
const attempt = beginRequestAttempt(1, "cursor", "claude-4.6-opus-high", "cursor");
74+
noteAttemptSend(attempt, 500_000);
75+
finishRequestAttempt(attempt, 200, 10);
76+
expect(attempt.inputTokenEstimate).toBe(CURSOR_CLAUDE_WINDOW);
77+
});
78+
79+
test("unknown adapters stay uncapped (a window is never invented)", () => {
80+
const attempt = beginRequestAttempt(1, "anthropic", "claude-sonnet-4-6", "anthropic");
81+
noteAttemptSend(attempt, 500_000);
82+
finishRequestAttempt(attempt, 200, 10);
83+
expect(attempt.inputTokenEstimate).toBe(500_000);
84+
});
85+
86+
test("without a window the existing max(reported, estimate) behavior is unchanged", () => {
87+
const attempt = beginRequestAttempt(1, "anthropic", "claude-sonnet-4-6", "anthropic");
88+
noteAttemptSend(attempt, 500_000);
89+
finishRequestAttempt(attempt, 200, 10, { inputTokens: 100_000, outputTokens: 5 });
90+
expect(attempt.usage?.inputTokens).toBe(500_000);
91+
});
92+
93+
test("kiro auto has no fixed window and is never guessed", () => {
94+
const attempt = beginRequestAttempt(1, "kiro", "kiro-auto", "kiro");
95+
noteAttemptSend(attempt, 500_000);
96+
finishRequestAttempt(attempt, 200, 10);
97+
expect(attempt.inputTokenEstimate).toBe(500_000);
98+
});
99+
});

‎tests/request-log.test.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1053,10 +1053,12 @@ describe("request log metadata", () => {
10531053
const text = await response.text();
10541054
expect(text).toContain("\"input_tokens\":9");
10551055
expect(entries).toHaveLength(1);
1056+
// The 240k estimate exceeds claude-sonnet-4.5's 200k window; a request the provider
1057+
// answered cannot have exceeded the window, so the estimate is capped (codex-router PR #140).
10561058
expect(entries[0]).toMatchObject({
10571059
usageStatus: "estimated",
1058-
totalTokens: 240_004,
1059-
usage: { inputTokens: 240_000, outputTokens: 4, estimated: true },
1060+
totalTokens: 200_004,
1061+
usage: { inputTokens: 200_000, outputTokens: 4, estimated: true },
10601062
});
10611063
});
10621064

‎tests/token-estimate.test.ts‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { describe, expect, test } from "bun:test";
2-
import { charsPerToken, estimateTokens } from "../src/lib/token-estimate";
2+
import { capEstimateAtContextWindow, charsPerToken, estimateTokens } from "../src/lib/token-estimate";
33

44
describe("CJK-aware ratio (devlog 260712 B3)", () => {
55
const korean = "한국어 텍스트는 토큰 밀도가 높아서 영어 기준 추정이 과소계산됩니다 ".repeat(10);
@@ -61,3 +61,42 @@ describe("token-estimate sidecar", () => {
6161
}
6262
});
6363
});
64+
65+
describe("context-window cap (codex-router PR #140)", () => {
66+
const GPT56_SOL_WINDOW = 272_000;
67+
const DEEPSEEK_WINDOW = 128_000;
68+
// gpt-5.6-sol uses the generic 4-char ratio: 1.2M chars ~= 300k tokens, far over 272k.
69+
const OVER_WINDOW_TEXT = "x".repeat(1_200_000);
70+
71+
test("estimate is capped at the model's context window", () => {
72+
expect(estimateTokens(OVER_WINDOW_TEXT, "gpt-5.6-sol", GPT56_SOL_WINDOW)).toBe(GPT56_SOL_WINDOW);
73+
expect(estimateTokens(OVER_WINDOW_TEXT, "gpt-5.6-sol", DEEPSEEK_WINDOW)).toBe(DEEPSEEK_WINDOW);
74+
});
75+
76+
test("below-window estimates are unchanged", () => {
77+
const text = "x".repeat(100_000); // ~28.5k tokens at the kiro ratio
78+
const plain = estimateTokens(text, "gpt-5.6-sol");
79+
expect(plain).toBeLessThan(GPT56_SOL_WINDOW);
80+
expect(estimateTokens(text, "gpt-5.6-sol", GPT56_SOL_WINDOW)).toBe(plain);
81+
});
82+
83+
test("only a positive integer window caps (unknown window leaves the estimate untouched)", () => {
84+
const text = "x".repeat(100_000);
85+
const plain = estimateTokens(text, "gpt-5.6-sol");
86+
for (const window of [undefined, 0, -1, 1.5, Number.NaN]) {
87+
expect(estimateTokens(text, "gpt-5.6-sol", window)).toBe(plain);
88+
}
89+
});
90+
91+
test("the min-1 floor survives the cap for tiny inputs", () => {
92+
expect(estimateTokens("a", "claude-opus-4.8", 1)).toBe(1);
93+
expect(estimateTokens("a", "claude-opus-4.8", GPT56_SOL_WINDOW)).toBe(1);
94+
});
95+
96+
test("capEstimateAtContextWindow caps only at positive integer windows", () => {
97+
expect(capEstimateAtContextWindow(300_000, GPT56_SOL_WINDOW)).toBe(GPT56_SOL_WINDOW);
98+
expect(capEstimateAtContextWindow(100_000, GPT56_SOL_WINDOW)).toBe(100_000);
99+
expect(capEstimateAtContextWindow(300_000, undefined)).toBe(300_000);
100+
expect(capEstimateAtContextWindow(300_000, 0)).toBe(300_000);
101+
});
102+
});

0 commit comments

Comments
 (0)