Skip to content

Commit 1c51029

Browse files
committed
fix(zen): stop double-counting reasoning_tokens in oa-compat usage (anomalyco#24268)
The OpenAI chat-completions usage spec says `completion_tokens` already includes `completion_tokens_details.reasoning_tokens`. Zen's downstream `calculateCost` bills `outputCost + reasoningCost` separately, so when the oa-compat normalizer reported `outputTokens = completion_tokens` and `reasoningTokens = reasoning_tokens`, reasoning was billed twice. Mirror the OpenAI Responses helper (openai.ts) and subtract reasoning from completion before returning. Clamp at 0 because some providers (e.g. Moonshot Kimi K2.6) report `reasoning_tokens > completion_tokens`. Adds unit tests for the reporter's exact payloads: - Kimi K2.6 "Hi": prompt 22 / completion 77 / reasoning 78 -> output 0 - Real session: prompt N / completion 1226 / reasoning 790 -> output 436 - No-reasoning case: outputTokens unchanged - Parity with `openaiHelper.normalizeUsage` for the same logical usage
1 parent 5a5a2e5 commit 1c51029

2 files changed

Lines changed: 91 additions & 3 deletions

File tree

‎packages/console/app/src/routes/zen/util/provider/openai-compatible.ts‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -61,8 +61,16 @@ export const oaCompatHelper: ProviderHelper = ({ adjustCacheUsage, safetyIdentif
6161
},
6262
normalizeUsage: (usage: Usage) => {
6363
let inputTokens = usage.prompt_tokens ?? 0
64-
const outputTokens = usage.completion_tokens ?? 0
65-
const reasoningTokens = usage.completion_tokens_details?.reasoning_tokens ?? undefined
64+
const completionTokens = usage.completion_tokens ?? 0
65+
const reasoningTokensRaw = usage.completion_tokens_details?.reasoning_tokens
66+
// Per OpenAI chat-completions spec, completion_tokens already includes reasoning_tokens.
67+
// Downstream cost calculation bills outputCost + reasoningCost separately, so we must
68+
// subtract here to avoid double-counting reasoning. Some providers (e.g. Moonshot Kimi
69+
// K2.6) report reasoning_tokens > completion_tokens; in that case clamp reasoning down
70+
// to completion so the invariant `outputTokens + reasoningTokens === completion_tokens`
71+
// holds and we charge the same total the upstream API billed (no over-charge).
72+
const reasoningTokens =
73+
reasoningTokensRaw !== undefined ? Math.min(reasoningTokensRaw, completionTokens) : undefined
6674
let cacheReadTokens = usage.cached_tokens ?? usage.prompt_tokens_details?.cached_tokens ?? undefined
6775
const cacheWriteTokens = usage.prompt_tokens_details?.cache_creation_input_tokens ?? undefined
6876

@@ -72,7 +80,7 @@ export const oaCompatHelper: ProviderHelper = ({ adjustCacheUsage, safetyIdentif
7280

7381
return {
7482
inputTokens: inputTokens - (cacheReadTokens ?? 0),
75-
outputTokens,
83+
outputTokens: completionTokens - (reasoningTokens ?? 0),
7684
reasoningTokens,
7785
cacheReadTokens,
7886
cacheWrite5mTokens: cacheWriteTokens,
Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
import { describe, expect, test } from "bun:test"
2+
import { oaCompatHelper } from "../src/routes/zen/util/provider/openai-compatible"
3+
import { openaiHelper } from "../src/routes/zen/util/provider/openai"
4+
5+
const helper = (h: ReturnType<typeof oaCompatHelper>) => h
6+
const ctx = { reqModel: "kimi-k2.6", providerModel: "moonshotai/kimi-k2.6-20260420" }
7+
8+
describe("oaCompatHelper.normalizeUsage (#24268)", () => {
9+
test("subtracts reasoning_tokens from completion_tokens so billing does not double-count", () => {
10+
const h = helper(oaCompatHelper(ctx))
11+
12+
const usage = {
13+
prompt_tokens: 22,
14+
completion_tokens: 1226,
15+
total_tokens: 1248,
16+
completion_tokens_details: { reasoning_tokens: 790 },
17+
}
18+
19+
const result = h.normalizeUsage(usage)
20+
21+
expect(result.outputTokens).toBe(436)
22+
expect(result.reasoningTokens).toBe(790)
23+
expect(result.outputTokens + (result.reasoningTokens ?? 0)).toBe(1226)
24+
})
25+
26+
test("clamps reasoning to completion when reasoning_tokens > completion_tokens (reporter's 'Hi' example)", () => {
27+
const h = helper(oaCompatHelper(ctx))
28+
29+
const usage = {
30+
prompt_tokens: 22,
31+
completion_tokens: 77,
32+
total_tokens: 99,
33+
completion_tokens_details: { reasoning_tokens: 78 },
34+
}
35+
36+
const result = h.normalizeUsage(usage)
37+
38+
// outputTokens floors at 0; reasoningTokens is clamped to completion_tokens so the
39+
// invariant `outputTokens + reasoningTokens === completion_tokens` holds and we bill
40+
// exactly what the upstream API billed (no over-charge of the extra reasoning unit).
41+
expect(result.outputTokens).toBe(0)
42+
expect(result.reasoningTokens).toBe(77)
43+
expect(result.outputTokens + (result.reasoningTokens ?? 0)).toBe(77)
44+
})
45+
46+
test("leaves outputTokens unchanged when no reasoning_tokens are reported", () => {
47+
const h = helper(oaCompatHelper(ctx))
48+
49+
const usage = {
50+
prompt_tokens: 22,
51+
completion_tokens: 77,
52+
total_tokens: 99,
53+
}
54+
55+
const result = h.normalizeUsage(usage)
56+
57+
expect(result.outputTokens).toBe(77)
58+
expect(result.reasoningTokens).toBeUndefined()
59+
})
60+
61+
test("matches OpenAI Responses helper convention for the same logical usage", () => {
62+
const compat = helper(oaCompatHelper(ctx))
63+
const responses = openaiHelper(ctx)
64+
65+
const compatResult = compat.normalizeUsage({
66+
prompt_tokens: 22,
67+
completion_tokens: 1226,
68+
completion_tokens_details: { reasoning_tokens: 790 },
69+
})
70+
const responsesResult = responses.normalizeUsage({
71+
input_tokens: 22,
72+
output_tokens: 1226,
73+
output_tokens_details: { reasoning_tokens: 790 },
74+
})
75+
76+
expect(compatResult.outputTokens).toBe(responsesResult.outputTokens)
77+
expect(compatResult.reasoningTokens).toBe(responsesResult.reasoningTokens)
78+
expect(compatResult.inputTokens).toBe(responsesResult.inputTokens)
79+
})
80+
})

0 commit comments

Comments
 (0)