Skip to content

Commit e0d2725

Browse files
PierrunoYTampagentclaude
authored
fix(openai): count compatible cache reads (#1662)
* fix(openai): count compatible cache reads Amp-Thread-ID: https://ampcode.com/threads/T-01a0aa02-b6de-7763-97b8-1650ea4b46da * fix(api): validate compatible cache usage * fix: reject fractional OpenAI cache reads and cover precedence Amp-Thread-ID: https://ampcode.com/threads/T-01a0c52c-51a6-73ae-bafb-2fb2bc840b2b * fix(openai): preserve proxy cache counters and cover nonstreaming o3 * test(openai): pin the cached_tokens upper bound when it equals prompt_tokens Co-Authored-By: Claude Sonnet 5.5 <[email protected]> --------- Co-authored-by: Amp <[email protected]> Co-authored-by: Claude Sonnet 5.5 <[email protected]>
1 parent ca05b73 commit e0d2725

3 files changed

Lines changed: 245 additions & 9 deletions

File tree

‎src/api/providers/__tests__/openai-usage-tracking.spec.ts‎

Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,31 @@ describe("OpenAiHandler with usage tracking fix", () => {
181181
})
182182
})
183183

184+
it("should report OpenAI-compatible cached prompt tokens", async () => {
185+
mockCreate.mockImplementationOnce(async () =>
186+
asyncStreamFrom([
187+
{
188+
choices: [{ delta: { content: "Cached response" }, index: 0 }],
189+
usage: {
190+
prompt_tokens: 5_053,
191+
completion_tokens: 16,
192+
total_tokens: 5_069,
193+
prompt_tokens_details: { cached_tokens: 4_864 },
194+
},
195+
},
196+
]),
197+
)
198+
199+
const chunks = await collectStream(handler.createMessage(systemPrompt, messages))
200+
201+
expect(chunks).toContainEqual({
202+
type: "usage",
203+
inputTokens: 5_053,
204+
outputTokens: 16,
205+
cacheReadTokens: 4_864,
206+
})
207+
})
208+
184209
it("should handle case where no usage is provided", async () => {
185210
// Override the mock for this specific test
186211
mockCreate.mockImplementationOnce(async (options) => {
@@ -212,4 +237,195 @@ describe("OpenAiHandler with usage tracking fix", () => {
212237
expect(usageChunks).toHaveLength(0)
213238
})
214239
})
240+
241+
it.each(["gpt-4", "o3-mini"])(
242+
"reports cached prompt tokens from a non-streaming %s response",
243+
async (openAiModelId) => {
244+
const nonStreamingHandler = new OpenAiHandler({
245+
...mockOptions,
246+
openAiModelId,
247+
openAiStreamingEnabled: false,
248+
})
249+
mockCreate.mockImplementationOnce(async () => ({
250+
id: "test-completion",
251+
choices: [{ message: { role: "assistant", content: "Cached response" } }],
252+
usage: {
253+
prompt_tokens: 4_621,
254+
completion_tokens: 16,
255+
total_tokens: 4_637,
256+
prompt_tokens_details: { cached_tokens: 4_608 },
257+
},
258+
}))
259+
260+
const chunks = await collectStream(nonStreamingHandler.createMessage("system prompt", []))
261+
262+
expect(chunks).toContainEqual({
263+
type: "usage",
264+
inputTokens: 4_621,
265+
outputTokens: 16,
266+
cacheReadTokens: 4_608,
267+
})
268+
},
269+
)
270+
271+
it("reports cached prompt tokens for a streaming O3 response", async () => {
272+
const o3Handler = new OpenAiHandler({ ...mockOptions, openAiModelId: "o3-mini" })
273+
mockCreate.mockImplementationOnce(async () =>
274+
asyncStreamFrom([
275+
{
276+
choices: [{ delta: { content: "Cached response" }, index: 0 }],
277+
usage: {
278+
prompt_tokens: 5_053,
279+
completion_tokens: 16,
280+
total_tokens: 5_069,
281+
prompt_tokens_details: { cached_tokens: 4_864 },
282+
},
283+
},
284+
]),
285+
)
286+
287+
const chunks = await collectStream(o3Handler.createMessage("system prompt", []))
288+
289+
expect(chunks).toContainEqual({
290+
type: "usage",
291+
inputTokens: 5_053,
292+
outputTokens: 16,
293+
cacheReadTokens: 4_864,
294+
})
295+
})
296+
297+
it.each([
298+
["string", "10"],
299+
["object", { tokens: 10 }],
300+
["negative", -1],
301+
["fractional", 10.5],
302+
["non-finite", Number.POSITIVE_INFINITY],
303+
["greater than prompt tokens", 101],
304+
])("ignores invalid %s cached prompt tokens in streaming responses", async (_name, cachedTokens) => {
305+
mockCreate.mockImplementationOnce(async () =>
306+
asyncStreamFrom([
307+
{
308+
choices: [{ delta: { content: "Response" }, index: 0 }],
309+
usage: {
310+
prompt_tokens: 100,
311+
completion_tokens: 5,
312+
prompt_tokens_details: { cached_tokens: cachedTokens },
313+
},
314+
},
315+
]),
316+
)
317+
318+
const chunks = await collectStream(handler.createMessage("system prompt", []))
319+
320+
expect(chunks).toContainEqual({ type: "usage", inputTokens: 100, outputTokens: 5 })
321+
})
322+
323+
it.each([
324+
["string", "10"],
325+
["object", { tokens: 10 }],
326+
["negative", -1],
327+
["fractional", 10.5],
328+
["non-finite", Number.POSITIVE_INFINITY],
329+
["greater than prompt tokens", 101],
330+
])("ignores invalid %s cached prompt tokens in non-streaming responses", async (_name, cachedTokens) => {
331+
const nonStreamingHandler = new OpenAiHandler({ ...mockOptions, openAiStreamingEnabled: false })
332+
mockCreate.mockImplementationOnce(async () => ({
333+
id: "test-completion",
334+
choices: [{ message: { role: "assistant", content: "Response" } }],
335+
usage: {
336+
prompt_tokens: 100,
337+
completion_tokens: 5,
338+
prompt_tokens_details: { cached_tokens: cachedTokens },
339+
},
340+
}))
341+
342+
const chunks = await collectStream(nonStreamingHandler.createMessage("system prompt", []))
343+
344+
expect(chunks).toContainEqual({ type: "usage", inputTokens: 100, outputTokens: 5 })
345+
})
346+
347+
describe.each([true, false])("cache accounting with streaming=%s", (openAiStreamingEnabled) => {
348+
it.each([
349+
["authoritative count", 23, 23],
350+
["authoritative zero", 0, undefined],
351+
["fully cached prompt", 100, 100],
352+
["null falls back", null, 71],
353+
["absent falls back", undefined, 71],
354+
["negative", -1, undefined],
355+
["fractional", 10.5, undefined],
356+
["infinite", Infinity, undefined],
357+
["NaN", NaN, undefined],
358+
["string", "23", undefined],
359+
["object", { tokens: 23 }, undefined],
360+
["proxy cache reads exceed uncached input", 101, 101],
361+
])("respects %s without substituting a conflicting fallback", async (_name, reported, expected) => {
362+
const cacheHandler = new OpenAiHandler({ ...mockOptions, openAiStreamingEnabled })
363+
const usage = {
364+
prompt_tokens: 100,
365+
completion_tokens: 5,
366+
cache_creation_input_tokens: 7,
367+
cache_read_input_tokens: reported,
368+
prompt_tokens_details: { cached_tokens: 71 },
369+
}
370+
mockCreate.mockResolvedValueOnce(
371+
openAiStreamingEnabled
372+
? asyncStreamFrom([{ choices: [], usage }])
373+
: { choices: [{ message: { content: "Response" } }], usage },
374+
)
375+
376+
const chunks = await collectStream(cacheHandler.createMessage("system prompt", []))
377+
378+
expect(chunks.filter((chunk) => chunk.type === "usage")).toEqual([
379+
{
380+
type: "usage",
381+
inputTokens: 100,
382+
outputTokens: 5,
383+
cacheWriteTokens: 7,
384+
cacheReadTokens: expected,
385+
},
386+
])
387+
})
388+
389+
// Only OpenAI's nested counter must be a subset of prompt_tokens; this pins the inclusive boundary.
390+
it.each([
391+
["below the prompt size", 99, 99],
392+
["equal to the prompt size", 100, 100],
393+
["above the prompt size", 101, undefined],
394+
])(
395+
"accepts a nested cached_tokens count %s with no cache_read_input_tokens",
396+
async (_name, cached, expected) => {
397+
const cacheHandler = new OpenAiHandler({ ...mockOptions, openAiStreamingEnabled })
398+
const usage = {
399+
prompt_tokens: 100,
400+
completion_tokens: 5,
401+
prompt_tokens_details: { cached_tokens: cached },
402+
}
403+
mockCreate.mockResolvedValueOnce(
404+
openAiStreamingEnabled
405+
? asyncStreamFrom([{ choices: [], usage }])
406+
: { choices: [{ message: { content: "Response" } }], usage },
407+
)
408+
409+
const chunks = await collectStream(cacheHandler.createMessage("system prompt", []))
410+
411+
expect(chunks.filter((chunk) => chunk.type === "usage")).toEqual([
412+
{ type: "usage", inputTokens: 100, outputTokens: 5, cacheReadTokens: expected },
413+
])
414+
},
415+
)
416+
})
417+
418+
it.each([null, undefined, {}, { prompt_tokens_details: null }])(
419+
"defaults missing non-streaming usage counters to zero: %j",
420+
async (usage) => {
421+
const nonStreamingHandler = new OpenAiHandler({ ...mockOptions, openAiStreamingEnabled: false })
422+
mockCreate.mockResolvedValueOnce({ choices: [{ message: { content: "Response" } }], usage })
423+
424+
const chunks = await collectStream(nonStreamingHandler.createMessage("system prompt", []))
425+
426+
expect(chunks.filter((chunk) => chunk.type === "usage")).toEqual([
427+
{ type: "usage", inputTokens: 0, outputTokens: 0 },
428+
])
429+
},
430+
)
215431
})

‎src/api/providers/openai.ts‎

Lines changed: 28 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,17 @@ import type { SingleCompletionHandler, ApiHandlerCreateMessageMetadata, Complete
3030
import { handleOpenAIError } from "./utils/error-handler"
3131
import { extractReasoningFromDelta } from "./utils/extract-reasoning"
3232

33+
type OpenAiUsage =
34+
| {
35+
prompt_tokens?: number
36+
completion_tokens?: number
37+
cache_creation_input_tokens?: number
38+
cache_read_input_tokens?: unknown
39+
prompt_tokens_details?: { cached_tokens?: unknown } | null
40+
}
41+
| null
42+
| undefined
43+
3344
// TODO: Rename this to OpenAICompatibleHandler. Also, I think the
3445
// `OpenAINativeHandler` can subclass from this, since it's obviously
3546
// compatible with the OpenAI API. We can also rename it to `OpenAIHandler`.
@@ -288,13 +299,26 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl
288299
}
289300
}
290301

291-
protected processUsageMetrics(usage: any, _modelInfo?: ModelInfo): ApiStreamUsageChunk {
302+
protected processUsageMetrics(usage: OpenAiUsage, _modelInfo?: ModelInfo): ApiStreamUsageChunk {
303+
const inputTokens = usage?.prompt_tokens || 0
304+
const reportedCacheReadTokens = usage?.cache_read_input_tokens ?? usage?.prompt_tokens_details?.cached_tokens
305+
const cacheReadTokens =
306+
typeof reportedCacheReadTokens === "number" &&
307+
Number.isFinite(reportedCacheReadTokens) &&
308+
Number.isInteger(reportedCacheReadTokens) &&
309+
reportedCacheReadTokens >= 0 &&
310+
// Proxy-specific counters may exclude cache reads from prompt_tokens.
311+
// Only OpenAI's nested cached_tokens must be a subset of prompt_tokens.
312+
(usage?.cache_read_input_tokens != null || reportedCacheReadTokens <= inputTokens)
313+
? reportedCacheReadTokens || undefined
314+
: undefined
315+
292316
return {
293317
type: "usage",
294-
inputTokens: usage?.prompt_tokens || 0,
318+
inputTokens,
295319
outputTokens: usage?.completion_tokens || 0,
296320
cacheWriteTokens: usage?.cache_creation_input_tokens || undefined,
297-
cacheReadTokens: usage?.cache_read_input_tokens || undefined,
321+
cacheReadTokens,
298322
}
299323
}
300324

@@ -499,11 +523,7 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl
499523
}
500524

501525
if (chunk.usage) {
502-
yield {
503-
type: "usage",
504-
inputTokens: chunk.usage.prompt_tokens || 0,
505-
outputTokens: chunk.usage.completion_tokens || 0,
506-
}
526+
yield this.processUsageMetrics(chunk.usage)
507527
}
508528
}
509529
}

‎src/eslint-suppressions.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -396,7 +396,7 @@
396396
},
397397
"api/providers/openai.ts": {
398398
"@typescript-eslint/no-explicit-any": {
399-
"count": 3
399+
"count": 2
400400
}
401401
},
402402
"api/providers/openrouter.ts": {

0 commit comments

Comments
 (0)