Skip to content

Commit f6eb03b

Browse files
committed
fix(openai): preserve proxy cache counters and cover nonstreaming o3
1 parent ef29ef3 commit f6eb03b

2 files changed

Lines changed: 31 additions & 22 deletions

File tree

‎src/api/providers/__tests__/openai-usage-tracking.spec.ts‎

Lines changed: 28 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -238,28 +238,35 @@ describe("OpenAiHandler with usage tracking fix", () => {
238238
})
239239
})
240240

241-
it("should report cached prompt tokens from a non-streaming response", async () => {
242-
const nonStreamingHandler = new OpenAiHandler({ ...mockOptions, openAiStreamingEnabled: false })
243-
mockCreate.mockImplementationOnce(async () => ({
244-
id: "test-completion",
245-
choices: [{ message: { role: "assistant", content: "Cached response" } }],
246-
usage: {
247-
prompt_tokens: 4_621,
248-
completion_tokens: 16,
249-
total_tokens: 4_637,
250-
prompt_tokens_details: { cached_tokens: 4_608 },
251-
},
252-
}))
241+
it.each(["gpt-4", "o3-mini"])(
242+
"reports cached prompt tokens from a non-streaming %s response",
243+
async (openAiModelId) => {
244+
const nonStreamingHandler = new OpenAiHandler({
245+
...mockOptions,
246+
openAiModelId,
247+
openAiStreamingEnabled: false,
248+
})
249+
mockCreate.mockImplementationOnce(async () => ({
250+
id: "test-completion",
251+
choices: [{ message: { role: "assistant", content: "Cached response" } }],
252+
usage: {
253+
prompt_tokens: 4_621,
254+
completion_tokens: 16,
255+
total_tokens: 4_637,
256+
prompt_tokens_details: { cached_tokens: 4_608 },
257+
},
258+
}))
253259

254-
const chunks = await collectStream(nonStreamingHandler.createMessage("system prompt", []))
260+
const chunks = await collectStream(nonStreamingHandler.createMessage("system prompt", []))
255261

256-
expect(chunks).toContainEqual({
257-
type: "usage",
258-
inputTokens: 4_621,
259-
outputTokens: 16,
260-
cacheReadTokens: 4_608,
261-
})
262-
})
262+
expect(chunks).toContainEqual({
263+
type: "usage",
264+
inputTokens: 4_621,
265+
outputTokens: 16,
266+
cacheReadTokens: 4_608,
267+
})
268+
},
269+
)
263270

264271
it("reports cached prompt tokens for a streaming O3 response", async () => {
265272
const o3Handler = new OpenAiHandler({ ...mockOptions, openAiModelId: "o3-mini" })
@@ -350,7 +357,7 @@ describe("OpenAiHandler with usage tracking fix", () => {
350357
["NaN", NaN, undefined],
351358
["string", "23", undefined],
352359
["object", { tokens: 23 }, undefined],
353-
["exceeds input", 101, undefined],
360+
["proxy cache reads exceed uncached input", 101, 101],
354361
])("respects %s without substituting a conflicting fallback", async (_name, reported, expected) => {
355362
const cacheHandler = new OpenAiHandler({ ...mockOptions, openAiStreamingEnabled })
356363
const usage = {

‎src/api/providers/openai.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -300,7 +300,9 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl
300300
Number.isFinite(reportedCacheReadTokens) &&
301301
Number.isInteger(reportedCacheReadTokens) &&
302302
reportedCacheReadTokens >= 0 &&
303-
reportedCacheReadTokens <= inputTokens
303+
// Proxy-specific counters may exclude cache reads from prompt_tokens.
304+
// Only OpenAI's nested cached_tokens must be a subset of prompt_tokens.
305+
(usage?.cache_read_input_tokens != null || reportedCacheReadTokens <= inputTokens)
304306
? reportedCacheReadTokens || undefined
305307
: undefined
306308

0 commit comments

Comments
 (0)