@@ -181,6 +181,31 @@ describe("OpenAiHandler with usage tracking fix", () => {
181181 } )
182182 } )
183183
184+ it ( "should report OpenAI-compatible cached prompt tokens" , async ( ) => {
185+ mockCreate . mockImplementationOnce ( async ( ) =>
186+ asyncStreamFrom ( [
187+ {
188+ choices : [ { delta : { content : "Cached response" } , index : 0 } ] ,
189+ usage : {
190+ prompt_tokens : 5_053 ,
191+ completion_tokens : 16 ,
192+ total_tokens : 5_069 ,
193+ prompt_tokens_details : { cached_tokens : 4_864 } ,
194+ } ,
195+ } ,
196+ ] ) ,
197+ )
198+
199+ const chunks = await collectStream ( handler . createMessage ( systemPrompt , messages ) )
200+
201+ expect ( chunks ) . toContainEqual ( {
202+ type : "usage" ,
203+ inputTokens : 5_053 ,
204+ outputTokens : 16 ,
205+ cacheReadTokens : 4_864 ,
206+ } )
207+ } )
208+
184209 it ( "should handle case where no usage is provided" , async ( ) => {
185210 // Override the mock for this specific test
186211 mockCreate . mockImplementationOnce ( async ( options ) => {
@@ -212,4 +237,195 @@ describe("OpenAiHandler with usage tracking fix", () => {
212237 expect ( usageChunks ) . toHaveLength ( 0 )
213238 } )
214239 } )
240+
241+ it . each ( [ "gpt-4" , "o3-mini" ] ) (
242+ "reports cached prompt tokens from a non-streaming %s response" ,
243+ async ( openAiModelId ) => {
244+ const nonStreamingHandler = new OpenAiHandler ( {
245+ ...mockOptions ,
246+ openAiModelId,
247+ openAiStreamingEnabled : false ,
248+ } )
249+ mockCreate . mockImplementationOnce ( async ( ) => ( {
250+ id : "test-completion" ,
251+ choices : [ { message : { role : "assistant" , content : "Cached response" } } ] ,
252+ usage : {
253+ prompt_tokens : 4_621 ,
254+ completion_tokens : 16 ,
255+ total_tokens : 4_637 ,
256+ prompt_tokens_details : { cached_tokens : 4_608 } ,
257+ } ,
258+ } ) )
259+
260+ const chunks = await collectStream ( nonStreamingHandler . createMessage ( "system prompt" , [ ] ) )
261+
262+ expect ( chunks ) . toContainEqual ( {
263+ type : "usage" ,
264+ inputTokens : 4_621 ,
265+ outputTokens : 16 ,
266+ cacheReadTokens : 4_608 ,
267+ } )
268+ } ,
269+ )
270+
271+ it ( "reports cached prompt tokens for a streaming O3 response" , async ( ) => {
272+ const o3Handler = new OpenAiHandler ( { ...mockOptions , openAiModelId : "o3-mini" } )
273+ mockCreate . mockImplementationOnce ( async ( ) =>
274+ asyncStreamFrom ( [
275+ {
276+ choices : [ { delta : { content : "Cached response" } , index : 0 } ] ,
277+ usage : {
278+ prompt_tokens : 5_053 ,
279+ completion_tokens : 16 ,
280+ total_tokens : 5_069 ,
281+ prompt_tokens_details : { cached_tokens : 4_864 } ,
282+ } ,
283+ } ,
284+ ] ) ,
285+ )
286+
287+ const chunks = await collectStream ( o3Handler . createMessage ( "system prompt" , [ ] ) )
288+
289+ expect ( chunks ) . toContainEqual ( {
290+ type : "usage" ,
291+ inputTokens : 5_053 ,
292+ outputTokens : 16 ,
293+ cacheReadTokens : 4_864 ,
294+ } )
295+ } )
296+
297+ it . each ( [
298+ [ "string" , "10" ] ,
299+ [ "object" , { tokens : 10 } ] ,
300+ [ "negative" , - 1 ] ,
301+ [ "fractional" , 10.5 ] ,
302+ [ "non-finite" , Number . POSITIVE_INFINITY ] ,
303+ [ "greater than prompt tokens" , 101 ] ,
304+ ] ) ( "ignores invalid %s cached prompt tokens in streaming responses" , async ( _name , cachedTokens ) => {
305+ mockCreate . mockImplementationOnce ( async ( ) =>
306+ asyncStreamFrom ( [
307+ {
308+ choices : [ { delta : { content : "Response" } , index : 0 } ] ,
309+ usage : {
310+ prompt_tokens : 100 ,
311+ completion_tokens : 5 ,
312+ prompt_tokens_details : { cached_tokens : cachedTokens } ,
313+ } ,
314+ } ,
315+ ] ) ,
316+ )
317+
318+ const chunks = await collectStream ( handler . createMessage ( "system prompt" , [ ] ) )
319+
320+ expect ( chunks ) . toContainEqual ( { type : "usage" , inputTokens : 100 , outputTokens : 5 } )
321+ } )
322+
323+ it . each ( [
324+ [ "string" , "10" ] ,
325+ [ "object" , { tokens : 10 } ] ,
326+ [ "negative" , - 1 ] ,
327+ [ "fractional" , 10.5 ] ,
328+ [ "non-finite" , Number . POSITIVE_INFINITY ] ,
329+ [ "greater than prompt tokens" , 101 ] ,
330+ ] ) ( "ignores invalid %s cached prompt tokens in non-streaming responses" , async ( _name , cachedTokens ) => {
331+ const nonStreamingHandler = new OpenAiHandler ( { ...mockOptions , openAiStreamingEnabled : false } )
332+ mockCreate . mockImplementationOnce ( async ( ) => ( {
333+ id : "test-completion" ,
334+ choices : [ { message : { role : "assistant" , content : "Response" } } ] ,
335+ usage : {
336+ prompt_tokens : 100 ,
337+ completion_tokens : 5 ,
338+ prompt_tokens_details : { cached_tokens : cachedTokens } ,
339+ } ,
340+ } ) )
341+
342+ const chunks = await collectStream ( nonStreamingHandler . createMessage ( "system prompt" , [ ] ) )
343+
344+ expect ( chunks ) . toContainEqual ( { type : "usage" , inputTokens : 100 , outputTokens : 5 } )
345+ } )
346+
347+ describe . each ( [ true , false ] ) ( "cache accounting with streaming=%s" , ( openAiStreamingEnabled ) => {
348+ it . each ( [
349+ [ "authoritative count" , 23 , 23 ] ,
350+ [ "authoritative zero" , 0 , undefined ] ,
351+ [ "fully cached prompt" , 100 , 100 ] ,
352+ [ "null falls back" , null , 71 ] ,
353+ [ "absent falls back" , undefined , 71 ] ,
354+ [ "negative" , - 1 , undefined ] ,
355+ [ "fractional" , 10.5 , undefined ] ,
356+ [ "infinite" , Infinity , undefined ] ,
357+ [ "NaN" , NaN , undefined ] ,
358+ [ "string" , "23" , undefined ] ,
359+ [ "object" , { tokens : 23 } , undefined ] ,
360+ [ "proxy cache reads exceed uncached input" , 101 , 101 ] ,
361+ ] ) ( "respects %s without substituting a conflicting fallback" , async ( _name , reported , expected ) => {
362+ const cacheHandler = new OpenAiHandler ( { ...mockOptions , openAiStreamingEnabled } )
363+ const usage = {
364+ prompt_tokens : 100 ,
365+ completion_tokens : 5 ,
366+ cache_creation_input_tokens : 7 ,
367+ cache_read_input_tokens : reported ,
368+ prompt_tokens_details : { cached_tokens : 71 } ,
369+ }
370+ mockCreate . mockResolvedValueOnce (
371+ openAiStreamingEnabled
372+ ? asyncStreamFrom ( [ { choices : [ ] , usage } ] )
373+ : { choices : [ { message : { content : "Response" } } ] , usage } ,
374+ )
375+
376+ const chunks = await collectStream ( cacheHandler . createMessage ( "system prompt" , [ ] ) )
377+
378+ expect ( chunks . filter ( ( chunk ) => chunk . type === "usage" ) ) . toEqual ( [
379+ {
380+ type : "usage" ,
381+ inputTokens : 100 ,
382+ outputTokens : 5 ,
383+ cacheWriteTokens : 7 ,
384+ cacheReadTokens : expected ,
385+ } ,
386+ ] )
387+ } )
388+
389+ // Only OpenAI's nested counter must be a subset of prompt_tokens; this pins the inclusive boundary.
390+ it . each ( [
391+ [ "below the prompt size" , 99 , 99 ] ,
392+ [ "equal to the prompt size" , 100 , 100 ] ,
393+ [ "above the prompt size" , 101 , undefined ] ,
394+ ] ) (
395+ "accepts a nested cached_tokens count %s with no cache_read_input_tokens" ,
396+ async ( _name , cached , expected ) => {
397+ const cacheHandler = new OpenAiHandler ( { ...mockOptions , openAiStreamingEnabled } )
398+ const usage = {
399+ prompt_tokens : 100 ,
400+ completion_tokens : 5 ,
401+ prompt_tokens_details : { cached_tokens : cached } ,
402+ }
403+ mockCreate . mockResolvedValueOnce (
404+ openAiStreamingEnabled
405+ ? asyncStreamFrom ( [ { choices : [ ] , usage } ] )
406+ : { choices : [ { message : { content : "Response" } } ] , usage } ,
407+ )
408+
409+ const chunks = await collectStream ( cacheHandler . createMessage ( "system prompt" , [ ] ) )
410+
411+ expect ( chunks . filter ( ( chunk ) => chunk . type === "usage" ) ) . toEqual ( [
412+ { type : "usage" , inputTokens : 100 , outputTokens : 5 , cacheReadTokens : expected } ,
413+ ] )
414+ } ,
415+ )
416+ } )
417+
418+ it . each ( [ null , undefined , { } , { prompt_tokens_details : null } ] ) (
419+ "defaults missing non-streaming usage counters to zero: %j" ,
420+ async ( usage ) => {
421+ const nonStreamingHandler = new OpenAiHandler ( { ...mockOptions , openAiStreamingEnabled : false } )
422+ mockCreate . mockResolvedValueOnce ( { choices : [ { message : { content : "Response" } } ] , usage } )
423+
424+ const chunks = await collectStream ( nonStreamingHandler . createMessage ( "system prompt" , [ ] ) )
425+
426+ expect ( chunks . filter ( ( chunk ) => chunk . type === "usage" ) ) . toEqual ( [
427+ { type : "usage" , inputTokens : 0 , outputTokens : 0 } ,
428+ ] )
429+ } ,
430+ )
215431} )
0 commit comments