@@ -33,6 +33,24 @@ import { asyncStreamFrom, collectStream } from "../../../test-utils/stream"
3333import { makeCreateMessageMetadata } from "../../../test-utils/api"
3434import { clearAllMocks } from "../../../test-utils/reset"
3535
36+ /**
37+ * Flip only `supportsPromptCache` on the handler's resolved model, so the rest of
38+ * the catalog entry (maxTokens, temperature, reasoning) stays as the real one.
39+ *
40+ * The xAI catalog is declared `as const`, so every entry's `supportsPromptCache`
41+ * is the literal `true` and no real uncached model exists to point at. The double
42+ * assertion is confined to this test double and is the only way to reach the
43+ * `supportsPromptCache === false` arm; the production code is untouched by it.
44+ */
45+ function stubSupportsPromptCache ( handler : XAIHandler , supportsPromptCache : boolean ) : void {
46+ const realModel = handler . getModel ( )
47+ vitest
48+ . spyOn ( handler , "getModel" )
49+ . mockReturnValue ( { ...realModel , info : { ...realModel . info , supportsPromptCache } } as unknown as ReturnType <
50+ XAIHandler [ "getModel" ]
51+ > )
52+ }
53+
3654describe ( "XAIHandler" , ( ) => {
3755 let handler : XAIHandler
3856
@@ -372,6 +390,8 @@ describe("XAIHandler", () => {
372390 model : xaiDefaultModelId ,
373391 input : [ { role : "user" , content : [ { type : "input_text" , text : "test prompt" } ] } ] ,
374392 store : false ,
393+ // No task metadata exists on this path, so the handler session scopes the cache.
394+ prompt_cache_key : expect . any ( String ) ,
375395 } ,
376396 undefined ,
377397 )
@@ -722,4 +742,130 @@ describe("XAIHandler", () => {
722742 expect ( removeEventListenerSpy ) . toHaveBeenCalledTimes ( 1 )
723743 expect ( removeEventListenerSpy ) . toHaveBeenCalledWith ( "abort" , listener )
724744 } )
745+
746+ describe ( "prompt cache affinity" , ( ) => {
747+ const lastRequestBody = ( ) => mockResponsesCreate . mock . calls [ mockResponsesCreate . mock . calls . length - 1 ] [ 0 ]
748+
749+ it ( "should send the task id as the body prompt_cache_key" , async ( ) => {
750+ mockResponsesCreate . mockResolvedValueOnce ( asyncStreamFrom ( [ ] ) )
751+
752+ const stream = handler . createMessage ( "test prompt" , [ ] , makeCreateMessageMetadata ( { taskId : "task-abc" } ) )
753+ await stream . next ( )
754+
755+ expect ( lastRequestBody ( ) . prompt_cache_key ) . toBe ( "task-abc" )
756+ } )
757+
758+ it ( "should keep the key stable across two turns of the same task" , async ( ) => {
759+ mockResponsesCreate . mockResolvedValue ( asyncStreamFrom ( [ ] ) )
760+
761+ const first = handler . createMessage ( "first turn" , [ ] , makeCreateMessageMetadata ( { taskId : "task-stable" } ) )
762+ await first . next ( )
763+ const second = handler . createMessage (
764+ "second turn" ,
765+ [ ] ,
766+ makeCreateMessageMetadata ( { taskId : "task-stable" } ) ,
767+ )
768+ await second . next ( )
769+
770+ // Two separate requests, one value: routing is pinned to one cache server.
771+ expect ( mockResponsesCreate ) . toHaveBeenCalledTimes ( 2 )
772+ expect ( mockResponsesCreate . mock . calls [ 0 ] [ 0 ] . prompt_cache_key ) . toBe ( "task-stable" )
773+ expect ( mockResponsesCreate . mock . calls [ 1 ] [ 0 ] . prompt_cache_key ) . toBe (
774+ mockResponsesCreate . mock . calls [ 0 ] [ 0 ] . prompt_cache_key ,
775+ )
776+ } )
777+
778+ it ( "should give distinct keys to distinct tasks" , async ( ) => {
779+ mockResponsesCreate . mockResolvedValue ( asyncStreamFrom ( [ ] ) )
780+
781+ for ( const taskId of [ "task-one" , "task-two" ] ) {
782+ const stream = handler . createMessage ( "test prompt" , [ ] , makeCreateMessageMetadata ( { taskId } ) )
783+ await stream . next ( )
784+ }
785+
786+ expect ( mockResponsesCreate . mock . calls [ 0 ] [ 0 ] . prompt_cache_key ) . toBe ( "task-one" )
787+ expect ( mockResponsesCreate . mock . calls [ 1 ] [ 0 ] . prompt_cache_key ) . toBe ( "task-two" )
788+ } )
789+
790+ it ( "should fall back to the handler session id when no task id is supplied" , async ( ) => {
791+ mockResponsesCreate . mockResolvedValue ( asyncStreamFrom ( [ ] ) )
792+
793+ const stream = handler . createMessage ( "test prompt" , [ ] )
794+ await stream . next ( )
795+
796+ const key = lastRequestBody ( ) . prompt_cache_key
797+ expect ( typeof key ) . toBe ( "string" )
798+ expect ( key . length ) . toBeGreaterThan ( 0 )
799+
800+ // The same handler keeps the same fallback key on the next turn.
801+ const second = handler . createMessage ( "test prompt" , [ ] )
802+ await second . next ( )
803+ expect ( lastRequestBody ( ) . prompt_cache_key ) . toBe ( key )
804+ } )
805+
806+ it ( "should clamp an over-length task id to the 64-character limit" , async ( ) => {
807+ mockResponsesCreate . mockResolvedValueOnce ( asyncStreamFrom ( [ ] ) )
808+
809+ const overLongTaskId = "t" . repeat ( 80 )
810+ const stream = handler . createMessage (
811+ "test prompt" ,
812+ [ ] ,
813+ makeCreateMessageMetadata ( { taskId : overLongTaskId } ) ,
814+ )
815+ await stream . next ( )
816+
817+ const key = lastRequestBody ( ) . prompt_cache_key
818+ expect ( key ) . toHaveLength ( 64 )
819+ expect ( key ) . toBe ( overLongTaskId . slice ( 0 , 64 ) )
820+ } )
821+
822+ it ( "should never send an empty-string prompt_cache_key" , async ( ) => {
823+ mockResponsesCreate . mockResolvedValue ( asyncStreamFrom ( [ ] ) )
824+
825+ const stream = handler . createMessage ( "test prompt" , [ ] , makeCreateMessageMetadata ( { taskId : "" } ) )
826+ await stream . next ( )
827+
828+ const key = lastRequestBody ( ) . prompt_cache_key
829+ // An empty task id falls back to the session id instead of sending "".
830+ expect ( key ) . not . toBe ( "" )
831+ expect ( typeof key ) . toBe ( "string" )
832+ } )
833+
834+ it ( "should omit the key when the model does not support prompt caching" , async ( ) => {
835+ const uncachedHandler = new XAIHandler ( { apiModelId : xaiDefaultModelId } )
836+ stubSupportsPromptCache ( uncachedHandler , false )
837+
838+ mockResponsesCreate . mockResolvedValueOnce ( asyncStreamFrom ( [ ] ) )
839+ const stream = uncachedHandler . createMessage (
840+ "test prompt" ,
841+ [ ] ,
842+ makeCreateMessageMetadata ( { taskId : "task-x" } ) ,
843+ )
844+ await stream . next ( )
845+
846+ expect ( lastRequestBody ( ) ) . not . toHaveProperty ( "prompt_cache_key" )
847+ } )
848+
849+ it ( "completePrompt should scope the prompt_cache_key to the handler session" , async ( ) => {
850+ mockResponsesCreate . mockResolvedValue ( { output_text : "response" } )
851+
852+ await handler . completePrompt ( "first prompt" )
853+ await handler . completePrompt ( "second prompt" )
854+
855+ const firstKey = mockResponsesCreate . mock . calls [ 0 ] [ 0 ] . prompt_cache_key
856+ expect ( typeof firstKey ) . toBe ( "string" )
857+ expect ( firstKey . length ) . toBeGreaterThan ( 0 )
858+ expect ( mockResponsesCreate . mock . calls [ 1 ] [ 0 ] . prompt_cache_key ) . toBe ( firstKey )
859+ } )
860+
861+ it ( "completePrompt should omit the key when the model does not support prompt caching" , async ( ) => {
862+ const uncachedHandler = new XAIHandler ( { apiModelId : xaiDefaultModelId } )
863+ stubSupportsPromptCache ( uncachedHandler , false )
864+
865+ mockResponsesCreate . mockResolvedValueOnce ( { output_text : "response" } )
866+ await uncachedHandler . completePrompt ( "test prompt" )
867+
868+ expect ( mockResponsesCreate . mock . calls [ 0 ] [ 0 ] ) . not . toHaveProperty ( "prompt_cache_key" )
869+ } )
870+ } )
725871} )
0 commit comments