Skip to content

Commit 44dd4b2

Browse files
committed
fix(xai): send body prompt_cache_key for Responses cache affinity
What: add a task-scoped `prompt_cache_key` to the xAI Responses API request body in both `createMessage` and `completePrompt`, gated on the model's `supportsPromptCache` flag and clamped to 64 characters. Why: xAI's prompt cache is stored per server, so every turn of one conversation must be routed to the same server to reuse it. For the Responses API the documented affinity lever is the body field `prompt_cache_key`; the `x-grok-conv-id` header is the Chat Completions / gRPC equivalent and is not read on this path, so it was deliberately NOT added. Before this change `xai.ts` sent no affinity identifier at all, leaving server-side cache routing to load-balancer luck. Impact: xAI turns of one task now pin to one cache server, raising cache-read tokens and lowering cost on long conversations. The key is omitted (never sent as an empty string) when no identifier is available or the model does not support prompt caching. Usage accounting is unchanged: the shared normalizer already reads `input_tokens_details.cached_tokens`.
1 parent 159c42e commit 44dd4b2

2 files changed

Lines changed: 194 additions & 0 deletions

File tree

‎src/api/providers/__tests__/xai.spec.ts‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,24 @@ import { asyncStreamFrom, collectStream } from "../../../test-utils/stream"
3333
import { makeCreateMessageMetadata } from "../../../test-utils/api"
3434
import { clearAllMocks } from "../../../test-utils/reset"
3535

36+
/**
37+
* Flip only `supportsPromptCache` on the handler's resolved model, so the rest of
38+
* the catalog entry (maxTokens, temperature, reasoning) stays as the real one.
39+
*
40+
* The xAI catalog is declared `as const`, so every entry's `supportsPromptCache`
41+
* is the literal `true` and no real uncached model exists to point at. The double
42+
* assertion is confined to this test double and is the only way to reach the
43+
* `supportsPromptCache === false` arm; the production code is untouched by it.
44+
*/
45+
function stubSupportsPromptCache(handler: XAIHandler, supportsPromptCache: boolean): void {
46+
const realModel = handler.getModel()
47+
vitest
48+
.spyOn(handler, "getModel")
49+
.mockReturnValue({ ...realModel, info: { ...realModel.info, supportsPromptCache } } as unknown as ReturnType<
50+
XAIHandler["getModel"]
51+
>)
52+
}
53+
3654
describe("XAIHandler", () => {
3755
let handler: XAIHandler
3856

@@ -372,6 +390,8 @@ describe("XAIHandler", () => {
372390
model: xaiDefaultModelId,
373391
input: [{ role: "user", content: [{ type: "input_text", text: "test prompt" }] }],
374392
store: false,
393+
// No task metadata exists on this path, so the handler session scopes the cache.
394+
prompt_cache_key: expect.any(String),
375395
},
376396
undefined,
377397
)
@@ -722,4 +742,130 @@ describe("XAIHandler", () => {
722742
expect(removeEventListenerSpy).toHaveBeenCalledTimes(1)
723743
expect(removeEventListenerSpy).toHaveBeenCalledWith("abort", listener)
724744
})
745+
746+
describe("prompt cache affinity", () => {
747+
const lastRequestBody = () => mockResponsesCreate.mock.calls[mockResponsesCreate.mock.calls.length - 1][0]
748+
749+
it("should send the task id as the body prompt_cache_key", async () => {
750+
mockResponsesCreate.mockResolvedValueOnce(asyncStreamFrom([]))
751+
752+
const stream = handler.createMessage("test prompt", [], makeCreateMessageMetadata({ taskId: "task-abc" }))
753+
await stream.next()
754+
755+
expect(lastRequestBody().prompt_cache_key).toBe("task-abc")
756+
})
757+
758+
it("should keep the key stable across two turns of the same task", async () => {
759+
mockResponsesCreate.mockResolvedValue(asyncStreamFrom([]))
760+
761+
const first = handler.createMessage("first turn", [], makeCreateMessageMetadata({ taskId: "task-stable" }))
762+
await first.next()
763+
const second = handler.createMessage(
764+
"second turn",
765+
[],
766+
makeCreateMessageMetadata({ taskId: "task-stable" }),
767+
)
768+
await second.next()
769+
770+
// Two separate requests, one value: routing is pinned to one cache server.
771+
expect(mockResponsesCreate).toHaveBeenCalledTimes(2)
772+
expect(mockResponsesCreate.mock.calls[0][0].prompt_cache_key).toBe("task-stable")
773+
expect(mockResponsesCreate.mock.calls[1][0].prompt_cache_key).toBe(
774+
mockResponsesCreate.mock.calls[0][0].prompt_cache_key,
775+
)
776+
})
777+
778+
it("should give distinct keys to distinct tasks", async () => {
779+
mockResponsesCreate.mockResolvedValue(asyncStreamFrom([]))
780+
781+
for (const taskId of ["task-one", "task-two"]) {
782+
const stream = handler.createMessage("test prompt", [], makeCreateMessageMetadata({ taskId }))
783+
await stream.next()
784+
}
785+
786+
expect(mockResponsesCreate.mock.calls[0][0].prompt_cache_key).toBe("task-one")
787+
expect(mockResponsesCreate.mock.calls[1][0].prompt_cache_key).toBe("task-two")
788+
})
789+
790+
it("should fall back to the handler session id when no task id is supplied", async () => {
791+
mockResponsesCreate.mockResolvedValue(asyncStreamFrom([]))
792+
793+
const stream = handler.createMessage("test prompt", [])
794+
await stream.next()
795+
796+
const key = lastRequestBody().prompt_cache_key
797+
expect(typeof key).toBe("string")
798+
expect(key.length).toBeGreaterThan(0)
799+
800+
// The same handler keeps the same fallback key on the next turn.
801+
const second = handler.createMessage("test prompt", [])
802+
await second.next()
803+
expect(lastRequestBody().prompt_cache_key).toBe(key)
804+
})
805+
806+
it("should clamp an over-length task id to the 64-character limit", async () => {
807+
mockResponsesCreate.mockResolvedValueOnce(asyncStreamFrom([]))
808+
809+
const overLongTaskId = "t".repeat(80)
810+
const stream = handler.createMessage(
811+
"test prompt",
812+
[],
813+
makeCreateMessageMetadata({ taskId: overLongTaskId }),
814+
)
815+
await stream.next()
816+
817+
const key = lastRequestBody().prompt_cache_key
818+
expect(key).toHaveLength(64)
819+
expect(key).toBe(overLongTaskId.slice(0, 64))
820+
})
821+
822+
it("should never send an empty-string prompt_cache_key", async () => {
823+
mockResponsesCreate.mockResolvedValue(asyncStreamFrom([]))
824+
825+
const stream = handler.createMessage("test prompt", [], makeCreateMessageMetadata({ taskId: "" }))
826+
await stream.next()
827+
828+
const key = lastRequestBody().prompt_cache_key
829+
// An empty task id falls back to the session id instead of sending "".
830+
expect(key).not.toBe("")
831+
expect(typeof key).toBe("string")
832+
})
833+
834+
it("should omit the key when the model does not support prompt caching", async () => {
835+
const uncachedHandler = new XAIHandler({ apiModelId: xaiDefaultModelId })
836+
stubSupportsPromptCache(uncachedHandler, false)
837+
838+
mockResponsesCreate.mockResolvedValueOnce(asyncStreamFrom([]))
839+
const stream = uncachedHandler.createMessage(
840+
"test prompt",
841+
[],
842+
makeCreateMessageMetadata({ taskId: "task-x" }),
843+
)
844+
await stream.next()
845+
846+
expect(lastRequestBody()).not.toHaveProperty("prompt_cache_key")
847+
})
848+
849+
it("completePrompt should scope the prompt_cache_key to the handler session", async () => {
850+
mockResponsesCreate.mockResolvedValue({ output_text: "response" })
851+
852+
await handler.completePrompt("first prompt")
853+
await handler.completePrompt("second prompt")
854+
855+
const firstKey = mockResponsesCreate.mock.calls[0][0].prompt_cache_key
856+
expect(typeof firstKey).toBe("string")
857+
expect(firstKey.length).toBeGreaterThan(0)
858+
expect(mockResponsesCreate.mock.calls[1][0].prompt_cache_key).toBe(firstKey)
859+
})
860+
861+
it("completePrompt should omit the key when the model does not support prompt caching", async () => {
862+
const uncachedHandler = new XAIHandler({ apiModelId: xaiDefaultModelId })
863+
stubSupportsPromptCache(uncachedHandler, false)
864+
865+
mockResponsesCreate.mockResolvedValueOnce({ output_text: "response" })
866+
await uncachedHandler.completePrompt("test prompt")
867+
868+
expect(mockResponsesCreate.mock.calls[0][0]).not.toHaveProperty("prompt_cache_key")
869+
})
870+
})
725871
})

‎src/api/providers/xai.ts‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { Anthropic } from "@anthropic-ai/sdk"
22
import OpenAI from "openai"
3+
import { v7 as uuidv7 } from "uuid"
34

45
import { type XAIModelId, xaiDefaultModelId, xaiModels, ApiProviderError } from "@roo-code/types"
56
import { TelemetryService } from "@roo-code/telemetry"
@@ -26,10 +27,15 @@ import { isMcpTool } from "../../utils/mcp-name"
2627

2728
const XAI_DEFAULT_TEMPERATURE = 0
2829

30+
// xAI caps `prompt_cache_key` at 64 characters, the same bound the OpenAI Responses
31+
// API enforces on the field. Task ids are uuid v7 (36 chars), so this is a guard.
32+
const XAI_MAX_PROMPT_CACHE_KEY_LENGTH = 64
33+
2934
export class XAIHandler extends BaseProvider implements SingleCompletionHandler {
3035
protected options: ApiHandlerOptions
3136
private client: OpenAI
3237
private readonly providerName = "xAI"
38+
private readonly sessionId: string = uuidv7()
3339

3440
constructor(options: ApiHandlerOptions) {
3541
super()
@@ -45,6 +51,38 @@ export class XAIHandler extends BaseProvider implements SingleCompletionHandler
4551
})
4652
}
4753

54+
/**
55+
* Cache routing affinity key for the Responses API.
56+
*
57+
* xAI's prompt cache lives on individual servers, so every turn of one
58+
* conversation has to reach the same server to reuse it. For the Responses API
59+
* the documented lever is the body field `prompt_cache_key`; the
60+
* `x-grok-conv-id` header is the Chat Completions / gRPC equivalent and is not
61+
* read on this path.
62+
*
63+
* The task id is the longest-lived identifier available here: `Task.taskId` is
64+
* assigned once and restored from history on resume, so it survives restarts
65+
* and provider switches, which a per-handler session id does not. The handler
66+
* session id is the fallback for callers that send no task id, and is the sole
67+
* key for `completePrompt`, which carries no task metadata at all.
68+
*
69+
* Returns undefined (never an empty string) when there is nothing to key on or
70+
* when the model does not support prompt caching, so the field is omitted
71+
* rather than sent blank.
72+
*/
73+
private getPromptCacheKey(model: ReturnType<XAIHandler["getModel"]>, taskId?: string): string | undefined {
74+
if (!model.info.supportsPromptCache) {
75+
return undefined
76+
}
77+
78+
const key = taskId || this.sessionId
79+
if (!key) {
80+
return undefined
81+
}
82+
83+
return key.length > XAI_MAX_PROMPT_CACHE_KEY_LENGTH ? key.slice(0, XAI_MAX_PROMPT_CACHE_KEY_LENGTH) : key
84+
}
85+
4886
override getModel() {
4987
const id =
5088
this.options.apiModelId && this.options.apiModelId in xaiModels
@@ -121,6 +159,12 @@ export class XAIHandler extends BaseProvider implements SingleCompletionHandler
121159
include: ["reasoning.encrypted_content"],
122160
}
123161

162+
// Route every turn of this task to the server holding its cache.
163+
const promptCacheKey = this.getPromptCacheKey(model, metadata?.taskId)
164+
if (promptCacheKey) {
165+
requestBody.prompt_cache_key = promptCacheKey
166+
}
167+
124168
if (model.maxTokens) {
125169
requestBody.max_output_tokens = model.maxTokens
126170
}
@@ -202,11 +246,15 @@ export class XAIHandler extends BaseProvider implements SingleCompletionHandler
202246
requestOptions.timeout = toSdkTimeoutMs(timeoutMs)
203247
}
204248

249+
const promptCacheKey = this.getPromptCacheKey(model)
205250
const response = await this.client.responses.create(
206251
{
207252
model: model.id,
208253
input: [{ role: "user", content: [{ type: "input_text", text: prompt }] }],
209254
store: false,
255+
// A completion is its own cache scope: no task metadata exists on
256+
// this path, so the handler session id is the whole key.
257+
...(promptCacheKey ? { prompt_cache_key: promptCacheKey } : {}),
210258
},
211259
Object.keys(requestOptions).length > 0 ? requestOptions : undefined,
212260
)

0 commit comments

Comments
 (0)