Skip to content

Commit 611e82a

Browse files
myk1ytmyk1yt
committed
fix(openrouter): send task session_id and cache-write usage for prompt caching
What: - Send the task ID as OpenRouter's top-level `session_id` so provider sticky routing pins every turn of a task to the endpoint holding its warm cache. - Treat a Claude model's live `supportsPromptCache` catalog flag as sufficient to add explicit cache breakpoints, keeping the static allowlist as fallback. - Map `usage.prompt_tokens_details.cache_write_tokens` into the usage chunk's `cacheWriteTokens` so cache writes reach cost accounting. Why: OpenRouter only activates sticky routing after a cache hit unless `session_id` is sent, so turns of one task could drift across upstream endpoints and miss the cache. The static allowlist also lagged behind new Claude releases, leaving them without explicit breakpoints, and cache-write tokens were dropped entirely so cost tracking under-reported writes. Reported as ~5x cost increase on Claude Opus via OpenRouter in #1277. Impact: Behavior is additive: no `session_id` is sent without a task ID, Gemini and other providers keep allowlist-only breakpoint placement, and the usage chunk only gains an optional field. Fixes #1277
1 parent 159c42e commit 611e82a

2 files changed

Lines changed: 240 additions & 3 deletions

File tree

‎src/api/providers/__tests__/openrouter.spec.ts‎

Lines changed: 209 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,20 @@ vitest.mock("../fetchers/modelCache", () => ({
8181
description: "Claude 4.5 Sonnet",
8282
thinking: false,
8383
},
84+
// Not in OPEN_ROUTER_PROMPT_CACHING_MODELS: stands in for a Claude
85+
// model released after the static allowlist was last edited.
86+
"anthropic/claude-opus-4.8": {
87+
maxTokens: 128000,
88+
contextWindow: 1000000,
89+
supportsImages: true,
90+
supportsPromptCache: true,
91+
inputPrice: 5,
92+
outputPrice: 25,
93+
cacheWritesPrice: 6.25,
94+
cacheReadsPrice: 0.5,
95+
description: "Claude 4.8 Opus",
96+
thinking: false,
97+
},
8498
"anthropic/claude-3.7-sonnet:thinking": {
8599
maxTokens: 128000,
86100
contextWindow: 200000,
@@ -485,6 +499,201 @@ describe("OpenRouterHandler", () => {
485499
)
486500
})
487501

502+
it("adds cache control for a Claude model missing from the static allowlist", async () => {
503+
const handler = new OpenRouterHandler(
504+
makeApiHandlerOptions({
505+
...mockOptions,
506+
openRouterModelId: "anthropic/claude-opus-4.8",
507+
}),
508+
)
509+
510+
const mockStream = asyncStreamFrom([{ id: "test-id", choices: [{ delta: { content: "test response" } }] }])
511+
const mockCreate = vitest.fn().mockResolvedValue(mockStream)
512+
const chatStub = { completions: { create: mockCreate } }
513+
const openAiPrototype = OpenAI as unknown as { prototype: { chat?: unknown } }
514+
const originalChat = openAiPrototype.prototype.chat
515+
openAiPrototype.prototype.chat = chatStub
516+
517+
try {
518+
await handler.createMessage("test system", [{ role: "user", content: "test message" }]).next()
519+
520+
const [requestParams] = mockCreate.mock.calls[0] as [Record<string, unknown>]
521+
expect(requestParams).toMatchObject({
522+
messages: expect.arrayContaining([
523+
expect.objectContaining({
524+
role: "system",
525+
content: expect.arrayContaining([
526+
expect.objectContaining({ cache_control: { type: "ephemeral" } }),
527+
]),
528+
}),
529+
]),
530+
})
531+
} finally {
532+
openAiPrototype.prototype.chat = originalChat
533+
}
534+
})
535+
536+
it("keeps Gemini cache breakpoints allowlist-only", async () => {
537+
// A Gemini model absent from the allowlist must not gain Anthropic-style
538+
// breakpoints just because its catalog record advertises cache pricing.
539+
const handler = new OpenRouterHandler(
540+
makeApiHandlerOptions({
541+
...mockOptions,
542+
openRouterModelId: "google/gemini-9-pro",
543+
}),
544+
)
545+
546+
const mockStream = asyncStreamFrom([{ id: "test-id", choices: [{ delta: { content: "test response" } }] }])
547+
const mockCreate = vitest.fn().mockResolvedValue(mockStream)
548+
const chatStub = { completions: { create: mockCreate } }
549+
const openAiPrototype = OpenAI as unknown as { prototype: { chat?: unknown } }
550+
const originalChat = openAiPrototype.prototype.chat
551+
openAiPrototype.prototype.chat = chatStub
552+
553+
try {
554+
await handler.createMessage("test system", [{ role: "user", content: "test message" }]).next()
555+
556+
const [requestParams] = mockCreate.mock.calls[0] as [Record<string, unknown>]
557+
expect(requestParams).toMatchObject({
558+
messages: [{ role: "system", content: "test system" }, expect.anything()],
559+
})
560+
} finally {
561+
openAiPrototype.prototype.chat = originalChat
562+
}
563+
})
564+
565+
it("sends the task ID as the OpenRouter cache session affinity key", async () => {
566+
const handler = new OpenRouterHandler(mockOptions)
567+
568+
const mockStream = asyncStreamFrom([{ id: "test-id", choices: [{ delta: { content: "test response" } }] }])
569+
const mockCreate = vitest.fn().mockResolvedValue(mockStream)
570+
const chatStub = { completions: { create: mockCreate } }
571+
const openAiPrototype = OpenAI as unknown as { prototype: { chat?: unknown } }
572+
const originalChat = openAiPrototype.prototype.chat
573+
openAiPrototype.prototype.chat = chatStub
574+
575+
try {
576+
await handler
577+
.createMessage("test system", [{ role: "user", content: "test message" }], { taskId: "task-123" })
578+
.next()
579+
580+
const [requestParams] = mockCreate.mock.calls[0] as [Record<string, unknown>]
581+
expect(requestParams).toMatchObject({ session_id: "task-123" })
582+
} finally {
583+
openAiPrototype.prototype.chat = originalChat
584+
}
585+
})
586+
587+
it("omits session_id when no task ID is available", async () => {
588+
const handler = new OpenRouterHandler(mockOptions)
589+
590+
const mockStream = asyncStreamFrom([{ id: "test-id", choices: [{ delta: { content: "test response" } }] }])
591+
const mockCreate = vitest.fn().mockResolvedValue(mockStream)
592+
const chatStub = { completions: { create: mockCreate } }
593+
const openAiPrototype = OpenAI as unknown as { prototype: { chat?: unknown } }
594+
const originalChat = openAiPrototype.prototype.chat
595+
openAiPrototype.prototype.chat = chatStub
596+
597+
try {
598+
await handler.createMessage("test system", [{ role: "user", content: "test message" }]).next()
599+
600+
const [requestParams] = mockCreate.mock.calls[0] as [Record<string, unknown>]
601+
expect(requestParams).not.toHaveProperty("session_id")
602+
} finally {
603+
openAiPrototype.prototype.chat = originalChat
604+
}
605+
})
606+
607+
it("truncates an over-long task ID to OpenRouter's 256-character session_id limit", async () => {
608+
const handler = new OpenRouterHandler(mockOptions)
609+
610+
const mockStream = asyncStreamFrom([{ id: "test-id", choices: [{ delta: { content: "test response" } }] }])
611+
const mockCreate = vitest.fn().mockResolvedValue(mockStream)
612+
const chatStub = { completions: { create: mockCreate } }
613+
const openAiPrototype = OpenAI as unknown as { prototype: { chat?: unknown } }
614+
const originalChat = openAiPrototype.prototype.chat
615+
openAiPrototype.prototype.chat = chatStub
616+
617+
try {
618+
await handler
619+
.createMessage("test system", [{ role: "user", content: "test message" }], {
620+
taskId: "a".repeat(300),
621+
})
622+
.next()
623+
624+
const [requestParams] = mockCreate.mock.calls[0] as [Record<string, unknown>]
625+
expect(requestParams.session_id).toBe("a".repeat(256))
626+
} finally {
627+
openAiPrototype.prototype.chat = originalChat
628+
}
629+
})
630+
631+
it("maps OpenRouter cache read and write usage tokens", async () => {
632+
const handler = new OpenRouterHandler(mockOptions)
633+
634+
const mockStream = asyncStreamFrom([
635+
{
636+
id: "test-id",
637+
choices: [{ delta: {} }],
638+
usage: {
639+
prompt_tokens: 10,
640+
completion_tokens: 2,
641+
prompt_tokens_details: { cached_tokens: 6, cache_write_tokens: 4 },
642+
},
643+
},
644+
])
645+
const mockCreate = vitest.fn().mockResolvedValue(mockStream)
646+
const chatStub = { completions: { create: mockCreate } }
647+
const openAiPrototype = OpenAI as unknown as { prototype: { chat?: unknown } }
648+
const originalChat = openAiPrototype.prototype.chat
649+
openAiPrototype.prototype.chat = chatStub
650+
651+
try {
652+
const chunks = await collectStream(
653+
handler.createMessage("test system", [{ role: "user", content: "test message" }]),
654+
)
655+
656+
expect(chunks).toContainEqual({
657+
type: "usage",
658+
inputTokens: 10,
659+
outputTokens: 2,
660+
cacheReadTokens: 6,
661+
cacheWriteTokens: 4,
662+
totalCost: 0,
663+
})
664+
} finally {
665+
openAiPrototype.prototype.chat = originalChat
666+
}
667+
})
668+
669+
it("leaves cache token fields unset when OpenRouter reports no cache usage", async () => {
670+
const handler = new OpenRouterHandler(mockOptions)
671+
672+
const mockStream = asyncStreamFrom([
673+
{
674+
id: "test-id",
675+
choices: [{ delta: {} }],
676+
usage: { prompt_tokens: 10, completion_tokens: 2 },
677+
},
678+
])
679+
const mockCreate = vitest.fn().mockResolvedValue(mockStream)
680+
const chatStub = { completions: { create: mockCreate } }
681+
const openAiPrototype = OpenAI as unknown as { prototype: { chat?: unknown } }
682+
const originalChat = openAiPrototype.prototype.chat
683+
openAiPrototype.prototype.chat = chatStub
684+
685+
try {
686+
const chunks = await collectStream(
687+
handler.createMessage("test system", [{ role: "user", content: "test message" }]),
688+
)
689+
690+
const usageChunk = chunks.find((chunk) => chunk.type === "usage")
691+
expect(usageChunk).toMatchObject({ cacheReadTokens: undefined, cacheWriteTokens: undefined })
692+
} finally {
693+
openAiPrototype.prototype.chat = originalChat
694+
}
695+
})
696+
488697
it("handles API errors and captures telemetry", async () => {
489698
const handler = new OpenRouterHandler(mockOptions)
490699
const mockStream = asyncStreamFrom([{ error: { message: "API Error", code: 500 } }])

‎src/api/providers/openrouter.ts‎

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,10 +49,16 @@ import { applyRouterToolPreferences } from "./utils/router-tool-preferences"
4949
type OpenRouterChatCompletionParams = OpenAI.Chat.ChatCompletionCreateParams & {
5050
transforms?: string[]
5151
include_reasoning?: boolean
52+
// https://openrouter.ai/docs/guides/best-practices/prompt-caching#using-session_id-for-sticky-sessions
53+
session_id?: string
5254
// https://openrouter.ai/docs/use-cases/reasoning-tokens
5355
reasoning?: OpenRouterReasoningParams
5456
}
5557

58+
// OpenRouter rejects a `session_id` longer than this.
59+
// https://openrouter.ai/docs/guides/best-practices/prompt-caching#using-session_id-for-sticky-sessions
60+
const OPENROUTER_MAX_SESSION_ID_LENGTH = 256
61+
5662
// Zod schema for OpenRouter error response structure (for caught exceptions)
5763
const OpenRouterErrorResponseSchema = z.object({
5864
error: z
@@ -125,6 +131,8 @@ interface CompletionUsage {
125131
prompt_tokens?: number
126132
prompt_tokens_details?: {
127133
cached_tokens?: number
134+
// https://openrouter.ai/docs/guides/best-practices/prompt-caching#usage-object-fields
135+
cache_write_tokens?: number
128136
}
129137
total_tokens?: number
130138
cost?: number
@@ -219,6 +227,10 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH
219227

220228
let { id: modelId, maxTokens, temperature, topP, reasoning } = model
221229

230+
// OpenRouter caps `session_id` at 256 characters. Task IDs are UUIDs and
231+
// far shorter, so this only guards against an over-long caller value.
232+
const sessionId = metadata?.taskId?.slice(0, OPENROUTER_MAX_SESSION_ID_LENGTH)
233+
222234
// Reset reasoning_details accumulator for this request
223235
this.currentReasoningDetails = []
224236

@@ -302,9 +314,19 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH
302314
})
303315
}
304316

305-
// https://openrouter.ai/docs/features/prompt-caching
306-
// TODO: Add a `promptCacheStratey` field to `ModelInfo`.
307-
if (OPEN_ROUTER_PROMPT_CACHING_MODELS.has(modelId)) {
317+
// https://openrouter.ai/docs/guides/best-practices/prompt-caching
318+
// TODO: Add a `promptCacheStrategy` field to `ModelInfo`.
319+
// The allowlist is a static snapshot, so a Claude model released after it
320+
// was last edited got no breakpoints even though OpenRouter advertises
321+
// cache pricing for it. Trust the live capability flag for Claude and keep
322+
// the allowlist as the fallback; Gemini and every other provider keep
323+
// allowlist-only behavior because breakpoint placement is not
324+
// interchangeable between them.
325+
const supportsCacheBreakpoints =
326+
OPEN_ROUTER_PROMPT_CACHING_MODELS.has(modelId) ||
327+
(modelId.startsWith("anthropic/") && model.info.supportsPromptCache)
328+
329+
if (supportsCacheBreakpoints) {
308330
if (modelId.startsWith("google")) {
309331
addGeminiCacheBreakpoints(systemPrompt, openAiMessages)
310332
} else {
@@ -315,6 +337,11 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH
315337
// https://openrouter.ai/docs/transforms
316338
const completionParams: OpenRouterChatCompletionParams = {
317339
model: modelId,
340+
// OpenRouter derives its sticky-routing key from the opening messages
341+
// unless `session_id` is sent, and without it stickiness only starts
342+
// after a cache hit is observed. A task ID pins every turn of the task
343+
// to the endpoint holding its warm cache from the first request.
344+
...(sessionId ? { session_id: sessionId } : {}),
318345
...(maxTokens && maxTokens > 0 && { max_tokens: maxTokens }),
319346
temperature,
320347
top_p: topP,
@@ -540,6 +567,7 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH
540567
inputTokens: lastUsage.prompt_tokens || 0,
541568
outputTokens: lastUsage.completion_tokens || 0,
542569
cacheReadTokens: lastUsage.prompt_tokens_details?.cached_tokens,
570+
cacheWriteTokens: lastUsage.prompt_tokens_details?.cache_write_tokens,
543571
reasoningTokens: lastUsage.completion_tokens_details?.reasoning_tokens,
544572
totalCost: (lastUsage.cost_details?.upstream_inference_cost || 0) + (lastUsage.cost || 0),
545573
}

0 commit comments

Comments
 (0)