Skip to content

Commit 65ac862

Browse files
committed
feat(api): emit an opt-in prompt cache TTL gated by catalog capability
What: add a `promptCacheTtl` provider-settings knob ("5m" | "1h"), a `ModelInfo.promptCacheTtlSupport` capability array, and a shared `resolveCacheTtl`/`cacheControl` helper that the anthropic, vertex and vercel-ai-gateway caching transforms plus the anthropic, anthropic-vertex, openrouter, vercel-ai-gateway and zoo-gateway providers thread through. Why: ADR-0006 phase 2. C5a (#2260) landed per-TTL write accounting, so a 1h cache write is billed at 2x instead of the 5m rate. This is the matching request-side mechanism: once the settings UI ships in C6, a user selecting 1h causes Anthropic-shaped providers to emit `cache_control: { ttl: "1h" }`. Impact: no behavior change by default. An unset knob resolves to `undefined` and the marker stays the byte-identical `{ type: "ephemeral" }` literal, so no `ttl` key is emitted unless the user opts in AND the resolved ModelInfo advertises the capability. Absent capability means 5m only (I2), capability is read from ModelInfo rather than id substrings, and every marker in a request shares one resolved TTL (I4). Dynamic catalogs (OpenRouter, the gateways) carry no capability data and therefore stay at 5m, as the ADR requires.
1 parent 282ee30 commit 65ac862

18 files changed

Lines changed: 774 additions & 23 deletions

File tree

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
// npx vitest run src/__tests__/prompt-cache-ttl.test.ts
2+
3+
import { describe, it, expect } from "vitest"
4+
5+
import { anthropicModels } from "../providers/anthropic.js"
6+
import { vertexModels } from "../providers/vertex.js"
7+
import { openRouterDefaultModelInfo } from "../providers/openrouter.js"
8+
import {
9+
vercelAiGatewayDefaultModelId,
10+
VERCEL_AI_GATEWAY_PROMPT_CACHING_MODELS,
11+
} from "../providers/vercel-ai-gateway.js"
12+
import {
13+
DEFAULT_PROMPT_CACHE_TTL,
14+
modelInfoSchema,
15+
promptCacheTtlSchema,
16+
promptCacheTtlValues,
17+
type ModelInfo,
18+
} from "../model.js"
19+
import { anthropicProviderDefinition } from "../provider-settings/anthropic.js"
20+
import { baseProviderSettingsShape } from "../provider-settings/common.js"
21+
import { providerIdentifiers } from "../provider-identifiers.js"
22+
import { globalSettingsSchema } from "../global-settings.js"
23+
24+
/**
25+
* Catalog invariant for the request-side TTL knob (ADR-0006 phase 2, PR-C5b).
26+
*
27+
* Emission is gated by `ModelInfo.promptCacheTtlSupport`, so a model that can
28+
* report a 1h cache write but carries no capability array would silently never
29+
* emit the TTL the user asked for. These tests pin both directions: capable
30+
* catalogs advertise it, and catalogs that cannot know stay absent (fail closed).
31+
*/
32+
describe("prompt cache TTL schema", () => {
33+
it("accepts exactly the 5m and 1h classes", () => {
34+
expect(promptCacheTtlValues).toEqual(["5m", "1h"])
35+
expect(promptCacheTtlSchema.parse("5m")).toBe("5m")
36+
expect(promptCacheTtlSchema.parse("1h")).toBe("1h")
37+
expect(promptCacheTtlSchema.safeParse("30m").success).toBe(false)
38+
expect(promptCacheTtlSchema.safeParse("12h").success).toBe(false)
39+
})
40+
41+
it("defaults to undefined so an unset knob emits no ttl key", () => {
42+
expect(DEFAULT_PROMPT_CACHE_TTL).toBeUndefined()
43+
})
44+
45+
it("is optional on ModelInfo", () => {
46+
const parsed = modelInfoSchema.parse({ contextWindow: 200_000, supportsPromptCache: true })
47+
expect(parsed.promptCacheTtlSupport).toBeUndefined()
48+
49+
const withCapability = modelInfoSchema.parse({
50+
contextWindow: 200_000,
51+
supportsPromptCache: true,
52+
promptCacheTtlSupport: ["5m", "1h"],
53+
})
54+
expect(withCapability.promptCacheTtlSupport).toEqual(["5m", "1h"])
55+
})
56+
})
57+
58+
describe("prompt cache TTL knob location", () => {
59+
it("lives in provider settings so handlers can see it", () => {
60+
// ApiHandlerOptions = Omit<ProviderSettings, "apiProvider">, so only a
61+
// provider-settings key reaches a handler (ADR-0006 decision item 2).
62+
expect(baseProviderSettingsShape.promptCacheTtl).toBeDefined()
63+
64+
const parsed = anthropicProviderDefinition.schema.safeParse({
65+
apiProvider: providerIdentifiers.anthropic,
66+
promptCacheTtl: "1h",
67+
})
68+
expect(parsed.success).toBe(true)
69+
expect(parsed.success && parsed.data.promptCacheTtl).toBe("1h")
70+
})
71+
72+
it("is not a global setting", () => {
73+
const shape = globalSettingsSchema.shape as Record<string, unknown>
74+
expect(shape.promptCacheTtl).toBeUndefined()
75+
})
76+
})
77+
78+
describe("prompt cache TTL capability catalog", () => {
79+
it("every Anthropic entry with prompt caching advertises both TTL classes", () => {
80+
const missing = Object.entries(anthropicModels as Record<string, ModelInfo>)
81+
.filter(([, info]) => info.supportsPromptCache && !info.promptCacheTtlSupport?.includes("1h"))
82+
.map(([id]) => id)
83+
84+
expect(missing).toEqual([])
85+
})
86+
87+
it("leaves the Vertex catalog without capability so it fails closed", () => {
88+
// PR-C5b scopes the catalog capability to the Anthropic entries; the Vertex
89+
// catalog is populated separately, so its models stay 5m-only for now.
90+
const withCapability = Object.entries(vertexModels as Record<string, ModelInfo>)
91+
.filter(([, info]) => info.promptCacheTtlSupport !== undefined)
92+
.map(([id]) => id)
93+
94+
expect(withCapability).toEqual([])
95+
})
96+
97+
it("leaves dynamic-catalog defaults without capability so they fail closed", () => {
98+
// OpenRouter and the gateway catalogs are fetched at runtime and carry no TTL
99+
// capability data, which ADR-0006 maps to "5m only" (invariant I2).
100+
expect(openRouterDefaultModelInfo.promptCacheTtlSupport).toBeUndefined()
101+
expect(VERCEL_AI_GATEWAY_PROMPT_CACHING_MODELS.has(vercelAiGatewayDefaultModelId)).toBe(true)
102+
})
103+
})

‎packages/types/src/model.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,20 @@ export type ModelParameter = z.infer<typeof modelParametersSchema>
9595
export const isModelParameter = (value: string): value is ModelParameter =>
9696
modelParameters.includes(value as ModelParameter)
9797

98+
/**
99+
* Prompt cache TTL classes a request-side cache marker can carry. `"5m"` is the
100+
* vendor default and is never emitted explicitly, so only `"1h"` changes the wire
101+
* shape and an unset knob stays byte-identical to a pre-knob build (ADR-0006 I3).
102+
*/
103+
export const promptCacheTtlValues = ["5m", "1h"] as const
104+
105+
export const promptCacheTtlSchema = z.enum(promptCacheTtlValues)
106+
107+
export type PromptCacheTtl = z.infer<typeof promptCacheTtlSchema>
108+
109+
/** Deliberately `undefined`, not `"5m"`: emitting `ttl: "5m"` would change every request's bytes. */
110+
export const DEFAULT_PROMPT_CACHE_TTL: PromptCacheTtl | undefined = undefined
111+
98112
/**
99113
* ModelInfo
100114
*/
@@ -112,6 +126,8 @@ export const modelInfoSchema = z.object({
112126
// When set to "24h", extended prompt caching will be requested; when omitted
113127
// or set to "in_memory", the default in‑memory cache is used.
114128
promptCacheRetention: z.enum(["in_memory", "24h"]).optional(),
129+
// Cache TTLs this model accepts. Absent means "5m only" (ADR-0006 I2, fail closed).
130+
promptCacheTtlSupport: z.array(promptCacheTtlSchema).optional(),
115131
// Capability flag to indicate whether the model supports an output verbosity parameter
116132
supportsVerbosity: z.boolean().optional(),
117133
// Capability flag to indicate whether the model exposes a user-configurable max output

‎packages/types/src/provider-settings/common.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { z } from "zod"
22

3-
import { reasoningEffortSettingSchema, verbosityLevelsSchema } from "../model.js"
3+
import { reasoningEffortSettingSchema, verbosityLevelsSchema, promptCacheTtlSchema } from "../model.js"
44
import type { ProviderIdentifier } from "../provider-identifiers.js"
55

66
export const API_PROVIDER_FIELD = "apiProvider"
@@ -33,6 +33,11 @@ export const baseProviderSettingsShape = {
3333
modelMaxTokens: z.number().optional(),
3434
modelMaxThinkingTokens: z.number().optional(),
3535
verbosity: verbosityLevelsSchema.optional(),
36+
// Opt-in prompt cache TTL for this provider profile. Lives here, not in global
37+
// settings, because `ApiHandlerOptions = Omit<ProviderSettings, "apiProvider">`
38+
// (src/shared/api.ts) makes a global key invisible to handlers. Unset emits no
39+
// ttl key at all, not "5m" (ADR-0006 I3).
40+
promptCacheTtl: promptCacheTtlSchema.optional(),
3641
}
3742

3843
export const apiModelIdProviderModelShape = {

‎packages/types/src/providers/anthropic.ts‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@ import type { ModelInfo } from "../model.js"
33
// https://docs.anthropic.com/en/docs/about-claude/models
44
// https://platform.claude.com/docs/en/about-claude/pricing
55

6+
// Every entry here is an Anthropic-hosted Claude model, and 1h cache writes are GA
7+
// on all active Claude models, so each one advertises both TTL classes. The 5m class
8+
// is the vendor default and is never emitted explicitly (ADR-0006 I3).
69
export type AnthropicModelId = keyof typeof anthropicModels
710
export const anthropicDefaultModelId: AnthropicModelId = "claude-sonnet-4-5"
811
export const ANTHROPIC_API_PROTOCOL = "anthropic"
@@ -13,6 +16,7 @@ export const anthropicModels = {
1316
contextWindow: 200_000, // Default 200K, extendable to 1M with beta flag 'context-1m-2025-08-07'
1417
supportsImages: true,
1518
supportsPromptCache: true,
19+
promptCacheTtlSupport: ["5m", "1h"],
1620
inputPrice: 3.0, // $3 per million input tokens (≤200K context)
1721
outputPrice: 15.0, // $15 per million output tokens (≤200K context)
1822
cacheWritesPrice: 3.75, // $3.75 per million tokens
@@ -36,6 +40,7 @@ export const anthropicModels = {
3640
contextWindow: 1_000_000, // 1M context window native (no beta header required)
3741
supportsImages: true,
3842
supportsPromptCache: true,
43+
promptCacheTtlSupport: ["5m", "1h"],
3944
inputPrice: 2.0, // $2 per million input tokens (introductory pricing through Aug 31, 2026)
4045
outputPrice: 10.0, // $10 per million output tokens (introductory pricing through Aug 31, 2026)
4146
cacheWritesPrice: 2.5, // $2.50 per million tokens (introductory pricing through Aug 31, 2026)
@@ -56,6 +61,7 @@ export const anthropicModels = {
5661
contextWindow: 200_000, // Default 200K, extendable to 1M with beta flag 'context-1m-2025-08-07'
5762
supportsImages: true,
5863
supportsPromptCache: true,
64+
promptCacheTtlSupport: ["5m", "1h"],
5965
inputPrice: 3.0, // $3 per million input tokens (≤200K context)
6066
outputPrice: 15.0, // $15 per million output tokens (≤200K context)
6167
cacheWritesPrice: 3.75, // $3.75 per million tokens
@@ -79,6 +85,7 @@ export const anthropicModels = {
7985
contextWindow: 200_000, // Default 200K, extendable to 1M with beta flag 'context-1m-2025-08-07'
8086
supportsImages: true,
8187
supportsPromptCache: true,
88+
promptCacheTtlSupport: ["5m", "1h"],
8289
inputPrice: 3.0, // $3 per million input tokens (≤200K context)
8390
outputPrice: 15.0, // $15 per million output tokens (≤200K context)
8491
cacheWritesPrice: 3.75, // $3.75 per million tokens
@@ -102,6 +109,7 @@ export const anthropicModels = {
102109
contextWindow: 200_000, // Default 200K, extendable to 1M with beta flag
103110
supportsImages: true,
104111
supportsPromptCache: true,
112+
promptCacheTtlSupport: ["5m", "1h"],
105113
inputPrice: 5.0, // $5 per million input tokens (≤200K context)
106114
outputPrice: 25.0, // $25 per million output tokens (≤200K context)
107115
cacheWritesPrice: 6.25, // $6.25 per million tokens
@@ -125,6 +133,7 @@ export const anthropicModels = {
125133
contextWindow: 1_000_000,
126134
supportsImages: true,
127135
supportsPromptCache: true,
136+
promptCacheTtlSupport: ["5m", "1h"],
128137
inputPrice: 5.0, // $5 per million input tokens
129138
outputPrice: 25.0, // $25 per million output tokens
130139
cacheWritesPrice: 6.25, // $6.25 per million tokens
@@ -143,6 +152,7 @@ export const anthropicModels = {
143152
contextWindow: 1_000_000, // 1M context window native (no beta header required, same as 4.7)
144153
supportsImages: true,
145154
supportsPromptCache: true,
155+
promptCacheTtlSupport: ["5m", "1h"],
146156
inputPrice: 5.0, // $5 per million input tokens (regular tier)
147157
outputPrice: 25.0, // $25 per million output tokens (regular tier)
148158
cacheWritesPrice: 6.25, // $6.25 per million tokens
@@ -162,6 +172,7 @@ export const anthropicModels = {
162172
contextWindow: 1_000_000, // 1M context window native (no beta header required)
163173
supportsImages: true,
164174
supportsPromptCache: true,
175+
promptCacheTtlSupport: ["5m", "1h"],
165176
inputPrice: 5.0, // $5 per million input tokens
166177
outputPrice: 25.0, // $25 per million output tokens
167178
cacheWritesPrice: 6.25, // $6.25 per million tokens
@@ -181,6 +192,7 @@ export const anthropicModels = {
181192
contextWindow: 1_000_000, // 1M context window native (no beta header required)
182193
supportsImages: true,
183194
supportsPromptCache: true,
195+
promptCacheTtlSupport: ["5m", "1h"],
184196
inputPrice: 4.0, // $4 per million input tokens
185197
outputPrice: 20.0, // $20 per million output tokens
186198
cacheWritesPrice: 5.0, // $5 per million tokens
@@ -197,6 +209,7 @@ export const anthropicModels = {
197209
contextWindow: 1_000_000, // 1M context window native (no beta header required)
198210
supportsImages: true,
199211
supportsPromptCache: true,
212+
promptCacheTtlSupport: ["5m", "1h"],
200213
// Standard pricing (prompts of 100,000 tokens or fewer) from Anthropic's
201214
// pricing page. Prompts over 100,000 tokens bill at tiered rates
202215
// ($0.50 in / $2.50 out / $0.625 5m cache write / $0.05 cache read per
@@ -222,6 +235,7 @@ export const anthropicModels = {
222235
contextWindow: 1_000_000,
223236
supportsImages: true,
224237
supportsPromptCache: true,
238+
promptCacheTtlSupport: ["5m", "1h"],
225239
inputPrice: 10.0, // $10 per million input tokens
226240
outputPrice: 50.0, // $50 per million output tokens
227241
cacheWritesPrice: 12.5, // $12.50 per million tokens (5-minute cache)
@@ -240,6 +254,7 @@ export const anthropicModels = {
240254
contextWindow: 1_000_000,
241255
supportsImages: true,
242256
supportsPromptCache: true,
257+
promptCacheTtlSupport: ["5m", "1h"],
243258
inputPrice: 10.0, // $10 per million input tokens
244259
outputPrice: 50.0, // $50 per million output tokens
245260
cacheWritesPrice: 12.5, // $12.50 per million tokens
@@ -258,6 +273,7 @@ export const anthropicModels = {
258273
contextWindow: 200_000,
259274
supportsImages: true,
260275
supportsPromptCache: true,
276+
promptCacheTtlSupport: ["5m", "1h"],
261277
inputPrice: 5.0, // $5 per million input tokens
262278
outputPrice: 25.0, // $25 per million output tokens
263279
cacheWritesPrice: 6.25, // $6.25 per million tokens
@@ -270,6 +286,7 @@ export const anthropicModels = {
270286
contextWindow: 200_000,
271287
supportsImages: true,
272288
supportsPromptCache: true,
289+
promptCacheTtlSupport: ["5m", "1h"],
273290
inputPrice: 15.0, // $15 per million input tokens
274291
outputPrice: 75.0, // $75 per million output tokens
275292
cacheWritesPrice: 18.75, // $18.75 per million tokens
@@ -282,6 +299,7 @@ export const anthropicModels = {
282299
contextWindow: 200_000,
283300
supportsImages: true,
284301
supportsPromptCache: true,
302+
promptCacheTtlSupport: ["5m", "1h"],
285303
inputPrice: 15.0, // $15 per million input tokens
286304
outputPrice: 75.0, // $75 per million output tokens
287305
cacheWritesPrice: 18.75, // $18.75 per million tokens
@@ -294,6 +312,7 @@ export const anthropicModels = {
294312
contextWindow: 200_000,
295313
supportsImages: true,
296314
supportsPromptCache: true,
315+
promptCacheTtlSupport: ["5m", "1h"],
297316
inputPrice: 3.0, // $3 per million input tokens
298317
outputPrice: 15.0, // $15 per million output tokens
299318
cacheWritesPrice: 3.75, // $3.75 per million tokens
@@ -307,6 +326,7 @@ export const anthropicModels = {
307326
contextWindow: 200_000,
308327
supportsImages: true,
309328
supportsPromptCache: true,
329+
promptCacheTtlSupport: ["5m", "1h"],
310330
inputPrice: 3.0, // $3 per million input tokens
311331
outputPrice: 15.0, // $15 per million output tokens
312332
cacheWritesPrice: 3.75, // $3.75 per million tokens
@@ -318,6 +338,7 @@ export const anthropicModels = {
318338
contextWindow: 200_000,
319339
supportsImages: true,
320340
supportsPromptCache: true,
341+
promptCacheTtlSupport: ["5m", "1h"],
321342
inputPrice: 3.0, // $3 per million input tokens
322343
outputPrice: 15.0, // $15 per million output tokens
323344
cacheWritesPrice: 3.75, // $3.75 per million tokens
@@ -329,6 +350,7 @@ export const anthropicModels = {
329350
contextWindow: 200_000,
330351
supportsImages: false,
331352
supportsPromptCache: true,
353+
promptCacheTtlSupport: ["5m", "1h"],
332354
inputPrice: 1.0,
333355
outputPrice: 5.0,
334356
cacheWritesPrice: 1.25,
@@ -340,6 +362,7 @@ export const anthropicModels = {
340362
contextWindow: 200_000,
341363
supportsImages: true,
342364
supportsPromptCache: true,
365+
promptCacheTtlSupport: ["5m", "1h"],
343366
inputPrice: 15.0,
344367
outputPrice: 75.0,
345368
cacheWritesPrice: 18.75,
@@ -351,6 +374,7 @@ export const anthropicModels = {
351374
contextWindow: 200_000,
352375
supportsImages: true,
353376
supportsPromptCache: true,
377+
promptCacheTtlSupport: ["5m", "1h"],
354378
inputPrice: 0.25,
355379
outputPrice: 1.25,
356380
cacheWritesPrice: 0.3,
@@ -362,6 +386,7 @@ export const anthropicModels = {
362386
contextWindow: 200_000,
363387
supportsImages: true,
364388
supportsPromptCache: true,
389+
promptCacheTtlSupport: ["5m", "1h"],
365390
inputPrice: 1.0,
366391
outputPrice: 5.0,
367392
cacheWritesPrice: 1.25,

0 commit comments

Comments
 (0)