Skip to content

Commit 556825f

Browse files
committed
fix(core): address prompt cache review feedback
1 parent 30ac5d7 commit 556825f

13 files changed

Lines changed: 551 additions & 335 deletions

‎packages/ai/src/cache-policy.ts‎

Lines changed: 25 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
import { CacheHint, type CachePolicy, type CachePolicyObject } from "./schema/options.js"
1414
import { LLMRequest, Message, ToolDefinition, type ContentPart, type ToolEntry } from "./schema/messages.js"
1515
import { effortUpdate } from "./effort-updates.js"
16+
import { Result } from "effect"
17+
import { BedrockCache } from "./protocols/utils/bedrock-cache.js"
1618

1719
const AUTO: CachePolicyObject = {
1820
tools: true,
@@ -67,6 +69,26 @@ const gatewayPolicy = (modelID: string): CachePolicyObject => {
6769
return NONE
6870
}
6971

72+
function automaticCachePolicy(model: LLMRequest["model"]): CachePolicyObject {
73+
if (!RESPECTS_INLINE_HINTS.has(model.route.id)) return NONE
74+
if (model.route.id === "openrouter" || model.route.id === "vercel-ai-gateway-messages") return gatewayPolicy(model.id)
75+
if (model.route.id === "alibaba-chat") return model.id.toLowerCase().startsWith("qwen") ? QWEN : NONE
76+
return AUTO
77+
}
78+
79+
/** Reuse automatic placements, but reject a requested TTL that lowering would drop or shorten. */
80+
export function withCacheTTL(model: LLMRequest["model"], ttlSeconds: number) {
81+
const policy = automaticCachePolicy(model)
82+
if (!policy.tools && !policy.system && !policy.messages)
83+
return Result.fail(`Route ${model.route.id} does not support cache_control for ${model.id}`)
84+
if (model.route.id === "bedrock-converse") {
85+
const supported = BedrockCache.breakpoints(model.id)
86+
if (!supported.supported || (ttlSeconds === 3600 && !supported.ttl1h))
87+
return Result.fail(`Bedrock cache_control TTL ${ttlSeconds}s is not supported for ${model.id}`)
88+
}
89+
return Result.succeed({ ...policy, ttlSeconds })
90+
}
91+
7092
const makeHint = (ttlSeconds: number | undefined): CacheHint =>
7193
ttlSeconds !== undefined ? new CacheHint({ type: "ephemeral", ttlSeconds }) : new CacheHint({ type: "ephemeral" })
7294

@@ -174,14 +196,9 @@ export const applyCachePolicy = (request: LLMRequest): LLMRequest => {
174196
const route = request.model.route.id
175197
if (!RESPECTS_INLINE_HINTS.has(route)) return request
176198
const policy =
177-
(route === "openrouter" || route === "vercel-ai-gateway-messages") &&
178-
(request.cache === undefined || request.cache === "auto")
179-
? gatewayPolicy(request.model.id)
180-
: route === "alibaba-chat" && (request.cache === undefined || request.cache === "auto")
181-
? request.model.id.toLowerCase().startsWith("qwen")
182-
? QWEN
183-
: NONE
184-
: resolve(request.cache)
199+
request.cache === undefined || request.cache === "auto"
200+
? automaticCachePolicy(request.model)
201+
: resolve(request.cache)
185202
if (!policy.tools && !policy.system && !policy.messages) return request
186203

187204
const hint = makeHint(policy.ttlSeconds)

‎packages/ai/src/prompt-cache.ts‎

Lines changed: 5 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
export * as PromptCache from "./prompt-cache.js"
22

33
import type { Options } from "@opencode/schema/prompt-cache"
4-
import { Effect } from "effect"
4+
import { Effect, Result } from "effect"
5+
import { withCacheTTL } from "./cache-policy.js"
56
import { LLMRequest } from "./schema/messages.js"
67
import { ProviderShared } from "./protocols/shared.js"
78

@@ -14,31 +15,16 @@ export const apply = Effect.fn("PromptCache.apply")(function* (request: LLMReque
1415
if (control !== undefined && (retention !== undefined || cache !== undefined))
1516
return yield* ProviderShared.invalidRequest("cache_control cannot be combined with OpenAI prompt cache options")
1617
if (control !== undefined) {
17-
if (route !== "anthropic-messages" && route !== "google-vertex-messages" && route !== "bedrock-mantle-messages")
18-
return yield* ProviderShared.invalidRequest(`cache_control is not supported by cache rules for route ${route}`)
19-
return LLMRequest.update(request, {
20-
cache: { tools: true, system: true, messages: { tail: 1 }, ttlSeconds: control.ttl === "1h" ? 3600 : 300 },
21-
})
18+
const policy = withCacheTTL(request.model, control.ttl === "1h" ? 3600 : 300)
19+
if (Result.isFailure(policy)) return yield* ProviderShared.invalidRequest(policy.failure)
20+
return LLMRequest.update(request, { cache: policy.success })
2221
}
2322
if (retention === undefined && cache === undefined) return request
2423
if (route !== "openai-chat" && route !== "openai-responses")
2524
return yield* ProviderShared.invalidRequest(
2625
`OpenAI prompt cache options are not supported by cache rules for route ${route}`,
2726
)
2827

29-
// Native model IDs expose these known incompatibilities; deployment aliases and account restrictions
30-
// are validated by the API. Never infer support from the configured provider's user-chosen name.
31-
const gpt = /^gpt-(\d+)(?:\.(\d+))?(?:-|$)/.exec(request.model.id)
32-
const version = gpt ? ([Number(gpt[1]), Number(gpt[2] ?? 0)] as const) : undefined
33-
if (cache !== undefined && version && (version[0] < 5 || (version[0] === 5 && version[1] < 6)))
34-
return yield* ProviderShared.invalidRequest(
35-
`${request.model.id} does not support prompt_cache_options; use prompt_cache_retention or restrict the rule to GPT-5.6 and later`,
36-
)
37-
if (retention === "in_memory" && version && (version[0] > 5 || (version[0] === 5 && version[1] >= 5)))
38-
return yield* ProviderShared.invalidRequest(`${request.model.id} only supports prompt_cache_retention: "24h"`)
39-
if (retention !== undefined && /^gpt-[123](?:\.|-|$)/.test(request.model.id))
40-
return yield* ProviderShared.invalidRequest(`${request.model.id} does not support prompt_cache_retention`)
41-
4228
return LLMRequest.update(request, {
4329
providerOptions: {
4430
...request.providerOptions,

0 commit comments

Comments
 (0)