|
13 | 13 | import { CacheHint, type CachePolicy, type CachePolicyObject } from "./schema/options.js" |
14 | 14 | import { LLMRequest, Message, ToolDefinition, type ContentPart, type ToolEntry } from "./schema/messages.js" |
15 | 15 | import { effortUpdate } from "./effort-updates.js" |
| 16 | +import { Result } from "effect" |
| 17 | +import { BedrockCache } from "./protocols/utils/bedrock-cache.js" |
16 | 18 |
|
17 | 19 | const AUTO: CachePolicyObject = { |
18 | 20 | tools: true, |
@@ -67,6 +69,26 @@ const gatewayPolicy = (modelID: string): CachePolicyObject => { |
67 | 69 | return NONE |
68 | 70 | } |
69 | 71 |
|
| 72 | +function automaticCachePolicy(model: LLMRequest["model"]): CachePolicyObject { |
| 73 | + if (!RESPECTS_INLINE_HINTS.has(model.route.id)) return NONE |
| 74 | + if (model.route.id === "openrouter" || model.route.id === "vercel-ai-gateway-messages") return gatewayPolicy(model.id) |
| 75 | + if (model.route.id === "alibaba-chat") return model.id.toLowerCase().startsWith("qwen") ? QWEN : NONE |
| 76 | + return AUTO |
| 77 | +} |
| 78 | + |
| 79 | +/** Reuse automatic placements, but reject a requested TTL that lowering would drop or shorten. */ |
| 80 | +export function withCacheTTL(model: LLMRequest["model"], ttlSeconds: number) { |
| 81 | + const policy = automaticCachePolicy(model) |
| 82 | + if (!policy.tools && !policy.system && !policy.messages) |
| 83 | + return Result.fail(`Route ${model.route.id} does not support cache_control for ${model.id}`) |
| 84 | + if (model.route.id === "bedrock-converse") { |
| 85 | + const supported = BedrockCache.breakpoints(model.id) |
| 86 | + if (!supported.supported || (ttlSeconds === 3600 && !supported.ttl1h)) |
| 87 | + return Result.fail(`Bedrock cache_control TTL ${ttlSeconds}s is not supported for ${model.id}`) |
| 88 | + } |
| 89 | + return Result.succeed({ ...policy, ttlSeconds }) |
| 90 | +} |
| 91 | + |
70 | 92 | const makeHint = (ttlSeconds: number | undefined): CacheHint => |
71 | 93 | ttlSeconds !== undefined ? new CacheHint({ type: "ephemeral", ttlSeconds }) : new CacheHint({ type: "ephemeral" }) |
72 | 94 |
|
@@ -174,14 +196,9 @@ export const applyCachePolicy = (request: LLMRequest): LLMRequest => { |
174 | 196 | const route = request.model.route.id |
175 | 197 | if (!RESPECTS_INLINE_HINTS.has(route)) return request |
176 | 198 | const policy = |
177 | | - (route === "openrouter" || route === "vercel-ai-gateway-messages") && |
178 | | - (request.cache === undefined || request.cache === "auto") |
179 | | - ? gatewayPolicy(request.model.id) |
180 | | - : route === "alibaba-chat" && (request.cache === undefined || request.cache === "auto") |
181 | | - ? request.model.id.toLowerCase().startsWith("qwen") |
182 | | - ? QWEN |
183 | | - : NONE |
184 | | - : resolve(request.cache) |
| 199 | + request.cache === undefined || request.cache === "auto" |
| 200 | + ? automaticCachePolicy(request.model) |
| 201 | + : resolve(request.cache) |
185 | 202 | if (!policy.tools && !policy.system && !policy.messages) return request |
186 | 203 |
|
187 | 204 | const hint = makeHint(policy.ttlSeconds) |
|
0 commit comments