Skip to content

Commit a22754f

Browse files
authored
fix(core): support provider-level compatibility and migrate setCacheKey (#54911)
1 parent 1438073 commit a22754f

21 files changed

Lines changed: 372 additions & 53 deletions

File tree

‎packages/ai/src/protocols/mistral-chat.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -417,7 +417,9 @@ export const fromRequest = Effect.fn("MistralChat.fromRequest")(function* (reque
417417
frequency_penalty: request.generation?.frequencyPenalty,
418418
presence_penalty: request.generation?.presencePenalty,
419419
stop: request.generation?.stop,
420-
prompt_cache_key: request.cache === "none" ? undefined : (options.promptCacheKey ?? request.promptCacheKey),
420+
prompt_cache_key: ProviderShared.sendsPromptCacheKey(request)
421+
? (options.promptCacheKey ?? request.promptCacheKey)
422+
: undefined,
421423
safe_prompt: options.safePrompt,
422424
document_image_limit: options.documentImageLimit,
423425
document_page_limit: options.documentPageLimit,

‎packages/ai/src/protocols/shared.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,15 @@ export const knownString = <Known extends string>() =>
3939

4040
export const OPENAI_PROMPT_CACHE_KEY_MAX_LENGTH = 64
4141

42+
// Body-level `prompt_cache_key` is off when caching is disabled or the model
43+
// explicitly rejects the field. Header-based affinity is unaffected.
44+
export const sendsPromptCacheKey = (request: LLMRequest) =>
45+
request.cache !== "none" && request.model.compatibility?.supportsPromptCacheKey !== false
46+
4247
// OpenAI limits `prompt_cache_key` to 64 chars; DeepSeek and Zai inherit the same
4348
// limit via their OpenAI-compatible APIs. Clamp with unicode-aware slicing.
4449
export const promptCacheKey = (request: LLMRequest): string | undefined => {
45-
if (request.cache === "none" || request.promptCacheKey === undefined) return undefined
50+
if (!sendsPromptCacheKey(request) || request.promptCacheKey === undefined) return undefined
4651
const chars = Array.from(request.promptCacheKey)
4752
if (chars.length <= OPENAI_PROMPT_CACHE_KEY_MAX_LENGTH) return request.promptCacheKey
4853
return chars.slice(0, OPENAI_PROMPT_CACHE_KEY_MAX_LENGTH).join("")

‎packages/ai/src/schema/options.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -199,8 +199,9 @@ export class LanguageModelCompatibility extends Schema.Class<LanguageModelCompat
199199
supportsStore: Schema.optional(Schema.Boolean),
200200
supportsUsageInStreaming: Schema.optional(Schema.Boolean),
201201
supportsStrictMode: Schema.optional(Schema.Boolean),
202-
// Accepts `prompt_cache_key` in the Chat Completions body. Chat omits the
203-
// key unless this is set; session-affinity headers still flow regardless.
202+
// Accepts `prompt_cache_key` in the request body. Chat omits the key unless
203+
// this is true; other protocols omit it only when this is false.
204+
// Session-affinity headers still flow regardless.
204205
supportsPromptCacheKey: Schema.optional(Schema.Boolean),
205206
zaiToolStream: Schema.optional(Schema.Boolean),
206207
requireSignature: Schema.optional(Schema.Boolean),

‎packages/ai/test/provider/mistral-chat.test.ts‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { describe, expect, test } from "bun:test"
22
import { ConfigProvider, Effect } from "effect"
33
import { HttpClientRequest } from "effect/http"
4-
import { LLM, LLMEvent, Message, SystemPart, ToolDefinition, Media } from "../../src/index.js"
4+
import { LanguageModel, LLM, LLMEvent, Message, SystemPart, ToolDefinition, Media } from "../../src/index.js"
55
import { Mistral } from "../../src/providers/index.js"
66
import { MistralChat } from "../../src/protocols/index.js"
77
import { LLMClient } from "../../src/route.js"
@@ -188,6 +188,16 @@ describe("Mistral Chat", () => {
188188
)
189189
expect(uncached.body.prompt_cache_key).toBeUndefined()
190190

191+
const unsupported = yield* compileRequest(
192+
LLM.request({
193+
model: LanguageModel.update(model, { compatibility: { supportsPromptCacheKey: false } }),
194+
prompt: "Hello",
195+
promptCacheKey: "common-key",
196+
providerOptions: { promptCacheKey: "native-key" },
197+
}),
198+
)
199+
expect(unsupported.body.prompt_cache_key).toBeUndefined()
200+
191201
const longKey = "cache-key-".repeat(10)
192202
const unbounded = yield* compileRequest(
193203
LLM.request({

‎packages/ai/test/provider/openai-responses.test.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2117,6 +2117,24 @@ describe("OpenAI Responses route", () => {
21172117
}),
21182118
)
21192119

2120+
it.effect("omits the prompt cache key when compatibility disables it", () =>
2121+
Effect.gen(function* () {
2122+
const prepared = yield* compileRequest(
2123+
LLM.request({
2124+
model: LanguageModel.update(model, { compatibility: { supportsPromptCacheKey: false } }),
2125+
prompt: "Hello",
2126+
promptCacheKey: "request_cache",
2127+
}),
2128+
)
2129+
2130+
expect(prepared.body).not.toHaveProperty("prompt_cache_key")
2131+
expect(model.compatibility?.supportsPromptCacheKey).toBeUndefined()
2132+
expect(
2133+
(yield* compileRequest(LLM.request({ model, prompt: "Hello", promptCacheKey: "request_cache" }))).body,
2134+
).toHaveProperty("prompt_cache_key", "request_cache")
2135+
}),
2136+
)
2137+
21202138
it.effect("parses text and usage stream fixtures", () =>
21212139
Effect.gen(function* () {
21222140
const body = sseEvents(

‎packages/client/src/promise/generated/types.ts‎

Lines changed: 23 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -630,6 +630,15 @@ export type ModelCompatibility = {
630630
supportsEffortUpdates?: boolean
631631
}
632632

633+
export type ProviderCompatibility = {
634+
reasoningField?: ModelReasoningField
635+
requireReasoning?: boolean
636+
maxTokensField?: ModelMaxTokensField
637+
requireFinishReason?: boolean
638+
requireAssistantAfterTool?: boolean
639+
supportsPromptCacheKey?: boolean
640+
}
641+
633642
export type ModelCost = {
634643
tier?: { type: "context"; size: number }
635644
input: MoneyUSDPerMillionTokens
@@ -887,18 +896,6 @@ export type ProviderRequest = {
887896
body: { [x: string]: any }
888897
}
889898

890-
export type ProviderInfo = {
891-
id: ProviderID
892-
canonical?: ProviderID
893-
integrationID?: IntegrationID
894-
name: string
895-
activation: "auto" | "enabled" | "disabled"
896-
package: string
897-
settings?: ProviderSettings
898-
headers?: { [x: string]: string }
899-
body?: { [x: string]: any }
900-
}
901-
902899
export type PermissionRuleset = Array<PermissionRule>
903900

904901
export type SessionMessageUser = {
@@ -1912,6 +1909,19 @@ export type CredentialKey = {
19121909
configuration?: FormAnswer
19131910
}
19141911

1912+
export type ProviderInfo = {
1913+
id: ProviderID
1914+
canonical?: ProviderID
1915+
integrationID?: IntegrationID
1916+
name: string
1917+
activation: "auto" | "enabled" | "disabled"
1918+
package: string
1919+
compatibility?: ProviderCompatibility
1920+
settings?: ProviderSettings
1921+
headers?: { [x: string]: string }
1922+
body?: { [x: string]: any }
1923+
}
1924+
19151925
export type ConnectionInfo = ConnectionCredentialInfo | ConnectionEnvInfo
19161926

19171927
export type PtyCreated = {
@@ -2241,6 +2251,7 @@ export type ConfigEntry =
22412251
name?: string
22422252
env?: Array<string>
22432253
package?: string
2254+
compatibility?: ProviderCompatibility
22442255
settings?: ConfigProviderSettings
22452256
headers?: { [x: string]: string }
22462257
body?: { [x: string]: JsonValue }

‎packages/core/src/config/plugin/provider.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,8 @@ export const Plugin = define({
6868
if (item.canonical !== undefined) provider.canonical = item.canonical
6969
if (item.name !== undefined) provider.name = item.name
7070
if (item.package !== undefined) provider.package = item.package
71+
if (item.compatibility !== undefined)
72+
provider.compatibility = { ...provider.compatibility, ...item.compatibility }
7173
if (item.settings !== undefined) provider.settings = Provider.mergeOverlay(provider.settings, item.settings)
7274
if (item.headers !== undefined) provider.headers = Provider.mergeHeaders(provider.headers, item.headers)
7375
if (item.body !== undefined) provider.body = Provider.mergeOverlay(provider.body, item.body)

‎packages/core/src/model.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,10 @@ const layer = Layer.effect(
201201
...model,
202202
...(provider?.canonical === undefined ? {} : { canonical: provider.canonical }),
203203
package: model.package ?? provider?.package,
204+
compatibility:
205+
provider?.compatibility === undefined
206+
? model.compatibility
207+
: { ...provider.compatibility, ...model.compatibility },
204208
settings: Provider.mergeOverlay(
205209
Provider.modelSettings(provider?.settings),
206210
Provider.modelSettings(model.settings),

‎packages/core/src/plugin/provider/prompt-cache-key.ts‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
import { Effect } from "effect"
22
import { define } from "@opencode/plugin/effect/plugin"
3-
import { Provider } from "../../provider.js"
43

54
// Chat-completions `prompt_cache_key` body support for catalog providers
65
// without a dedicated AI package. Audited per provider (docs + OpenAPI +
@@ -25,14 +24,13 @@ const providerIDs = [
2524
export const PromptCacheKeyPlugin = define({
2625
id: "opencode.provider.prompt-cache-key",
2726
effect: Effect.fn(function* (ctx) {
28-
yield* ctx.model.transform((models) => {
27+
// Set on the provider so configured provider and model compatibility can override it.
28+
yield* ctx.provider.transform((providers) => {
2929
for (const id of providerIDs) {
30-
const providerID = Provider.ID.make(id)
31-
for (const model of models.list(providerID)) {
32-
models.update(model.providerID, model.id, (draft) => {
33-
draft.compatibility = { ...draft.compatibility, supportsPromptCacheKey: true }
34-
})
35-
}
30+
if (!providers.get(id)) continue
31+
providers.update(id, (provider) => {
32+
provider.compatibility = { ...provider.compatibility, supportsPromptCacheKey: true }
33+
})
3634
}
3735
})
3836
}),

‎packages/core/src/v1/config/migrate.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,11 +110,14 @@ export function migrateProvider(sourceID: string, info: ConfigProviderV1.Info) {
110110
}
111111

112112
function migrateStandardProvider(info: ConfigProviderV1.Info) {
113-
const options = ConfigProviderOptionsV1.provider(info.options ?? {})
113+
// V1 `setCacheKey` toggled `prompt_cache_key` for the whole provider; it is not a provider package setting.
114+
const { setCacheKey, ...rest } = info.options ?? {}
115+
const options = ConfigProviderOptionsV1.provider(rest)
114116
return {
115117
name: info.name,
116118
env: info.env,
117119
package: info.npm ? Provider.aisdk(info.npm) : undefined,
120+
compatibility: setCacheKey === undefined ? undefined : { supportsPromptCacheKey: setCacheKey },
118121
settings: info.api ? { ...options.settings, baseURL: info.api } : info.options ? options.settings : undefined,
119122
headers: info.options && options.headers,
120123
body: info.options && options.body,

0 commit comments

Comments
 (0)