Skip to content

Commit 4672743

Browse files
fix(cloudflare-ai-gateway): curate reasoning_options, interleaved, and output limits per lab/peer baselines
The generator was deriving reasoning_options straight from catalog/docs schema enums instead of matching each host's real native controls, silently dropping interleaved fields and toggle wire comments on every regen, and writing limit.output from a catalog field that's verified wrong against lab/first-party for several models (gpt-5.5, claude-haiku-4.5, gpt-oss-20b, and more). - curation.toml gains interleaved and note (leading wire-path comment) fields; both proxied and hosted loops now consult them - proxied/hosted limit.output no longer derives from an unreliable field — curated only, so it correctly inherits from base_model when not host-specific - restored the hand-verified reasoning_options/interleaved/output/wire-comments that existed before this rework for gpt-oss-20b/120b, gemma-4-26b-a4b-it, kimi-k2.6, nemotron-3-120b-a12b, glm-4.7-flash, glm-5.2, and backfilled limit.output for every other previously-verified @cf model - fixed proxied deepseek-v4-pro, qwen3.7-plus, and grok-4.6 to match their lab/first-party reasoning controls instead of a generic schema-derived effort enum Verified live against Cloudflare (bun run cloudflare-ai-gateway:generate --check passes, bun validate passes). Co-Authored-By: Claude Sonnet 5 <[email protected]>
1 parent 581312b commit 4672743

63 files changed

Lines changed: 195 additions & 64 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎packages/core/script/generate-cloudflare-ai-gateway.ts‎

Lines changed: 38 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,13 @@ const CuratedModel = z
5353
structured_output: z.boolean().optional(),
5454
reasoning_options: z.array(ReasoningOption).optional(),
5555
limit: z.record(z.number()).optional(),
56+
interleaved: z
57+
.union([z.literal(true), z.object({ field: z.enum(["reasoning_content", "reasoning_details"]) }).strict()])
58+
.optional(),
59+
// Leading `#` comment lines, e.g. a toggle/effort wire-path note (AGENTS.md requires one
60+
// for every `toggle` reasoning option) or a source citation. Rendered verbatim above the
61+
// generated fields so hand-verified host behavior survives every regeneration.
62+
note: z.array(z.string()).optional(),
5663
})
5764
.strict();
5865

@@ -296,6 +303,14 @@ function deriveReasoningOptions(schemaInput: unknown): Array<Record<string, unkn
296303
return opts;
297304
}
298305

306+
// Prepend curated leading `#` comment lines (a toggle/effort wire-path note, a source
307+
// citation, etc.) above the generated content. AGENTS.md requires one for every `toggle`
308+
// reasoning option since sync strips mid-file comments on every regeneration.
309+
function withNote(note: string[] | undefined, content: string): string {
310+
if (!note || note.length === 0) return content;
311+
return `${note.map((line) => `# ${line}`).join("\n")}\n\n${content}`;
312+
}
313+
299314
// ---------------------------------------------------------------------------
300315
// base_model resolution
301316
// ---------------------------------------------------------------------------
@@ -361,6 +376,7 @@ async function main() {
361376
// the catalog only carries Cloudflare's own casing/marketing variants.
362377
const model: Record<string, unknown> = { base_model: base };
363378
if (cur.structured_output !== undefined) model.structured_output = cur.structured_output;
379+
if (cur.interleaved !== undefined) model.interleaved = cur.interleaved;
364380

365381
// reasoning_options: only meaningful when the base actually reasons. The catalog schema
366382
// advertises reasoning_effort for some non-reasoning models (gpt-4.1, gpt-4o) — schema
@@ -384,15 +400,16 @@ async function main() {
384400
if (Object.keys(cost).length === 0) errors.push(`proxied ${id}: catalog pricing empty`);
385401
model.cost = cost;
386402

403+
// limit.output: the catalog's max_output_tokens is not a reliable ceiling — verified wrong
404+
// against lab/first-party for gpt-5, gpt-5.5, and claude-haiku-4.5 (all understated by
405+
// 4-8x). Only context_length has checked out, so that's all we auto-derive; output is
406+
// either curated explicitly or left to inherit from base_model.
387407
const limit: Record<string, number> = {};
388408
if (cur.limit) Object.assign(limit, cur.limit);
389-
else {
390-
if (m.context_length != null) limit.context = m.context_length;
391-
if (m.max_output_tokens != null) limit.output = m.max_output_tokens;
392-
}
409+
else if (m.context_length != null) limit.context = m.context_length;
393410
if (Object.keys(limit).length) model.limit = limit;
394411

395-
wanted.set(path.join(MODELS_DIR, `${id}.toml`), formatToml(model as any));
412+
wanted.set(path.join(MODELS_DIR, `${id}.toml`), withNote(cur.note, formatToml(model as any)));
396413
}
397414

398415
// --- hosted @cf models ---
@@ -417,6 +434,10 @@ async function main() {
417434
if (hostedProp(docs, "function_calling") === "true") model.tool_call = true;
418435
if (hostedProp(docs, "vision") === "true") model.attachment = true;
419436
if (cur.structured_output !== undefined) model.structured_output = cur.structured_output;
437+
// interleaved: whether this @cf deployment returns reasoning content as a separate wire
438+
// field. Not part of the docs schema and not inheritable from base_model (the lab's own
439+
// API frequently differs), so it only ever comes from curation.
440+
if (cur.interleaved !== undefined) model.interleaved = cur.interleaved;
420441

421442
// reasoning_options: only when the base reasons (schema forbids them otherwise). Derived
422443
// from the docs input schema; curation may override for the rare model whose docs schema
@@ -439,10 +460,20 @@ async function main() {
439460
const cost = hostedCost(docs);
440461
if (cost) model.cost = cost;
441462

463+
// limit.output: the docs schema does not expose a hosted output ceiling, and Workers AI's
464+
// own deployment cap is frequently lower than the lab's SaaS limit (e.g. gpt-oss-20b caps
465+
// at 16_384 here vs 32_768 for the lab API) — inheriting from base_model would overstate
466+
// it. Only curation (hand-verified per model) can supply it.
442467
const context = hostedProp(docs, "context_window");
443-
if (context != null) model.limit = { context: Number(context) };
468+
const limit: Record<string, number> = {};
469+
if (context != null) limit.context = Number(context);
470+
if (cur.limit) Object.assign(limit, cur.limit);
471+
if (Object.keys(limit).length) model.limit = limit;
444472

445-
wanted.set(path.join(MODELS_DIR, `workers-ai/${cfId}.toml`), formatToml(model as any));
473+
wanted.set(
474+
path.join(MODELS_DIR, `workers-ai/${cfId}.toml`),
475+
withNote(cur.note, formatToml(model as any)),
476+
);
446477
}
447478

448479
// Guards: a curation model id that no longer appears in the live feeds (warn only).

‎providers/cloudflare-ai-gateway/curation.toml‎

Lines changed: 81 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,15 @@
88
# - structured_output: a quality judgement. Cloudflare advertises response_format for
99
# nearly every model, but several do not honour it (verified live: qwq-32b, qwen2.5-
1010
# coder, gemma-sea-lion return non-conforming output), so this stays hand-set.
11-
# - reasoning_options: proxied only (the catalog has no reasoning schema).
12-
# - limit: proxied context/output that deliberately differ from the catalog.
11+
# - reasoning_options: proxied (catalog has no reasoning schema) and any hosted @cf model
12+
# whose docs schema doesn't describe its real reasoning knob (native, non-graded, etc.).
13+
# - interleaved: whether a wire returns reasoning content as a separate field. Never
14+
# inheritable from base_model (this deployment's wire format can differ from the lab's).
15+
# - limit.output: neither the catalog's max_output_tokens nor the docs schema is a reliable
16+
# ceiling (both verified wrong against lab/first-party for several models), so output is
17+
# always hand-curated; only limit.context is auto-derived.
18+
# - note: leading `#` comment lines (wire-path docs, source citations) that would otherwise
19+
# be lost on every regeneration.
1320
# Consumed by: bun run cloudflare-ai-gateway:generate
1421

1522
# Proxied Text-Generation catalog ids with no lab file under models/ (cannot publish yet).
@@ -115,8 +122,26 @@ reasoning_options = [{ type = "effort", values = ["low", "medium", "high"] }]
115122
# Proxied reasoning models whose reasoning shape the Cloudflare catalog schema does not
116123
# expose (native-format providers). Values mirror models.dev's merge-gateway provider, which
117124
# proxies the same upstreams. Google uses thinkingConfig.thinkingBudget / thinking_level.
125+
#
126+
# DeepSeek: toggle-only would only be correct if AI Gateway drops the effort field on this
127+
# route (merge-gateway's own toggle-only shape is backed by a live-tested selected-route
128+
# comment specific to *their* relay, not evidence about Cloudflare's). Absent host-specific
129+
# proof that Cloudflare strips it, match the lab + Workers AI baseline: toggle + high/max.
118130
[models."deepseek/deepseek-v4-pro"]
119-
reasoning_options = [{ type = "toggle" }]
131+
note = ["Toggle: thinking.type = enabled|disabled.", "Effort: reasoning_effort = high|max."]
132+
reasoning_options = [{ type = "toggle" }, { type = "effort", values = ["high", "max"] }]
133+
134+
# Alibaba is a first-party lab; the catalog's OpenAI-compat schema exposes a generic
135+
# `reasoning_effort` enum, but Alibaba's real API (see providers/alibaba/models/qwen3.7-plus.toml)
136+
# is toggle + budget_tokens, not graded effort. Match the lab shape per AGENTS.md.
137+
[models."alibaba/qwen3.7-plus"]
138+
note = ["Toggle: enable_thinking true|false.", "Budget: thinking_budget (integer reasoning tokens)."]
139+
reasoning_options = [{ type = "toggle" }, { type = "budget_tokens" }]
140+
141+
# First-party xAI Grok 4.6 (providers/xai/models/grok-4.6.toml) includes xhigh; the catalog
142+
# schema enum is missing it.
143+
[models."xai/grok-4.6"]
144+
reasoning_options = [{ type = "effort", values = ["low", "medium", "high", "xhigh"] }]
120145

121146
[models."google/gemini-2.5-pro"]
122147
reasoning_options = [{ type = "budget_tokens", min = 128, max = 32_768 }]
@@ -157,39 +182,55 @@ base_model = "deepseek/deepseek-r1-distill-qwen-32b"
157182
structured_output = false
158183
# Always-on reasoner with no user-facing knob; docs schema exposes no reasoning parameter.
159184
reasoning_options = []
185+
limit = { output = 80_000 }
160186

161187
[models."workers-ai/@cf/deepseek-ai/deepseek-v4-flash-0731"]
162188
base_model = "deepseek/deepseek-v4-flash-0731"
163189

164190
[models."workers-ai/@cf/deepseek-ai/deepseek-v4-pro-0813"]
165191
base_model = "deepseek/deepseek-v4-pro-0813"
192+
note = ["Toggle: thinking.type = enabled|disabled.", "Effort: reasoning_effort = high|max."]
193+
reasoning_options = [{ type = "toggle" }, { type = "effort", values = ["high", "max"] }]
166194

167195
[models."workers-ai/@cf/google/gemma-4-26b-a4b-it"]
168196
base_model = "google/gemma-4-26b-a4b-it"
197+
note = [
198+
"Native `/ai/run` accepts `reasoning_effort = low|medium|high` and",
199+
"`chat_template_kwargs.enable_thinking = true|false`; no budget is documented.",
200+
"https://developers.cloudflare.com/workers-ai/models/gemma-4-26b-a4b-it/sync-input.json (accessed 2026-06-25)",
201+
]
202+
interleaved = true
203+
limit = { output = 16_384 }
169204

170205
[models."workers-ai/@cf/ibm-granite/granite-4.0-h-micro"]
171206
base_model = "ibm/granite-4-h-micro"
172207
structured_output = false
208+
limit = { output = 131_000 }
173209

174210
[models."workers-ai/@cf/meta/llama-3.1-8b-instruct-fp8"]
175211
base_model = "meta/llama-3.1-8b-instruct"
176212
structured_output = false
213+
limit = { output = 32_000 }
177214

178215
[models."workers-ai/@cf/meta/llama-3.2-11b-vision-instruct"]
179216
base_model = "meta/llama-3.2-11b-vision-instruct"
180217
structured_output = false
218+
limit = { output = 128_000 }
181219

182220
[models."workers-ai/@cf/meta/llama-3.2-1b-instruct"]
183221
base_model = "meta/llama-3.2-1b"
184222
structured_output = false
223+
limit = { output = 60_000 }
185224

186225
[models."workers-ai/@cf/meta/llama-3.2-3b-instruct"]
187226
base_model = "meta/llama-3.2-3b"
188227
structured_output = false
228+
limit = { output = 80_000 }
189229

190230
[models."workers-ai/@cf/meta/llama-3.3-70b-instruct-fp8-fast"]
191231
base_model = "meta/llama-3.3-70b-instruct"
192232
structured_output = false
233+
limit = { output = 24_000 }
193234

194235
[models."workers-ai/@cf/meta/llama-4-scout-17b-16e-instruct"]
195236
base_model = "meta/llama-4-scout-17b-instruct"
@@ -198,40 +239,73 @@ structured_output = false
198239
[models."workers-ai/@cf/meta/llama-guard-3-8b"]
199240
base_model = "meta/llama-guard-3-8b"
200241
structured_output = false
242+
limit = { output = 131_072 }
201243

202244
[models."workers-ai/@cf/mistralai/mistral-small-3.1-24b-instruct"]
203245
base_model = "mistral/mistral-small-3-1-24b-instruct-2503"
204246
structured_output = false
247+
limit = { output = 128_000 }
205248

206249
[models."workers-ai/@cf/moonshotai/kimi-k2.6"]
207250
base_model = "moonshotai/kimi-k2.6"
251+
note = [
252+
"Native `/ai/run` accepts `reasoning_effort = low|medium|high` and",
253+
"`chat_template_kwargs.thinking = true|false`; no budget is documented.",
254+
"https://developers.cloudflare.com/workers-ai/models/kimi-k2.6/sync-input.json (accessed 2026-06-25)",
255+
]
256+
interleaved = { field = "reasoning_content" }
257+
limit = { output = 256_000 }
208258

209259
[models."workers-ai/@cf/moonshotai/kimi-k2.7-code"]
210260
base_model = "moonshotai/kimi-k2.7-code"
211261

212262
[models."workers-ai/@cf/nvidia/nemotron-3-120b-a12b"]
213263
base_model = "nvidia/nemotron-3-super-120b-a12b"
214264
structured_output = true
265+
note = [
266+
"Native `/ai/run` accepts `reasoning_effort = low|medium|high` and",
267+
"`chat_template_kwargs.enable_thinking = true|false`; no budget is documented.",
268+
"https://developers.cloudflare.com/workers-ai/models/nemotron-3-120b-a12b/sync-input.json (accessed 2026-06-25)",
269+
]
270+
interleaved = true
271+
limit = { output = 256_000 }
215272

216273
[models."workers-ai/@cf/openai/gpt-oss-120b"]
217274
base_model = "openai/gpt-oss-120b"
275+
note = [
276+
"Native `/ai/run` schema has no reasoning control, and no model-specific",
277+
"OpenAI-compatible reasoning field was verified. No token budget is documented.",
278+
"https://developers.cloudflare.com/workers-ai/models/gpt-oss-120b/sync-input.json (accessed 2026-06-25)",
279+
]
280+
reasoning_options = []
281+
limit = { output = 16_384 }
218282

219283
[models."workers-ai/@cf/openai/gpt-oss-20b"]
220284
base_model = "openai/gpt-oss-20b"
285+
note = [
286+
"Native `/ai/run` schema has no reasoning control, and no model-specific",
287+
"OpenAI-compatible reasoning field was verified. No token budget is documented.",
288+
"https://developers.cloudflare.com/workers-ai/models/gpt-oss-20b/sync-input.json (accessed 2026-06-25)",
289+
]
290+
reasoning_options = []
291+
limit = { output = 16_384 }
221292

222293
[models."workers-ai/@cf/qwen/qwen2.5-coder-32b-instruct"]
223294
base_model = "alibaba/qwen2.5-coder-32b-instruct"
224295
structured_output = false
296+
limit = { output = 32_768 }
225297

226298
[models."workers-ai/@cf/qwen/qwen3-30b-a3b-fp8"]
227299
base_model = "alibaba/qwen3-30b-a3b"
228300
structured_output = false
229301
reasoning_options = []
302+
limit = { output = 32_768 }
230303

231304
[models."workers-ai/@cf/qwen/qwq-32b"]
232305
base_model = "alibaba/qwq-32b"
233306
structured_output = false
234307
reasoning_options = []
308+
limit = { output = 24_000 }
235309

236310
# structured_output inherited (base=true, verified live: honours response_format json_schema);
237311
# reasoning_options (toggle + effort) derived from the docs input schema.
@@ -241,7 +315,11 @@ base_model = "alibaba/qwen3.8-27b"
241315
[models."workers-ai/@cf/zai-org/glm-4.7-flash"]
242316
base_model = "zhipuai/glm-4.7-flash"
243317
structured_output = true
318+
interleaved = { field = "reasoning_content" }
319+
limit = { output = 131_072 }
244320

245321
[models."workers-ai/@cf/zai-org/glm-5.2"]
246322
base_model = "zhipuai/glm-5.2"
323+
note = ["Context and output limits verified against Cloudflare Workers AI's OpenAI-compatible endpoint."]
324+
limit = { output = 256_000 }
247325

‎providers/cloudflare-ai-gateway/models/alibaba/qwen3.7-plus.toml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,13 @@
1+
# Toggle: enable_thinking true|false.
2+
# Budget: thinking_budget (integer reasoning tokens).
3+
14
base_model = "alibaba/qwen3.7-plus"
25

36
[[reasoning_options]]
4-
type = "effort"
5-
values = ["none", "low", "medium", "high"]
7+
type = "toggle"
8+
9+
[[reasoning_options]]
10+
type = "budget_tokens"
611

712
[cost]
813
input = 0.32

‎providers/cloudflare-ai-gateway/models/anthropic/claude-fable-5.toml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,3 @@ cache_write = 12.5
1313

1414
[limit]
1515
context = 1_000_000
16-
output = 128_000

‎providers/cloudflare-ai-gateway/models/anthropic/claude-haiku-4.5.toml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,3 @@ cache_write = 1.25
1313

1414
[limit]
1515
context = 200_000
16-
output = 8_192

‎providers/cloudflare-ai-gateway/models/anthropic/claude-opus-4.5.toml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,3 @@ cache_write = 6.25
1717

1818
[limit]
1919
context = 200_000
20-
output = 32_000

‎providers/cloudflare-ai-gateway/models/anthropic/claude-opus-4.6.toml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,3 @@ cache_write = 6.25
1717

1818
[limit]
1919
context = 1_000_000
20-
output = 128_000

‎providers/cloudflare-ai-gateway/models/anthropic/claude-opus-4.7.toml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,3 @@ cache_write = 6.25
1313

1414
[limit]
1515
context = 1_000_000
16-
output = 128_000

‎providers/cloudflare-ai-gateway/models/anthropic/claude-opus-4.8.toml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,3 @@ cache_write = 6.25
1313

1414
[limit]
1515
context = 1_000_000
16-
output = 128_000

‎providers/cloudflare-ai-gateway/models/anthropic/claude-opus-5.toml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,3 @@ cache_write = 6.25
1313

1414
[limit]
1515
context = 1_000_000
16-
output = 128_000

0 commit comments

Comments
 (0)