Skip to content

Commit d6dbcdd

Browse files
committed
fix: validate release model catalog safely
1 parent 2f271de commit d6dbcdd

3 files changed

Lines changed: 407 additions & 108 deletions

File tree

‎packages/opencode/script/build.ts‎

Lines changed: 23 additions & 108 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ process.chdir(dir)
1818
import { Script } from "@opencode-ai/script"
1919
import pkg from "../package.json"
2020
import { walkInputs } from "./stamp-inputs"
21+
import { assertUsableCatalog, catalogDiagnosticOrigin, formatCatalogSummary } from "./models-catalog"
2122

2223
// Python engine has been eliminated — all methods run natively in TypeScript.
2324
// ALTIMATE_ENGINE_VERSION is no longer needed at runtime.
@@ -30,61 +31,21 @@ console.log(`Loaded CHANGELOG.md (${changelog.length} chars)`)
3031
const modelsUrlOverride = process.env.OPENCODE_MODELS_URL || undefined
3132
const modelsUrl = modelsUrlOverride ?? "https://models.dev"
3233

33-
// The providers whose absence would make a shipped binary visibly broken.
34-
const REQUIRED_CATALOG_PROVIDERS = ["anthropic", "openai", "google"]
35-
// Size floor for the real models.dev catalog: live carries 200+ providers and the
36-
// checked-in fixture 105, so anything under this is a fetch that went wrong rather
37-
// than a real shrink. Applied only in strict mode — a private catalog pinned via
38-
// OPENCODE_MODELS_URL / MODELS_DEV_API_JSON is legitimately allowed to be small.
39-
const MIN_CATALOG_PROVIDERS = 50
4034
const CATALOG_FETCH_TIMEOUT_MS = 60_000
4135
// The hard backstop must lose the race to `AbortSignal.timeout` in every case the
4236
// signal CAN handle, or it fires first and replaces the precise per-stage message
4337
// ("fetch failed", "body read failed") with its own generic one. The margin is
4438
// what makes it a backstop rather than the primary timeout.
4539
const CATALOG_HARD_DEADLINE_MS = CATALOG_FETCH_TIMEOUT_MS + 15_000
4640

47-
/** Describe the first structural problem in a provider entry, or undefined when
48-
* it is well formed.
49-
*
50-
* The provider-level rules match the runtime's own predicate (`isCatalogEntry` in
51-
* src/provider/models-catalog.ts, which `Provider.state()` screens every entry
52-
* with). The per-model rules cover exactly the fields the runtime dereferences
53-
* WITHOUT a guard: `Provider.fromModelsDevModel` reads `model.limit.context`
54-
* directly while every neighbouring field uses `?.`/`??`, so a model record
55-
* missing `limit` crashes provider initialisation on cold start instead of
56-
* degrading. Catching it here turns that into a failed release build.
57-
*
58-
* Deliberately NOT the zod `Provider` schema, which requires an `options` record
59-
* real models.dev entries do not carry and would reject valid live data — see the
60-
* note on `isCatalogEntry` for that history. Verified against every model in the
61-
* release fixture (4108), the committed snapshot (5299) and the live catalog
62-
* (7487): zero failures. */
63-
function providerEntryProblem(id: string, entry: unknown): string | undefined {
64-
if (typeof entry !== "object" || entry === null || Array.isArray(entry)) return `${id} (not an object)`
65-
if (!("id" in entry) || typeof entry.id !== "string") return `${id} (no string id)`
66-
if (!("models" in entry)) return `${id} (no models)`
67-
const models = entry.models
68-
if (typeof models !== "object" || models === null || Array.isArray(models)) return `${id} (models is not a map)`
69-
for (const [modelId, model] of Object.entries(models)) {
70-
const where = `${id}/${modelId}`
71-
if (typeof model !== "object" || model === null || Array.isArray(model)) return `${where} (not an object)`
72-
if (!("limit" in model)) return `${where} (no limit)`
73-
const limit = model.limit
74-
if (typeof limit !== "object" || limit === null || Array.isArray(limit)) return `${where} (limit is not an object)`
75-
if (!("context" in limit) || typeof limit.context !== "number") return `${where} (limit.context is not a number)`
76-
}
77-
return undefined
78-
}
79-
8041
/** Fetch the models.dev catalog, failing loudly rather than hanging or
8142
* returning an error page.
8243
*
8344
* `fetch` resolves for 4xx/5xx, so without the `res.ok` check a load-balancer
8445
* error page flows straight into the snapshot. An HTML body would at least break
8546
* the build at parse time, but a JSON error body (`{"error": ...}`) is valid
8647
* TypeScript and would ship as a catalog with no providers in it. */
87-
async function fetchModelsCatalog(url: string): Promise<string> {
48+
async function fetchModelsCatalog(url: string, diagnosticOrigin: string): Promise<string> {
8849
// Backstop for the case where the abort signal fires but the fetch promise never
8950
// settles, so the `catch` below is never reached. `AbortSignal.timeout` cannot
9051
// cancel a blocked `getaddrinfo()` — documented in src/provider/models.ts
@@ -99,7 +60,7 @@ async function fetchModelsCatalog(url: string): Promise<string> {
9960
// fails; it never falls through to a stale catalog.
10061
const deadline = setTimeout(() => {
10162
console.error(
102-
`error: models.dev fetch from ${url} did not settle within ${CATALOG_HARD_DEADLINE_MS}ms ` +
63+
`error: models.dev fetch from ${diagnosticOrigin} did not settle within ${CATALOG_HARD_DEADLINE_MS}ms ` +
10364
`(host unreachable or unresolvable); failing the build`,
10465
)
10566
process.exit(1)
@@ -108,84 +69,34 @@ async function fetchModelsCatalog(url: string): Promise<string> {
10869
let res: Response
10970
try {
11071
res = await fetch(url, { signal: AbortSignal.timeout(CATALOG_FETCH_TIMEOUT_MS) })
111-
} catch (e) {
112-
throw new Error(`models.dev fetch from ${url} failed or timed out after ${CATALOG_FETCH_TIMEOUT_MS}ms`, {
113-
cause: e,
114-
})
72+
} catch {
73+
throw new Error(
74+
`models.dev fetch from ${diagnosticOrigin} failed or timed out after ${CATALOG_FETCH_TIMEOUT_MS}ms`,
75+
)
11576
}
116-
if (!res.ok) throw new Error(`models.dev fetch failed: HTTP ${res.status} ${res.statusText} from ${url}`)
77+
if (!res.ok)
78+
throw new Error(`models.dev fetch failed: HTTP ${res.status} ${res.statusText} from ${diagnosticOrigin}`)
11779
try {
11880
// Inside its own try: a host that sends headers promptly then stalls
11981
// mid-body aborts here, and an uncaught abort surfaces as a bare
12082
// AbortError carrying none of the context above.
12183
return await res.text()
122-
} catch (e) {
123-
throw new Error(`models.dev body read from ${url} failed or timed out after ${CATALOG_FETCH_TIMEOUT_MS}ms`, {
124-
cause: e,
125-
})
84+
} catch {
85+
throw new Error(
86+
`models.dev body read from ${diagnosticOrigin} failed or timed out after ${CATALOG_FETCH_TIMEOUT_MS}ms`,
87+
)
12688
}
12789
} finally {
12890
clearTimeout(deadline)
12991
}
13092
}
13193

132-
/** Reject a catalog that parses but is not usable.
133-
*
134-
* Release builds embed this in every binary, so an empty, truncated or
135-
* structurally broken payload has to stop the release rather than ship a CLI
136-
* that offers no models.
137-
*
138-
* `strict` adds the checks that only make sense for the real models.dev catalog:
139-
* the size floor, and the requirement that the major providers carry models rather
140-
* than merely appear as keys. It is ON for every release build and for any plain
141-
* default-endpoint build, and OFF only when an operator has deliberately pointed
142-
* the build at a custom catalog, which is allowed to be small. The structural
143-
* checks run in both modes — a key count alone is not enough, since a payload can
144-
* carry 50+ keys whose values are junk and still parse. */
145-
function assertUsableCatalog(text: string, origin: string, strict: boolean): void {
146-
let parsed: unknown
94+
async function readModelsCatalog(file: string, diagnosticOrigin: string): Promise<string> {
14795
try {
148-
parsed = JSON.parse(text)
149-
} catch (e) {
150-
throw new Error(`models.dev catalog from ${origin} is not valid JSON`, { cause: e })
151-
}
152-
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed))
153-
throw new Error(`models.dev catalog from ${origin} is not a provider object`)
154-
const catalog = new Map<string, unknown>(Object.entries(parsed))
155-
if (catalog.size === 0) throw new Error(`models.dev catalog from ${origin} is empty`)
156-
const problems = [...catalog.entries()]
157-
.map(([id, entry]) => providerEntryProblem(id, entry))
158-
.filter((p): p is string => p !== undefined)
159-
if (problems.length > 0)
160-
throw new Error(
161-
`models.dev catalog from ${origin} has ${problems.length} malformed provider entries: ` +
162-
`${problems.slice(0, 5).join(", ")}${problems.length > 5 ? ", …" : ""}`,
163-
)
164-
const modelCount = (id: string): number => {
165-
const entry = catalog.get(id)
166-
if (typeof entry !== "object" || entry === null || !("models" in entry)) return 0
167-
const models = entry.models
168-
return typeof models === "object" && models !== null ? Object.keys(models).length : 0
169-
}
170-
if (!strict) {
171-
console.log(`models.dev catalog from ${origin}: ${catalog.size} providers (custom catalog, size floor not applied)`)
172-
return
96+
return await Bun.file(file).text()
97+
} catch {
98+
throw new Error(`models.dev catalog read from ${diagnosticOrigin} failed`)
17399
}
174-
if (catalog.size < MIN_CATALOG_PROVIDERS)
175-
throw new Error(
176-
`models.dev catalog from ${origin} has only ${catalog.size} providers, expected at least ${MIN_CATALOG_PROVIDERS}`,
177-
)
178-
const missing = REQUIRED_CATALOG_PROVIDERS.filter((p) => !catalog.has(p))
179-
if (missing.length > 0)
180-
throw new Error(`models.dev catalog from ${origin} is missing required providers: ${missing.join(", ")}`)
181-
// Every required provider must actually carry models, not just exist as a key.
182-
const empty = REQUIRED_CATALOG_PROVIDERS.filter((p) => modelCount(p) === 0)
183-
if (empty.length > 0)
184-
throw new Error(`models.dev catalog from ${origin} has no usable models for: ${empty.join(", ")}`)
185-
console.log(
186-
`models.dev catalog from ${origin}: ${catalog.size} providers ` +
187-
`(${REQUIRED_CATALOG_PROVIDERS.map((p) => `${p}=${modelCount(p)}`).join(", ")})`,
188-
)
189100
}
190101

191102
// Fetch and generate models.dev snapshot. MODELS_DEV_API_JSON pins the catalog to
@@ -196,12 +107,16 @@ function assertUsableCatalog(text: string, origin: string, strict: boolean): voi
196107
// and the build dies on `fetch("")` with ERR_INVALID_URL.
197108
const modelsFile = process.env.MODELS_DEV_API_JSON || undefined
198109
const modelsOrigin = modelsFile ?? `${modelsUrl}/api.json`
199-
const modelsData = modelsFile ? await Bun.file(modelsFile).text() : await fetchModelsCatalog(modelsOrigin)
110+
const modelsDiagnosticOrigin = catalogDiagnosticOrigin(modelsOrigin, modelsFile ? "file" : "url")
111+
const modelsData = modelsFile
112+
? await readModelsCatalog(modelsFile, modelsDiagnosticOrigin)
113+
: await fetchModelsCatalog(modelsOrigin, modelsDiagnosticOrigin)
200114
// A release is held to the full floor however its catalog was sourced, so pointing
201115
// a release build at a custom catalog cannot quietly skip the size and
202116
// required-provider checks.
203117
const strictCatalog = !!process.env.OPENCODE_RELEASE || (!modelsFile && !modelsUrlOverride)
204-
assertUsableCatalog(modelsData, modelsOrigin, strictCatalog)
118+
const catalogSummary = assertUsableCatalog(modelsData, modelsDiagnosticOrigin, strictCatalog)
119+
console.log(formatCatalogSummary(catalogSummary, modelsDiagnosticOrigin))
205120
await Bun.write(
206121
path.join(dir, "src/provider/models-snapshot.ts"),
207122
`// Auto-generated by build.ts - do not edit\nexport const snapshot = ${modelsData.trim()} as const\n`,
Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
export const REQUIRED_CATALOG_PROVIDERS = ["anthropic", "openai", "google"] as const
2+
export const MIN_CATALOG_PROVIDERS = 50
3+
4+
type CatalogSummary = {
5+
providerCount: number
6+
requiredModelCounts: Record<string, number>
7+
strict: boolean
8+
}
9+
10+
function isRecord(value: unknown): value is Record<string, unknown> {
11+
return typeof value === "object" && value !== null && !Array.isArray(value)
12+
}
13+
14+
function optionalStringProblem(value: unknown, where: string, allowEmpty = true): string | undefined {
15+
if (value === undefined || value === null) return undefined
16+
if (typeof value !== "string") return `${where} is not a string`
17+
if (!allowEmpty && value.length === 0) return `${where} is empty`
18+
return undefined
19+
}
20+
21+
function stringArrayProblem(value: unknown, where: string): string | undefined {
22+
if (value === undefined || value === null) return undefined
23+
if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) return `${where} is not a string array`
24+
return undefined
25+
}
26+
27+
/** Describe the first runtime-relevant structural problem in a provider entry. */
28+
function providerEntryProblem(id: string, entry: unknown): string | undefined {
29+
if (!isRecord(entry)) return `${id} (not an object)`
30+
if (typeof entry.id !== "string" || entry.id.length === 0) return `${id} (no non-empty string id)`
31+
if (entry.id !== id) return `${id} (id does not match catalog key)`
32+
33+
const envProblem = stringArrayProblem(entry.env, `${id}.env`)
34+
if (envProblem) return envProblem
35+
const apiProblem = optionalStringProblem(entry.api, `${id}.api`)
36+
if (apiProblem) return apiProblem
37+
const npmProblem = optionalStringProblem(entry.npm, `${id}.npm`, false)
38+
if (npmProblem) return npmProblem
39+
40+
if (!("models" in entry)) return `${id} (no models)`
41+
if (!isRecord(entry.models)) return `${id} (models is not a map)`
42+
43+
for (const [modelId, model] of Object.entries(entry.models)) {
44+
const where = `${id}/${modelId}`
45+
if (!isRecord(model)) return `${where} (not an object)`
46+
if (typeof model.id !== "string" || model.id.length === 0) return `${where} (no non-empty string id)`
47+
if (model.id !== modelId) return `${where} (id does not match catalog key)`
48+
49+
if (model.provider !== undefined && model.provider !== null) {
50+
if (!isRecord(model.provider)) return `${where}.provider is not an object`
51+
const modelApiProblem = optionalStringProblem(model.provider.api, `${where}.provider.api`)
52+
if (modelApiProblem) return modelApiProblem
53+
const modelNpmProblem = optionalStringProblem(model.provider.npm, `${where}.provider.npm`, false)
54+
if (modelNpmProblem) return modelNpmProblem
55+
}
56+
57+
if (!isRecord(model.limit)) return `${where} (limit is not an object)`
58+
if (typeof model.limit.context !== "number") return `${where} (limit.context is not a number)`
59+
if (typeof model.limit.output !== "number") return `${where} (limit.output is not a number)`
60+
if (model.limit.input !== undefined && model.limit.input !== null && typeof model.limit.input !== "number")
61+
return `${where} (limit.input is not a number)`
62+
63+
if (model.modalities !== undefined && model.modalities !== null) {
64+
if (!isRecord(model.modalities)) return `${where}.modalities is not an object`
65+
const inputProblem = stringArrayProblem(model.modalities.input, `${where}.modalities.input`)
66+
if (inputProblem) return inputProblem
67+
const outputProblem = stringArrayProblem(model.modalities.output, `${where}.modalities.output`)
68+
if (outputProblem) return outputProblem
69+
}
70+
}
71+
return undefined
72+
}
73+
74+
/** Return a log-safe description without URL userinfo, path, query or fragment. */
75+
export function catalogDiagnosticOrigin(source: string, kind: "file" | "url"): string {
76+
if (kind === "file") return "local catalog file"
77+
try {
78+
const url = new URL(source)
79+
if (url.protocol !== "http:" && url.protocol !== "https:") return "custom catalog endpoint"
80+
return url.origin
81+
} catch {
82+
return "custom catalog endpoint"
83+
}
84+
}
85+
86+
/** Reject a catalog that parses but cannot be consumed safely at runtime. */
87+
export function assertUsableCatalog(text: string, diagnosticOrigin: string, strict: boolean): CatalogSummary {
88+
let parsed: unknown
89+
try {
90+
parsed = JSON.parse(text)
91+
} catch {
92+
throw new Error(`models.dev catalog from ${diagnosticOrigin} is not valid JSON`)
93+
}
94+
if (!isRecord(parsed)) throw new Error(`models.dev catalog from ${diagnosticOrigin} is not a provider object`)
95+
96+
const catalog = new Map<string, unknown>(Object.entries(parsed))
97+
if (catalog.size === 0) throw new Error(`models.dev catalog from ${diagnosticOrigin} is empty`)
98+
99+
const problems = [...catalog.entries()]
100+
.map(([id, entry]) => providerEntryProblem(id, entry))
101+
.filter((problem): problem is string => problem !== undefined)
102+
if (problems.length > 0)
103+
throw new Error(
104+
`models.dev catalog from ${diagnosticOrigin} has ${problems.length} malformed provider entries: ` +
105+
`${problems.slice(0, 5).join(", ")}${problems.length > 5 ? ", …" : ""}`,
106+
)
107+
108+
const modelCount = (id: string): number => {
109+
const entry = catalog.get(id)
110+
if (!isRecord(entry) || !isRecord(entry.models)) return 0
111+
return Object.keys(entry.models).length
112+
}
113+
114+
if (strict) {
115+
if (catalog.size < MIN_CATALOG_PROVIDERS)
116+
throw new Error(
117+
`models.dev catalog from ${diagnosticOrigin} has only ${catalog.size} providers, ` +
118+
`expected at least ${MIN_CATALOG_PROVIDERS}`,
119+
)
120+
const missing = REQUIRED_CATALOG_PROVIDERS.filter((provider) => !catalog.has(provider))
121+
if (missing.length > 0)
122+
throw new Error(
123+
`models.dev catalog from ${diagnosticOrigin} is missing required providers: ${missing.join(", ")}`,
124+
)
125+
const empty = REQUIRED_CATALOG_PROVIDERS.filter((provider) => modelCount(provider) === 0)
126+
if (empty.length > 0)
127+
throw new Error(`models.dev catalog from ${diagnosticOrigin} has no usable models for: ${empty.join(", ")}`)
128+
}
129+
130+
return {
131+
providerCount: catalog.size,
132+
requiredModelCounts: Object.fromEntries(
133+
REQUIRED_CATALOG_PROVIDERS.map((provider) => [provider, modelCount(provider)]),
134+
),
135+
strict,
136+
}
137+
}
138+
139+
export function formatCatalogSummary(summary: CatalogSummary, diagnosticOrigin: string): string {
140+
if (!summary.strict)
141+
return `models.dev catalog from ${diagnosticOrigin}: ${summary.providerCount} providers (custom catalog, size floor not applied)`
142+
return (
143+
`models.dev catalog from ${diagnosticOrigin}: ${summary.providerCount} providers ` +
144+
`(${REQUIRED_CATALOG_PROVIDERS.map((provider) => `${provider}=${summary.requiredModelCounts[provider]}`).join(
145+
", ",
146+
)})`
147+
)
148+
}

0 commit comments

Comments
 (0)