Skip to content

Commit 5995ba1

Browse files
committed
fix(pricing): keep asset-priced models out of the models.dev snapshot
Dropping the model-name filter let two kinds of non-token pricing into the snapshot, where the runtime divides the rate by a million and multiplies it by token counts. `whisper-large-v3` prices per second of audio. It reached the snapshot at 0.003 USD because the catalogs serving it advertise text output, and the previous check only looked at the output side. `gemini-2.5-flash-image` and `gemini-3-pro-image-preview` price per generated image - 30 and 120 USD respectively. Google authors both with image output, so the output check did reject Google's entry, but `302ai` lists the same models as text-only and its entry was embedded instead. A sloppier reseller description could therefore override the author's. Read the modalities off the authored catalog rather than the catalog being loaded, so one reseller's description cannot decide eligibility, and require that the input modalities include text. A model that accepts no text is a transcription or vision-only endpoint billed by duration; accepting audio, video, image or PDF alongside text is normal for chat models and is tokenised, so `kimi-k3` and `gemini-3-flash-preview`, which take video and audio input, stay eligible. An earlier attempt rejected any audio or video input and dropped both of them. The snapshot goes from 422 to 418 entries. The Rust test asserts both directions against the committed artifact so a future regeneration cannot reintroduce either case.
1 parent 14ea185 commit 5995ba1

5 files changed

Lines changed: 178 additions & 69 deletions

File tree

‎nix/tools/models-dev-gen/compact.test.ts‎

Lines changed: 78 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -5,26 +5,34 @@ import {
55
formatDuplicateModelsDevPricingKeyWarning,
66
isEmbeddableModelsDevCandidate,
77
isPriceableModelsDevCost,
8-
isTextOutputModel,
8+
isTokenPricedModel,
99
MODELS_DEV_PROVIDER_TRUST,
1010
modelsDevProviderTrust,
1111
modelsDevProviderTrustArtifact,
1212
shouldReplaceModelsDevPricingCandidate,
1313
selectModelsDevPricingKey,
1414
} from './compact.ts';
1515

16-
const index = buildModelsDevCatalogIndex([
17-
'anthropic/claude-opus-5',
18-
'anthropic/claude-3-5-haiku-20241022',
19-
'moonshotai/kimi-k2.7-code',
20-
'xai/grok-build-0.1',
21-
'zhipuai/glm-5-turbo',
22-
]);
16+
const index = buildModelsDevCatalogIndex({
17+
'anthropic/claude-opus-5': { modalities: { input: ['text', 'image', 'pdf'], output: ['text'] } },
18+
'anthropic/claude-3-5-haiku-20241022': { modalities: { input: ['text'], output: ['text'] } },
19+
'moonshotai/kimi-k2.7-code': {
20+
modalities: { input: ['text', 'image', 'video'], output: ['text'] },
21+
},
22+
'xai/grok-build-0.1': { modalities: { input: ['text', 'image'], output: ['text'] } },
23+
'zhipuai/glm-5-turbo': { modalities: { input: ['text'], output: ['text'] } },
24+
'openai/whisper-large-v3': { modalities: { input: ['audio'], output: ['text'] } },
25+
'google/gemini-2.5-flash-image': {
26+
modalities: { input: ['text', 'image'], output: ['text', 'image'] },
27+
},
28+
});
2329

2430
void it('indexes authoring providers and bare model ids from the catalog keys', () => {
2531
assert.deepEqual([...index.authorProviderIds].sort(), [
2632
'anthropic',
33+
'google',
2734
'moonshotai',
35+
'openai',
2836
'xai',
2937
'zhipuai',
3038
]);
@@ -187,16 +195,72 @@ void it('rejects flat-fee catalogs that publish all-zero token costs', () => {
187195
assert.equal(isPriceableModelsDevCost({ input: 1, output: 2 }), true);
188196
});
189197

190-
void it('accepts only models that bill per output text token', () => {
191-
assert.equal(isTextOutputModel(undefined), true);
192-
assert.equal(isTextOutputModel({ output: ['text'] }), true);
193-
assert.equal(isTextOutputModel({ output: ['image'] }), false);
194-
assert.equal(isTextOutputModel({ output: ['text', 'image'] }), false);
198+
void it('accepts chat models whose audio and video inputs are tokenised', () => {
199+
// kimi-k2.7-code takes video input and gemini takes audio, both billed per
200+
// token, so a non-text input modality cannot disqualify a model on its own.
201+
assert.equal(
202+
isTokenPricedModel({ sourceModelId: 'claude-opus-5', modalities: undefined, index }),
203+
true,
204+
);
205+
assert.equal(
206+
isTokenPricedModel({ sourceModelId: 'kimi-k2.7-code', modalities: undefined, index }),
207+
true,
208+
);
209+
assert.equal(
210+
isTokenPricedModel({ sourceModelId: 'unlisted-model', modalities: undefined, index }),
211+
true,
212+
);
213+
});
214+
215+
void it('rejects duration-priced models however the serving catalog describes them', () => {
216+
// whisper-large-v3 accepts no text at all and prices per second, but the
217+
// catalogs serving it advertise text output, so its rate would read as a token
218+
// rate.
219+
assert.equal(
220+
isTokenPricedModel({
221+
sourceModelId: 'whisper-large-v3',
222+
modalities: { input: ['audio'], output: ['text'] },
223+
index,
224+
}),
225+
false,
226+
);
227+
});
228+
229+
void it('rejects image-output models a reseller catalog describes as text-only', () => {
230+
// google authors gemini-2.5-flash-image with image output; 302ai lists the same
231+
// model as text-only, which would embed a per-image rate as an output rate.
232+
assert.equal(
233+
isTokenPricedModel({
234+
sourceModelId: 'gemini-2.5-flash-image',
235+
modalities: { input: ['text', 'image'], output: ['text'] },
236+
index,
237+
}),
238+
false,
239+
);
240+
});
241+
242+
void it('falls back to the serving catalog for models the authored catalog omits', () => {
243+
assert.equal(
244+
isTokenPricedModel({
245+
sourceModelId: 'us.anthropic.claude-opus-5',
246+
modalities: { input: ['text', 'image'], output: ['text'] },
247+
index,
248+
}),
249+
true,
250+
);
251+
assert.equal(
252+
isTokenPricedModel({
253+
sourceModelId: 'some-tts-model',
254+
modalities: { input: ['text'], output: ['audio'] },
255+
index,
256+
}),
257+
false,
258+
);
195259
});
196260

197261
void it('exports sorted provider trust lists for the runtime loader', () => {
198262
assert.deepEqual(modelsDevProviderTrustArtifact(index), {
199-
owners: ['anthropic', 'moonshotai', 'xai', 'zai', 'zhipuai'],
263+
owners: ['anthropic', 'google', 'moonshotai', 'openai', 'xai', 'zai', 'zhipuai'],
200264
platforms: [
201265
'amazon-bedrock',
202266
'azure',

‎nix/tools/models-dev-gen/compact.ts‎

Lines changed: 63 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,11 @@ const FIRST_PARTY_PROVIDER_ID_ALIASES = [
5151
'zai',
5252
] as const satisfies readonly string[];
5353

54+
export type ModelsDevModalities = {
55+
input?: readonly string[];
56+
output?: readonly string[];
57+
};
58+
5459
/**
5560
* Index of the canonical `models/<author>/<id>.toml` catalog, used to decide
5661
* which provider authored a model without hardcoding a provider list.
@@ -62,6 +67,8 @@ export type ModelsDevCatalogIndex = {
6267
authorProviderIds: ReadonlySet<string>;
6368
/** The same keys with the author prefix stripped. */
6469
authoredModelIds: ReadonlySet<string>;
70+
/** Authored modalities by bare model id. */
71+
authoredModalities: ReadonlyMap<string, ModelsDevModalities>;
6572
};
6673

6774
export type ModelsDevPricingCandidate = {
@@ -74,26 +81,37 @@ export type ModelsDevPricingCandidate = {
7481
};
7582

7683
/**
77-
* Build the authorship index from the canonical catalog keys.
84+
* Build the authorship index from the canonical catalog.
7885
*
79-
* @param authoredModelKeys - `<author>/<id>` keys from `generateCatalog().models`.
86+
* @param authoredModels - `<author>/<id>` keyed models from `generateCatalog().models`.
8087
* @example
81-
* const index = buildModelsDevCatalogIndex(['anthropic/claude-opus-5']);
88+
* const index = buildModelsDevCatalogIndex({ 'anthropic/claude-opus-5': {} });
8289
* index.authorProviderIds.has('anthropic'); // true
8390
*/
8491
export function buildModelsDevCatalogIndex(
85-
authoredModelKeys: readonly string[],
92+
authoredModels: Readonly<Record<string, { modalities?: ModelsDevModalities }>>,
8693
): ModelsDevCatalogIndex {
8794
const authorProviderIds = new Set<string>();
8895
const authoredModelIds = new Set<string>();
89-
for (const key of authoredModelKeys) {
96+
const authoredModalities = new Map<string, ModelsDevModalities>();
97+
for (const [key, model] of Object.entries(authoredModels)) {
9098
const separator = key.indexOf('/');
91-
if (separator > 0) {
92-
authorProviderIds.add(key.slice(0, separator));
93-
authoredModelIds.add(key.slice(separator + 1));
99+
if (separator <= 0) {
100+
continue;
101+
}
102+
const modelId = key.slice(separator + 1);
103+
authorProviderIds.add(key.slice(0, separator));
104+
authoredModelIds.add(modelId);
105+
if (model.modalities != null) {
106+
authoredModalities.set(modelId, model.modalities);
94107
}
95108
}
96-
return { authoredKeys: new Set(authoredModelKeys), authorProviderIds, authoredModelIds };
109+
return {
110+
authoredKeys: new Set(Object.keys(authoredModels)),
111+
authorProviderIds,
112+
authoredModelIds,
113+
authoredModalities,
114+
};
97115
}
98116

99117
/**
@@ -185,12 +203,43 @@ export function isPriceableModelsDevCost<
185203
}
186204

187205
/**
188-
* Whether a model bills per text token, so its cost block fits the embedded
189-
* shape. Image, audio, and video models price per asset instead.
206+
* Whether a model bills per text token, so the embedded `input` and `output`
207+
* rates mean what the runtime assumes when it multiplies them by token counts.
208+
*
209+
* Two signals, both read off the modalities:
210+
*
211+
* - Output must be text only. An image or audio output rate is per asset, so
212+
* `gemini-2.5-flash-image`'s 30 USD output rate is per image, not per Mtok.
213+
* - Input must accept text. A model that accepts no text is a transcription or
214+
* vision-only endpoint billed by duration - `whisper-large-v3` accepts audio
215+
* alone and prices per second. Accepting audio, video, image or PDF *as well
216+
* as* text is normal for chat models and is tokenised, so it stays eligible.
217+
*
218+
* The authored catalog decides, not the catalog being read. Reseller catalogs
219+
* describe the same model less carefully, and one claiming text-only output for
220+
* an image model would otherwise smuggle a per-image rate into the snapshot.
221+
*
222+
* @example
223+
* // authored as input: ["audio"], so excluded whichever catalog serves it
224+
* isTokenPricedModel({ sourceModelId: 'whisper-large-v3', modalities: { output: ['text'] }, index });
225+
* // false
190226
*/
191-
export function isTextOutputModel(modalities: { output?: readonly string[] } | undefined): boolean {
192-
const output = modalities?.output ?? ['text'];
193-
return output.length === 1 && output[0] === 'text';
227+
export function isTokenPricedModel({
228+
sourceModelId,
229+
modalities,
230+
index,
231+
}: {
232+
sourceModelId: string;
233+
modalities: ModelsDevModalities | undefined;
234+
index: ModelsDevCatalogIndex;
235+
}): boolean {
236+
const authoritative = index.authoredModalities.get(sourceModelId) ?? modalities;
237+
const output = authoritative?.output ?? ['text'];
238+
if (output.length !== 1 || output[0] !== 'text') {
239+
return false;
240+
}
241+
const input = authoritative?.input ?? ['text'];
242+
return input.includes('text');
194243
}
195244

196245
export function selectModelsDevPricingKey(modelId: string, catalogId: string | undefined): string {

‎nix/tools/models-dev-gen/gen.ts‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ import {
2020
formatDuplicateModelsDevPricingKeyWarning,
2121
isEmbeddableModelsDevCandidate,
2222
isPriceableModelsDevCost,
23-
isTextOutputModel,
23+
isTokenPricedModel,
2424
type ModelsDevPricingCandidate,
2525
modelsDevProviderTrust,
2626
modelsDevProviderTrustArtifact,
@@ -38,11 +38,12 @@ type Model = {
3838
id?: string;
3939
cost?: Cost;
4040
limit?: { context?: number | null };
41-
modalities?: { output?: readonly string[] };
41+
modalities?: { input?: readonly string[]; output?: readonly string[] };
4242
};
4343
type ModelMetadata = {
4444
id?: string;
4545
release_date?: string;
46+
modalities?: { input?: readonly string[]; output?: readonly string[] };
4647
};
4748
type Provider = { id?: string; models?: Record<string, Model> };
4849
type EmbeddedModel = {
@@ -59,7 +60,7 @@ const { models, providers } = (await generateCatalog('.')) as {
5960
providers: Record<string, Provider>;
6061
};
6162

62-
const catalogIndex = buildModelsDevCatalogIndex(Object.keys(models));
63+
const catalogIndex = buildModelsDevCatalogIndex(models);
6364

6465
const selected: Record<string, { candidate: ModelsDevPricingCandidate; entry: EmbeddedModel }> = {};
6566
for (const [providerId, provider] of sortedEntries(providers)) {
@@ -74,8 +75,15 @@ for (const [providerId, provider] of sortedEntries(providers)) {
7475
}
7576
const cost = model.cost ?? {};
7677
// Skip entries without the base prices the runtime loader requires, and
77-
// models whose cost block prices assets rather than output text tokens.
78-
if (!isPriceableModelsDevCost(cost) || !isTextOutputModel(model.modalities)) {
78+
// models whose rates are per asset rather than per text token.
79+
if (
80+
!isPriceableModelsDevCost(cost) ||
81+
!isTokenPricedModel({
82+
sourceModelId: modelId,
83+
modalities: model.modalities,
84+
index: catalogIndex,
85+
})
86+
) {
7987
continue;
8088
}
8189
const pricingKey = selectModelsDevPricingKey(modelId, model.id);

‎rust/crates/ccusage-core/src/models-dev-pricing.json‎

Lines changed: 1 addition & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1152,15 +1152,6 @@
11521152
"context": 1048576
11531153
}
11541154
},
1155-
"gemini-2.5-flash-image": {
1156-
"cost": {
1157-
"input": 0.3,
1158-
"output": 30
1159-
},
1160-
"limit": {
1161-
"context": 32768
1162-
}
1163-
},
11641155
"gemini-2.5-flash-lite": {
11651156
"cost": {
11661157
"cache_read": 0.01,
@@ -1191,15 +1182,6 @@
11911182
"context": 1048576
11921183
}
11931184
},
1194-
"gemini-3-pro-image-preview": {
1195-
"cost": {
1196-
"input": 2,
1197-
"output": 120
1198-
},
1199-
"limit": {
1200-
"context": 32768
1201-
}
1202-
},
12031185
"gemini-3-pro-preview": {
12041186
"cost": {
12051187
"cache_read": 0.2,
@@ -1837,6 +1819,7 @@
18371819
},
18381820
"gpt-5.1-chat-latest": {
18391821
"cost": {
1822+
"cache_read": 0.12500000000000003,
18401823
"input": 1.25,
18411824
"output": 10
18421825
},
@@ -3506,15 +3489,6 @@
35063489
"context": 131072
35073490
}
35083491
},
3509-
"qwen3-asr-flash": {
3510-
"cost": {
3511-
"input": 0.035,
3512-
"output": 0.035
3513-
},
3514-
"limit": {
3515-
"context": 53248
3516-
}
3517-
},
35183492
"qwen3-coder-30b-a3b-instruct": {
35193493
"cost": {
35203494
"input": 0.45,
@@ -4042,15 +4016,6 @@
40424016
"context": 3500000
40434017
}
40444018
},
4045-
"whisper-large-v3": {
4046-
"cost": {
4047-
"input": 0.003,
4048-
"output": 0
4049-
},
4050-
"limit": {
4051-
"context": 0
4052-
}
4053-
},
40544019
"writer.palmyra-x4-v1:0": {
40554020
"cost": {
40564021
"input": 2.5,

‎rust/crates/ccusage-core/src/pricing.rs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1625,6 +1625,29 @@ mod tests {
16251625
assert_eq!(pricing.context_limit("grok-build-0.1"), Some(256_000));
16261626
}
16271627

1628+
#[test]
1629+
fn embedded_models_dev_omits_models_priced_per_asset() {
1630+
// These rates are per second of audio and per generated image. The
1631+
// runtime divides by a million and multiplies by token counts, so
1632+
// embedding them reports a wrong cost rather than no cost.
1633+
let embedded = embedded_models_dev_pricing();
1634+
1635+
for model in [
1636+
"whisper-large-v3",
1637+
"gemini-2.5-flash-image",
1638+
"gemini-3-pro-image-preview",
1639+
] {
1640+
assert!(
1641+
embedded.find_exact(model).is_none(),
1642+
"{model} prices assets rather than text tokens and must stay out of the snapshot"
1643+
);
1644+
}
1645+
// The guard keys on modalities, not on the name, so text models that
1646+
// merely accept audio or video input have to survive it.
1647+
assert!(embedded.find_exact("kimi-k3").is_some());
1648+
assert!(embedded.find_exact("gemini-3-flash-preview").is_some());
1649+
}
1650+
16281651
#[test]
16291652
fn embedded_models_dev_prices_resold_models_from_their_author() {
16301653
// models.dev lists kimi-k2.7-code once per catalog that serves it, and

0 commit comments

Comments
 (0)