Skip to content

Commit 63bf1f8

Browse files
committed
fix(server): stop overpricing cached Claude tokens
1 parent 86c9a92 commit 63bf1f8

4 files changed

Lines changed: 105 additions & 12 deletions

File tree

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
import { describe, expect, it } from "@effect/vitest";
2+
3+
import { lookupRate, normalizeModelName, parseRateTable } from "./usagePricing.ts";
4+
5+
const rate = (input: number, cacheRead?: number) => ({
6+
input_cost_per_token: input,
7+
output_cost_per_token: input * 5,
8+
...(cacheRead === undefined ? {} : { cache_read_input_token_cost: cacheRead }),
9+
});
10+
11+
describe("usage pricing", () => {
12+
it("keeps the existing model-name normalization contract", () => {
13+
expect(normalizeModelName(" Anthropic/Claude-Opus-5 ")).toBe("claude-opus-5");
14+
});
15+
16+
it("keeps the canonical Fable rate separate from DeepInfra in either order", () => {
17+
const canonical = ["claude-fable-5", rate(1e-5, 1e-6)] as const;
18+
const deepInfra = ["deepinfra/anthropic/claude-fable-5", rate(1e-5)] as const;
19+
20+
for (const entries of [
21+
[canonical, deepInfra],
22+
[deepInfra, canonical],
23+
]) {
24+
const table = parseRateTable(Object.fromEntries(entries));
25+
26+
expect(lookupRate(table, "claude-fable-5")?.cacheReadCostPerToken).toBe(1e-6);
27+
expect(lookupRate(table, "deepinfra/anthropic/claude-fable-5")?.cacheReadCostPerToken).toBe(
28+
1e-5,
29+
);
30+
expect(lookupRate(table, "other/claude-fable-5")).toBeNull();
31+
}
32+
});
33+
34+
it("adds a bare alias when every qualified entry has the same rate", () => {
35+
const table = parseRateTable({
36+
"provider-a/example-model": rate(1),
37+
"provider-b/example-model": rate(1),
38+
});
39+
40+
expect(lookupRate(table, "example-model")).toEqual(
41+
lookupRate(table, "provider-a/example-model"),
42+
);
43+
});
44+
45+
it("leaves an ambiguous bare name unpriced", () => {
46+
const table = parseRateTable({
47+
"provider-a/example-model": rate(1),
48+
"provider-b/example-model": rate(3),
49+
});
50+
51+
expect(lookupRate(table, "provider-a/example-model")?.inputCostPerToken).toBe(1);
52+
expect(lookupRate(table, "provider-b/example-model")?.inputCostPerToken).toBe(3);
53+
expect(lookupRate(table, "example-model")).toBeNull();
54+
});
55+
});

‎apps/server/src/usage/usagePricing.ts‎

Lines changed: 48 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,9 @@ function finiteNumber(value: unknown): number | null {
4444
* Entries without both an input and an output rate are dropped: a half-priced
4545
* model would silently under-report cost, which is worse than reporting the
4646
* model as unpriced.
47+
*
48+
* Entries keep their full normalized key; a bare name is aliased only when no
49+
* canonical entry exists and every qualified entry has the same rate.
4750
*/
4851
export function parseRateTable(document: unknown): RateTable {
4952
const table = new Map<string, ModelRate>();
@@ -56,7 +59,9 @@ export function parseRateTable(document: unknown): RateTable {
5659
const output = finiteNumber(entry.output_cost_per_token);
5760
if (input === null || output === null) continue;
5861

59-
table.set(normalizeModelName(name), {
62+
const key = normalizeRateKey(name);
63+
if (key.length === 0) continue;
64+
table.set(key, {
6065
inputCostPerToken: input,
6166
outputCostPerToken: output,
6267
// Anthropic bills cache reads at a discount and cache writes at a
@@ -66,20 +71,52 @@ export function parseRateTable(document: unknown): RateTable {
6671
cacheCreationCostPerToken: finiteNumber(entry.cache_creation_input_token_cost) ?? input,
6772
});
6873
}
74+
75+
// `null` marks a bare name claimed at conflicting rates: no alias for it.
76+
const aliasCandidates = new Map<string, ModelRate | null>();
77+
for (const [key, rate] of table) {
78+
const alias = bareModelName(key);
79+
if (alias.length === 0 || alias === key || table.has(alias)) continue;
80+
const held = aliasCandidates.get(alias);
81+
if (held === undefined) {
82+
aliasCandidates.set(alias, rate);
83+
} else if (held !== null && !sameRate(held, rate)) {
84+
aliasCandidates.set(alias, null);
85+
}
86+
}
87+
for (const [alias, rate] of aliasCandidates) {
88+
if (rate !== null) table.set(alias, rate);
89+
}
90+
6991
return table;
7092
}
7193

94+
function sameRate(a: ModelRate, b: ModelRate): boolean {
95+
return (
96+
a.inputCostPerToken === b.inputCostPerToken &&
97+
a.outputCostPerToken === b.outputCostPerToken &&
98+
a.cacheReadCostPerToken === b.cacheReadCostPerToken &&
99+
a.cacheCreationCostPerToken === b.cacheCreationCostPerToken
100+
);
101+
}
102+
103+
function normalizeRateKey(model: string): string {
104+
return model.trim().toLowerCase();
105+
}
106+
72107
/**
73108
* Canonicalises a model name for lookup.
74109
*
75-
* Strips a `provider/` prefix (LiteLLM publishes both `claude-opus-5` and
76-
* `anthropic/claude-opus-5`) and lowercases, since transcripts are inconsistent
77-
* about casing.
110+
* Strips a `provider/` prefix and lowercases, since transcripts are
111+
* inconsistent about casing.
78112
*/
79113
export function normalizeModelName(model: string): string {
80-
const trimmed = model.trim().toLowerCase();
81-
const slash = trimmed.lastIndexOf("/");
82-
return slash === -1 ? trimmed : trimmed.slice(slash + 1);
114+
return bareModelName(normalizeRateKey(model));
115+
}
116+
117+
function bareModelName(key: string): string {
118+
const slash = key.lastIndexOf("/");
119+
return slash === -1 ? key : key.slice(slash + 1);
83120
}
84121

85122
/**
@@ -99,9 +136,10 @@ const UNPRICEABLE_MODELS = new Set([
99136
]);
100137

101138
export function lookupRate(table: RateTable, model: string): ModelRate | null {
102-
const normalized = normalizeModelName(model);
103-
if (normalized.length === 0 || UNPRICEABLE_MODELS.has(normalized)) return null;
104-
return table.get(normalized) ?? null;
139+
const key = normalizeRateKey(model);
140+
const bareName = bareModelName(key);
141+
if (bareName.length === 0 || UNPRICEABLE_MODELS.has(bareName)) return null;
142+
return table.get(key) ?? null;
105143
}
106144

107145
export interface PricedUsage {

‎apps/server/src/usage/usageScanCache.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ describe("scan cache round trip", () => {
8888

8989
it("rejects the whole cache when an intern table holds a non-string", () => {
9090
// models: [1] would pass the undefined guard, put a number in a record's
91-
// model, and crash normalizeModelName at aggregate time.
91+
// model, and crash lookupRate at aggregate time.
9292
const encoded = encodeScanCache(cacheWith([["/a.jsonl", 100, [record()]]]));
9393
const poisoned = { ...encoded, models: [1] };
9494

‎apps/server/src/usage/usageScanCache.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ export function decodeScanCache(document: unknown): ScanCache {
124124

125125
// The intern tables must be all strings: a numeric entry would pass the
126126
// undefined guard below, land in a record's model, and crash the aggregate
127-
// at normalizeModelName. A corrupt table rejects the whole cache.
127+
// at lookupRate. A corrupt table rejects the whole cache.
128128
if (!root.models.every((value) => typeof value === "string")) return cache;
129129
if (!root.sessions.every((value) => typeof value === "string")) return cache;
130130
const models = root.models as readonly string[];

0 commit comments

Comments
 (0)