Skip to content

Commit faed05d

Browse files
fix(server): usage prices long-context requests at above-tier rates (pingdotgg#25430)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 458a792 commit faed05d

3 files changed

Lines changed: 196 additions & 18 deletions

File tree

‎apps/server/src/usage/usageAggregation.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ const rates: RateTable = new Map([
1919
cacheCreationCostPerToken: 2.5e-5,
2020
},
2121
ultrafast: null,
22+
contextTiers: [],
2223
},
2324
],
2425
]);

‎apps/server/src/usage/usagePricing.test.ts‎

Lines changed: 103 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -195,6 +195,104 @@ describe("usage pricing", () => {
195195
expect(cost("gpt-6-sol", "ultrafast")).toBe(cost("gpt-6-sol", "standard"));
196196
});
197197

198+
it("prices a request above the model's long-context threshold at its above-tier rates", () => {
199+
const table = parseRateTable({
200+
"claude-haiku-5-5": {
201+
input_cost_per_token: 1e-6,
202+
output_cost_per_token: 5e-6,
203+
cache_read_input_token_cost: 1e-7,
204+
cache_creation_input_token_cost: 1.25e-6,
205+
input_cost_per_token_above_100k_tokens: 2e-6,
206+
output_cost_per_token_above_100k_tokens: 7.5e-6,
207+
cache_read_input_token_cost_above_100k_tokens: 2e-7,
208+
cache_creation_input_token_cost_above_100k_tokens: 2.5e-6,
209+
},
210+
"gpt-6-sol": {
211+
...rate(2e-6, 2e-7),
212+
input_cost_per_token_priority: 4e-6,
213+
output_cost_per_token_priority: 2e-5,
214+
cache_read_input_token_cost_priority: 4e-7,
215+
input_cost_per_token_above_272k_tokens_priority: 8e-6,
216+
output_cost_per_token_above_272k_tokens_priority: 3e-5,
217+
cache_read_input_token_cost_above_272k_tokens_priority: 8e-7,
218+
},
219+
});
220+
const longRequest = (
221+
model: string,
222+
speed: UsageSpeed,
223+
totals: { uncached: number; cacheRead: number; cacheWrite: number; output: number },
224+
) => ({
225+
model,
226+
speed,
227+
reportedCostUsd: null,
228+
totals: {
229+
uncachedInputTokens: totals.uncached,
230+
cachedInputTokens: totals.cacheRead,
231+
cacheCreationTokens: totals.cacheWrite,
232+
outputTokens: totals.output,
233+
reasoningTokens: 0,
234+
},
235+
});
236+
237+
// The Claude prompt is input + cache read + cache write: 120K, above the 100K tier.
238+
expect
239+
.soft(
240+
priceUsage(
241+
table,
242+
longRequest("claude-haiku-5-5", "standard", {
243+
uncached: 10_000,
244+
cacheRead: 100_000,
245+
cacheWrite: 10_000,
246+
output: 1_000,
247+
}),
248+
).costUsd,
249+
)
250+
.toBeCloseTo(10_000 * 2e-6 + 100_000 * 2e-7 + 10_000 * 2.5e-6 + 1_000 * 7.5e-6, 12);
251+
expect(
252+
priceUsage(
253+
table,
254+
longRequest("gpt-6-sol", "fast", {
255+
uncached: 20_000,
256+
cacheRead: 280_000,
257+
cacheWrite: 0,
258+
output: 1_000,
259+
}),
260+
).costUsd,
261+
).toBeCloseTo(20_000 * 8e-6 + 280_000 * 8e-7 + 1_000 * 3e-5, 12);
262+
});
263+
264+
it("bills the long-context tier only above its threshold, filling missing tier fields from base", () => {
265+
const table = parseRateTable({
266+
"gpt-6-sol": {
267+
input_cost_per_token: 2e-6,
268+
output_cost_per_token: 1e-5,
269+
cache_read_input_token_cost: 2e-7,
270+
input_cost_per_token_priority: 4e-6,
271+
output_cost_per_token_priority: 2e-5,
272+
input_cost_per_token_above_272k_tokens: 4e-6,
273+
},
274+
});
275+
const cost = (prompt: number, speed: UsageSpeed = "standard") =>
276+
priceUsage(table, {
277+
model: "gpt-6-sol",
278+
speed,
279+
reportedCostUsd: null,
280+
totals: {
281+
uncachedInputTokens: prompt - 200_000,
282+
cachedInputTokens: 200_000,
283+
cacheCreationTokens: 0,
284+
outputTokens: 1_000,
285+
reasoningTokens: 0,
286+
},
287+
}).costUsd;
288+
289+
expect(cost(272_000)).toBeCloseTo(72_000 * 2e-6 + 200_000 * 2e-7 + 1_000 * 1e-5, 12);
290+
// Output keeps the base rate; cache reads keep the base discount relative to input.
291+
expect(cost(272_001)).toBeCloseTo(72_001 * 4e-6 + 200_000 * 4e-7 + 1_000 * 1e-5, 12);
292+
// Without a priority long-context rate, priority requests keep the priority rate.
293+
expect(cost(272_001, "fast")).toBeCloseTo(72_001 * 4e-6 + 200_000 * 4e-7 + 1_000 * 2e-5, 12);
294+
});
295+
198296
it("keeps the canonical Fable rate separate from DeepInfra in either order", () => {
199297
const canonical = ["claude-fable-5", rate(1e-5, 1e-6)] as const;
200298
const deepInfra = ["deepinfra/anthropic/claude-fable-5", rate(1e-5)] as const;
@@ -239,8 +337,12 @@ describe("usage pricing", () => {
239337
...rate(1),
240338
input_cost_per_token_priority: 2,
241339
output_cost_per_token_priority: 10,
340+
input_cost_per_token_above_272k_tokens: 2,
341+
input_cost_per_token_above_272k_tokens_priority: 4,
342+
output_cost_per_token_above_272k_tokens_priority: 20,
242343
},
243-
"provider-b/example-model": rate(1),
344+
"provider-b/example-model": { ...rate(1), input_cost_per_token_above_272k_tokens: 2 },
345+
"provider-c/example-model": rate(1),
244346
});
245347

246348
expect(lookupRate(table, "example-model")).toEqual(

‎apps/server/src/usage/usagePricing.ts‎

Lines changed: 92 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,23 @@ export interface TokenRates {
1414
readonly cacheCreationCostPerToken: number;
1515
}
1616

17-
/** Omits LiteLLM's above_272k_tokens, flex and batches variants, which transcripts do not record. */
18-
export interface ModelRate extends TokenRates {
17+
interface SpeedRates extends TokenRates {
1918
readonly fast: TokenRates | null;
2019
readonly ultrafast: TokenRates | null;
2120
}
2221

22+
/** A request whose prompt exceeds `thresholdTokens` bills every token at this tier. */
23+
interface ContextTier {
24+
readonly thresholdTokens: number;
25+
readonly rate: SpeedRates;
26+
}
27+
28+
/** Omits LiteLLM's flex and batches variants, which transcripts do not record. */
29+
export interface ModelRate extends SpeedRates {
30+
/** Sorted by descending threshold. */
31+
readonly contextTiers: readonly ContextTier[];
32+
}
33+
2334
export type RateTable = ReadonlyMap<string, ModelRate>;
2435

2536
export function createOverrideRateTable(
@@ -37,6 +48,7 @@ export function createOverrideRateTable(
3748
(prices.cacheWriteCostPerMillionTokens ?? prices.inputCostPerMillionTokens) / 1_000_000,
3849
fast: null,
3950
ultrafast: null,
51+
contextTiers: [],
4052
},
4153
]),
4254
);
@@ -53,9 +65,10 @@ function readTokenRates(
5365
entry: LiteLlmEntry,
5466
suffix: string,
5567
standard?: TokenRates,
68+
outputFallback?: number,
5669
): TokenRates | null {
5770
const input = finiteNumber(entry[`input_cost_per_token${suffix}`]);
58-
const output = finiteNumber(entry[`output_cost_per_token${suffix}`]);
71+
const output = finiteNumber(entry[`output_cost_per_token${suffix}`]) ?? outputFallback ?? null;
5972
if (input === null || output === null) return null;
6073
const cacheRate = (name: string, field: "cacheReadCostPerToken" | "cacheCreationCostPerToken") =>
6174
finiteNumber(entry[`${name}${suffix}`]) ??
@@ -89,6 +102,40 @@ function fastMultiplier(entry: LiteLlmEntry): number | null {
89102
return fast !== null && fast > 0 ? fast : null;
90103
}
91104

105+
function readSpeedRates(entry: LiteLlmEntry, standard: TokenRates, suffix = ""): SpeedRates {
106+
const multiple = fastMultiplier(entry);
107+
return {
108+
...standard,
109+
fast:
110+
multiple === null
111+
? readTokenRates(entry, `${suffix}_priority`, standard)
112+
: scaleTokenRates(standard, multiple),
113+
ultrafast: readTokenRates(entry, `${suffix}_ultrafast`, standard),
114+
};
115+
}
116+
117+
const CONTEXT_TIER_INPUT_KEY =
118+
/^input_cost_per_token_above_(\d{1,7})k_tokens(?:_priority|_ultrafast)?$/;
119+
120+
// Fields a tier omits fall back to base rates, so a priority-only tier still prices standard requests at base.
121+
function readContextTiers(entry: LiteLlmEntry, base: TokenRates): ContextTier[] {
122+
const thresholds = new Set<string>();
123+
for (const key of Object.keys(entry)) {
124+
const match = CONTEXT_TIER_INPUT_KEY.exec(key);
125+
if (match !== null) thresholds.add(match[1]!);
126+
}
127+
return [...thresholds]
128+
.map((threshold) => {
129+
const suffix = `_above_${threshold}k_tokens`;
130+
const standard = readTokenRates(entry, suffix, base, base.outputCostPerToken) ?? base;
131+
return {
132+
thresholdTokens: Number(threshold) * 1_000,
133+
rate: readSpeedRates(entry, standard, suffix),
134+
};
135+
})
136+
.toSorted((a, b) => b.thresholdTokens - a.thresholdTokens);
137+
}
138+
92139
// LiteLLM prefixes for the model makers' own APIs; their price beats a cloud's or reseller's for a bare name.
93140
const FIRST_PARTY_PROVIDERS = new Set([
94141
"anthropic",
@@ -113,14 +160,9 @@ export function parseRateTable(document: unknown): RateTable {
113160

114161
const key = normalizeRateKey(name);
115162
if (key.length === 0) continue;
116-
const multiple = fastMultiplier(entry);
117163
table.set(key, {
118-
...standard,
119-
fast:
120-
multiple === null
121-
? readTokenRates(entry, "_priority", standard)
122-
: scaleTokenRates(standard, multiple),
123-
ultrafast: readTokenRates(entry, "_ultrafast", standard),
164+
...readSpeedRates(entry, standard),
165+
contextTiers: readContextTiers(entry, standard),
124166
});
125167
}
126168

@@ -154,8 +196,8 @@ function sameTokenRates(a: TokenRates | null, b: TokenRates | null): boolean {
154196
);
155197
}
156198

157-
// Resellers often omit speed tiers, so a missing tier fills from the other entry instead of conflicting.
158-
function mergeAliasRate(a: ModelRate, b: ModelRate): ModelRate | null {
199+
// Resellers often omit speed and context tiers, so a missing tier fills from the other entry instead of conflicting.
200+
function mergeSpeedRates(a: SpeedRates, b: SpeedRates): SpeedRates | null {
159201
const fast = a.fast ?? b.fast;
160202
const ultrafast = a.ultrafast ?? b.ultrafast;
161203
if (
@@ -168,8 +210,39 @@ function mergeAliasRate(a: ModelRate, b: ModelRate): ModelRate | null {
168210
return { ...a, fast, ultrafast };
169211
}
170212

171-
function ratesAt(rate: ModelRate, speed: UsageSpeed): TokenRates {
172-
return (speed === "standard" ? null : rate[speed]) ?? rate;
213+
function mergeContextTiers(
214+
a: readonly ContextTier[],
215+
b: readonly ContextTier[],
216+
): readonly ContextTier[] | null {
217+
if (a.length === 0) return b;
218+
if (b.length === 0) return a;
219+
if (a.length !== b.length) return null;
220+
const merged: ContextTier[] = [];
221+
for (const [index, tier] of a.entries()) {
222+
const other = b[index]!;
223+
const rate = mergeSpeedRates(tier.rate, other.rate);
224+
if (tier.thresholdTokens !== other.thresholdTokens || rate === null) return null;
225+
merged.push({ thresholdTokens: tier.thresholdTokens, rate });
226+
}
227+
return merged;
228+
}
229+
230+
function mergeAliasRate(a: ModelRate, b: ModelRate): ModelRate | null {
231+
const speeds = mergeSpeedRates(a, b);
232+
const contextTiers = mergeContextTiers(a.contextTiers, b.contextTiers);
233+
return speeds === null || contextTiers === null ? null : { ...speeds, contextTiers };
234+
}
235+
236+
function promptTokens(totals: UsageTokenTotals): number {
237+
return totals.uncachedInputTokens + totals.cachedInputTokens + totals.cacheCreationTokens;
238+
}
239+
240+
// A tier without the requested speed keeps the base tier's rate for that speed.
241+
function ratesAt(rate: ModelRate, speed: UsageSpeed, totals: UsageTokenTotals): TokenRates {
242+
const prompt = promptTokens(totals);
243+
const tier = rate.contextTiers.find((tier) => prompt > tier.thresholdTokens)?.rate;
244+
if (speed === "standard") return tier ?? rate;
245+
return tier?.[speed] ?? rate[speed] ?? tier ?? rate;
173246
}
174247

175248
function normalizeRateKey(model: string): string {
@@ -274,11 +347,13 @@ export function priceUsage(
274347
return reported === null ? unsplit(0, "unpriced") : unsplit(reported, "providerReported");
275348
}
276349

277-
const listCost = costByCategory(totals, ratesAt(rate, record.speed));
350+
const listCost = costByCategory(totals, ratesAt(rate, record.speed, totals));
278351
const listCostUsd = sumCategories(listCost);
279352
if (reported !== null && listCostUsd <= 0) return unsplit(reported, "providerReported");
280353
const premiumUsd =
281-
record.speed === "standard" ? 0 : listCostUsd - sumCategories(costByCategory(totals, rate));
354+
record.speed === "standard"
355+
? 0
356+
: listCostUsd - sumCategories(costByCategory(totals, ratesAt(rate, "standard", totals)));
282357
const scale = reported === null ? 1 : reported / listCostUsd;
283358
return {
284359
costUsd: reported ?? listCostUsd,
@@ -301,6 +376,6 @@ export function cacheSavingsUsd(
301376
const rate =
302377
overrides?.get(record.model.trim()) ?? lookupRate(table, record.rateModel ?? record.model);
303378
if (rate === null) return 0;
304-
const rates = ratesAt(rate, record.speed);
379+
const rates = ratesAt(rate, record.speed, record.totals);
305380
return record.totals.cachedInputTokens * (rates.inputCostPerToken - rates.cacheReadCostPerToken);
306381
}

0 commit comments

Comments
 (0)