@@ -14,12 +14,23 @@ export interface TokenRates {
1414 readonly cacheCreationCostPerToken : number ;
1515}
1616
17- /** Omits LiteLLM's above_272k_tokens, flex and batches variants, which transcripts do not record. */
18- export interface ModelRate extends TokenRates {
17+ interface SpeedRates extends TokenRates {
1918 readonly fast : TokenRates | null ;
2019 readonly ultrafast : TokenRates | null ;
2120}
2221
22+ /** A request whose prompt exceeds `thresholdTokens` bills every token at this tier. */
23+ interface ContextTier {
24+ readonly thresholdTokens : number ;
25+ readonly rate : SpeedRates ;
26+ }
27+
28+ /** Omits LiteLLM's flex and batches variants, which transcripts do not record. */
29+ export interface ModelRate extends SpeedRates {
30+ /** Sorted by descending threshold. */
31+ readonly contextTiers : readonly ContextTier [ ] ;
32+ }
33+
2334export type RateTable = ReadonlyMap < string , ModelRate > ;
2435
2536export function createOverrideRateTable (
@@ -37,6 +48,7 @@ export function createOverrideRateTable(
3748 ( prices . cacheWriteCostPerMillionTokens ?? prices . inputCostPerMillionTokens ) / 1_000_000 ,
3849 fast : null ,
3950 ultrafast : null ,
51+ contextTiers : [ ] ,
4052 } ,
4153 ] ) ,
4254 ) ;
@@ -53,9 +65,10 @@ function readTokenRates(
5365 entry : LiteLlmEntry ,
5466 suffix : string ,
5567 standard ?: TokenRates ,
68+ outputFallback ?: number ,
5669) : TokenRates | null {
5770 const input = finiteNumber ( entry [ `input_cost_per_token${ suffix } ` ] ) ;
58- const output = finiteNumber ( entry [ `output_cost_per_token${ suffix } ` ] ) ;
71+ const output = finiteNumber ( entry [ `output_cost_per_token${ suffix } ` ] ) ?? outputFallback ?? null ;
5972 if ( input === null || output === null ) return null ;
6073 const cacheRate = ( name : string , field : "cacheReadCostPerToken" | "cacheCreationCostPerToken" ) =>
6174 finiteNumber ( entry [ `${ name } ${ suffix } ` ] ) ??
@@ -89,6 +102,40 @@ function fastMultiplier(entry: LiteLlmEntry): number | null {
89102 return fast !== null && fast > 0 ? fast : null ;
90103}
91104
105+ function readSpeedRates ( entry : LiteLlmEntry , standard : TokenRates , suffix = "" ) : SpeedRates {
106+ const multiple = fastMultiplier ( entry ) ;
107+ return {
108+ ...standard ,
109+ fast :
110+ multiple === null
111+ ? readTokenRates ( entry , `${ suffix } _priority` , standard )
112+ : scaleTokenRates ( standard , multiple ) ,
113+ ultrafast : readTokenRates ( entry , `${ suffix } _ultrafast` , standard ) ,
114+ } ;
115+ }
116+
117+ const CONTEXT_TIER_INPUT_KEY =
118+ / ^ i n p u t _ c o s t _ p e r _ t o k e n _ a b o v e _ ( \d { 1 , 7 } ) k _ t o k e n s (?: _ p r i o r i t y | _ u l t r a f a s t ) ? $ / ;
119+
120+ // Fields a tier omits fall back to base rates, so a priority-only tier still prices standard requests at base.
121+ function readContextTiers ( entry : LiteLlmEntry , base : TokenRates ) : ContextTier [ ] {
122+ const thresholds = new Set < string > ( ) ;
123+ for ( const key of Object . keys ( entry ) ) {
124+ const match = CONTEXT_TIER_INPUT_KEY . exec ( key ) ;
125+ if ( match !== null ) thresholds . add ( match [ 1 ] ! ) ;
126+ }
127+ return [ ...thresholds ]
128+ . map ( ( threshold ) => {
129+ const suffix = `_above_${ threshold } k_tokens` ;
130+ const standard = readTokenRates ( entry , suffix , base , base . outputCostPerToken ) ?? base ;
131+ return {
132+ thresholdTokens : Number ( threshold ) * 1_000 ,
133+ rate : readSpeedRates ( entry , standard , suffix ) ,
134+ } ;
135+ } )
136+ . toSorted ( ( a , b ) => b . thresholdTokens - a . thresholdTokens ) ;
137+ }
138+
92139// LiteLLM prefixes for the model makers' own APIs; their price beats a cloud's or reseller's for a bare name.
93140const FIRST_PARTY_PROVIDERS = new Set ( [
94141 "anthropic" ,
@@ -113,14 +160,9 @@ export function parseRateTable(document: unknown): RateTable {
113160
114161 const key = normalizeRateKey ( name ) ;
115162 if ( key . length === 0 ) continue ;
116- const multiple = fastMultiplier ( entry ) ;
117163 table . set ( key , {
118- ...standard ,
119- fast :
120- multiple === null
121- ? readTokenRates ( entry , "_priority" , standard )
122- : scaleTokenRates ( standard , multiple ) ,
123- ultrafast : readTokenRates ( entry , "_ultrafast" , standard ) ,
164+ ...readSpeedRates ( entry , standard ) ,
165+ contextTiers : readContextTiers ( entry , standard ) ,
124166 } ) ;
125167 }
126168
@@ -154,8 +196,8 @@ function sameTokenRates(a: TokenRates | null, b: TokenRates | null): boolean {
154196 ) ;
155197}
156198
157- // Resellers often omit speed tiers, so a missing tier fills from the other entry instead of conflicting.
158- function mergeAliasRate ( a : ModelRate , b : ModelRate ) : ModelRate | null {
199+ // Resellers often omit speed and context tiers, so a missing tier fills from the other entry instead of conflicting.
200+ function mergeSpeedRates ( a : SpeedRates , b : SpeedRates ) : SpeedRates | null {
159201 const fast = a . fast ?? b . fast ;
160202 const ultrafast = a . ultrafast ?? b . ultrafast ;
161203 if (
@@ -168,8 +210,39 @@ function mergeAliasRate(a: ModelRate, b: ModelRate): ModelRate | null {
168210 return { ...a , fast, ultrafast } ;
169211}
170212
171- function ratesAt ( rate : ModelRate , speed : UsageSpeed ) : TokenRates {
172- return ( speed === "standard" ? null : rate [ speed ] ) ?? rate ;
213+ function mergeContextTiers (
214+ a : readonly ContextTier [ ] ,
215+ b : readonly ContextTier [ ] ,
216+ ) : readonly ContextTier [ ] | null {
217+ if ( a . length === 0 ) return b ;
218+ if ( b . length === 0 ) return a ;
219+ if ( a . length !== b . length ) return null ;
220+ const merged : ContextTier [ ] = [ ] ;
221+ for ( const [ index , tier ] of a . entries ( ) ) {
222+ const other = b [ index ] ! ;
223+ const rate = mergeSpeedRates ( tier . rate , other . rate ) ;
224+ if ( tier . thresholdTokens !== other . thresholdTokens || rate === null ) return null ;
225+ merged . push ( { thresholdTokens : tier . thresholdTokens , rate } ) ;
226+ }
227+ return merged ;
228+ }
229+
230+ function mergeAliasRate ( a : ModelRate , b : ModelRate ) : ModelRate | null {
231+ const speeds = mergeSpeedRates ( a , b ) ;
232+ const contextTiers = mergeContextTiers ( a . contextTiers , b . contextTiers ) ;
233+ return speeds === null || contextTiers === null ? null : { ...speeds , contextTiers } ;
234+ }
235+
236+ function promptTokens ( totals : UsageTokenTotals ) : number {
237+ return totals . uncachedInputTokens + totals . cachedInputTokens + totals . cacheCreationTokens ;
238+ }
239+
240+ // A tier without the requested speed keeps the base tier's rate for that speed.
241+ function ratesAt ( rate : ModelRate , speed : UsageSpeed , totals : UsageTokenTotals ) : TokenRates {
242+ const prompt = promptTokens ( totals ) ;
243+ const tier = rate . contextTiers . find ( ( tier ) => prompt > tier . thresholdTokens ) ?. rate ;
244+ if ( speed === "standard" ) return tier ?? rate ;
245+ return tier ?. [ speed ] ?? rate [ speed ] ?? tier ?? rate ;
173246}
174247
175248function normalizeRateKey ( model : string ) : string {
@@ -274,11 +347,13 @@ export function priceUsage(
274347 return reported === null ? unsplit ( 0 , "unpriced" ) : unsplit ( reported , "providerReported" ) ;
275348 }
276349
277- const listCost = costByCategory ( totals , ratesAt ( rate , record . speed ) ) ;
350+ const listCost = costByCategory ( totals , ratesAt ( rate , record . speed , totals ) ) ;
278351 const listCostUsd = sumCategories ( listCost ) ;
279352 if ( reported !== null && listCostUsd <= 0 ) return unsplit ( reported , "providerReported" ) ;
280353 const premiumUsd =
281- record . speed === "standard" ? 0 : listCostUsd - sumCategories ( costByCategory ( totals , rate ) ) ;
354+ record . speed === "standard"
355+ ? 0
356+ : listCostUsd - sumCategories ( costByCategory ( totals , ratesAt ( rate , "standard" , totals ) ) ) ;
282357 const scale = reported === null ? 1 : reported / listCostUsd ;
283358 return {
284359 costUsd : reported ?? listCostUsd ,
@@ -301,6 +376,6 @@ export function cacheSavingsUsd(
301376 const rate =
302377 overrides ?. get ( record . model . trim ( ) ) ?? lookupRate ( table , record . rateModel ?? record . model ) ;
303378 if ( rate === null ) return 0 ;
304- const rates = ratesAt ( rate , record . speed ) ;
379+ const rates = ratesAt ( rate , record . speed , record . totals ) ;
305380 return record . totals . cachedInputTokens * ( rates . inputCostPerToken - rates . cacheReadCostPerToken ) ;
306381}
0 commit comments