Skip to content

Commit 3ab4fd3

Browse files
fix(pricing): align cache validation with loader
Use the same Models.dev token-pricing and nonzero-rate eligibility gate for cache revalidation that the refresh loader uses. Keep fresh responses on the bounded shape gate so a structurally valid zero-loaded body reaches the loader, then reject it on 304 and recover with one unconditional request. Co-authored-by: wishworldbetter <[email protected]>
1 parent 2f6b500 commit 3ab4fd3

2 files changed

Lines changed: 151 additions & 19 deletions

File tree

‎rust/crates/ccusage-core/src/pricing.rs‎

Lines changed: 67 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -73,8 +73,16 @@ impl PricingEndpoint {
7373
}
7474
}
7575

76-
/// Returns whether a body has a non-empty pricing shape for this endpoint.
76+
/// Returns whether a body contains at least one entry the endpoint loader can use.
7777
pub fn validates(self, json: &str) -> bool {
78+
match self {
79+
Self::LiteLlm => litellm_json_has_loader_usable_entry(json),
80+
Self::ModelsDev => models_dev_json_has_loader_usable_entry(json),
81+
}
82+
}
83+
84+
/// Returns whether a body has a non-empty pricing shape for this endpoint.
85+
pub fn validates_shape(self, json: &str) -> bool {
7886
let Ok(Value::Object(entries)) = serde_json::from_str::<Value>(json) else {
7987
return false;
8088
};
@@ -86,6 +94,33 @@ impl PricingEndpoint {
8694
}
8795
}
8896

97+
fn litellm_json_has_loader_usable_entry(json: &str) -> bool {
98+
let Ok(Value::Object(entries)) = serde_json::from_str::<Value>(json) else {
99+
return false;
100+
};
101+
entries
102+
.values()
103+
.any(|value| parse_litellm_pricing(value.clone()).is_some())
104+
}
105+
106+
fn models_dev_json_has_loader_usable_entry(json: &str) -> bool {
107+
let Some(raw) = parse_models_dev_json(json) else {
108+
return false;
109+
};
110+
let rules = models_dev_catalog_rules();
111+
match raw {
112+
ModelsDevJson::Providers(providers) => providers.values().any(|provider| {
113+
provider
114+
.models
115+
.iter()
116+
.any(|(model_key, model)| models_dev_usable_cost(rules, model_key, model).is_some())
117+
}),
118+
ModelsDevJson::Models(models) => models
119+
.iter()
120+
.any(|(model_key, model)| models_dev_usable_cost(rules, model_key, model).is_some()),
121+
}
122+
}
123+
89124
fn litellm_entry_has_required_cost(value: &Value) -> bool {
90125
let Some(entry) = value.as_object() else {
91126
return false;
@@ -698,6 +733,20 @@ fn per_token(per_million: f64) -> f64 {
698733
per_million / 1_000_000.0
699734
}
700735

736+
fn models_dev_usable_cost<'a>(
737+
rules: &ModelsDevCatalogRules,
738+
source_model_id: &str,
739+
model: &'a ModelsDevModel,
740+
) -> Option<&'a ModelsDevCost> {
741+
if !rules.is_token_priced(source_model_id, model.modalities.as_ref()) {
742+
return None;
743+
}
744+
let cost = model.cost.as_ref()?;
745+
let input = cost.input?;
746+
let output = cost.output?;
747+
(input != 0.0 || output != 0.0).then_some(cost)
748+
}
749+
701750
#[derive(Debug, Deserialize)]
702751
struct ModelsDevLimit {
703752
context: Option<u64>,
@@ -935,9 +984,9 @@ impl PricingMap {
935984
// slip past and bill per-image or per-second rates as per-token ones.
936985
// It is generation's only remaining gate, so the online refresh
937986
// carries the same ids the snapshot does.
938-
if !rules.is_token_priced(&model_key, model.modalities.as_ref()) {
987+
let Some(cost) = models_dev_usable_cost(rules, &model_key, &model) else {
939988
continue;
940-
}
989+
};
941990
// Same reason: the tier half of the verdict reads the source key,
942991
// before it is resolved away.
943992
let declared_id = model.id.as_deref().filter(|id| !id.is_empty());
@@ -949,8 +998,8 @@ impl PricingMap {
949998
// An empty declared id falls back to the source key, exactly as the
950999
// generator's `selectModelsDevPricingKey` does: keeping "" would
9511000
// store the model under a name no lookup ever asks for.
952-
let model_id = match model.id.filter(|id| !id.is_empty()) {
953-
Some(id) => id,
1001+
let model_id = match model.id.as_ref().filter(|id| !id.is_empty()) {
1002+
Some(id) => id.clone(),
9541003
None => source_key.clone(),
9551004
};
9561005
// Dotted, dashed and case spellings name one model, so they contend
@@ -962,21 +1011,13 @@ impl PricingMap {
9621011
if claimed.is_none() && self.entries.contains_key(&model_id) {
9631012
continue;
9641013
}
965-
let Some(cost) = model.cost else {
966-
continue;
967-
};
9681014
let Some(input) = cost.input else {
9691015
continue;
9701016
};
9711017
let Some(output) = cost.output else {
9721018
continue;
9731019
};
974-
// Flat-fee subscription catalogs such as `kimi-for-coding` publish
975-
// all-zero token costs, which would report every request as free.
976-
if input == 0.0 && output == 0.0 {
977-
continue;
978-
}
979-
let context_limit = model.limit.and_then(|limit| limit.context);
1020+
let context_limit = model.limit.as_ref().and_then(|limit| limit.context);
9801021
let long_context = cost.long_context_tier();
9811022
let claim = ModelsDevClaim {
9821023
trust,
@@ -2296,21 +2337,29 @@ mod tests {
22962337
}
22972338

22982339
#[test]
2299-
fn validates_only_the_schema_for_each_pricing_endpoint() {
2340+
fn validates_pricing_documents_per_endpoint() {
23002341
let litellm =
23012342
r#"{"gpt-test":{"input_cost_per_token":0.000001,"output_cost_per_token":0.000002}}"#;
23022343
let models_dev =
23032344
r#"{"openai":{"models":{"gpt-test":{"cost":{"input":1.0,"output":2.0}}}}}"#;
2345+
let zero_loaded_models_dev = r#"{"openai":{"models":{"unpriced":{"modalities":{"input":[],"output":["text"]},"cost":{"input":1.0,"output":2.0}}}}}"#;
23042346

23052347
assert!(PricingEndpoint::LiteLlm.validates(litellm));
23062348
assert!(PricingEndpoint::LiteLlm.validates(r#"{"gpt-test":{"i":1.0,"o":2.0}}"#));
23072349
assert!(!PricingEndpoint::LiteLlm.validates(models_dev));
2350+
assert!(PricingEndpoint::ModelsDev.validates_shape(models_dev));
23082351
assert!(PricingEndpoint::ModelsDev.validates(models_dev));
23092352
assert!(!PricingEndpoint::ModelsDev.validates(litellm));
23102353
assert!(!PricingEndpoint::ModelsDev.validates("{}"));
2311-
assert!(
2312-
!PricingEndpoint::ModelsDev
2313-
.validates(r#"{"openai":{"models":{"free":{"cost":{"input":0.0,"output":0.0}}}}}"#)
2354+
assert!(!PricingEndpoint::ModelsDev.validates_shape(
2355+
r#"{"openai":{"models":{"free":{"cost":{"input":0.0,"output":0.0}}}}}"#
2356+
));
2357+
assert!(PricingEndpoint::ModelsDev.validates_shape(zero_loaded_models_dev));
2358+
assert!(!PricingEndpoint::ModelsDev.validates(zero_loaded_models_dev));
2359+
let mut pricing = PricingMap::default();
2360+
assert_eq!(
2361+
pricing.load_models_dev_json_missing(zero_loaded_models_dev),
2362+
Some(0)
23142363
);
23152364
}
23162365

‎rust/crates/ccusage/src/http.rs‎

Lines changed: 84 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ fn fetch_json_with_cache_dir_inner(
118118
PRICING_FETCH_MAX_BYTES,
119119
)
120120
.map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error.to_string()))?;
121-
if !endpoint.validates(&body) {
121+
if !endpoint.validates_shape(&body) {
122122
return Err(io::Error::new(
123123
io::ErrorKind::InvalidData,
124124
format!("pricing response failed {endpoint:?} validation"),
@@ -276,6 +276,7 @@ mod tests {
276276
r#"{"gpt-test":{"input_cost_per_token":0.000001,"output_cost_per_token":0.000002}}"#;
277277
const MODELS_DEV_BODY: &str =
278278
r#"{"openai":{"models":{"gpt-test":{"cost":{"input":1.0,"output":2.0}}}}}"#;
279+
const ZERO_LOADED_MODELS_DEV_BODY: &str = r#"{"openai":{"models":{"unpriced":{"modalities":{"input":[],"output":["text"]},"cost":{"input":1.0,"output":2.0}}}}}"#;
279280

280281
fn accept_with_timeout(listener: &TcpListener) -> io::Result<TcpStream> {
281282
let deadline = Instant::now() + Duration::from_secs(5);
@@ -335,6 +336,54 @@ mod tests {
335336
(url, handle)
336337
}
337338

339+
fn accept_until_idle(
340+
listener: &TcpListener,
341+
idle_timeout: Duration,
342+
) -> io::Result<Option<TcpStream>> {
343+
let deadline = Instant::now() + idle_timeout;
344+
loop {
345+
match listener.accept() {
346+
Ok((stream, _)) => {
347+
stream.set_nonblocking(false)?;
348+
return Ok(Some(stream));
349+
}
350+
Err(error) if error.kind() == io::ErrorKind::WouldBlock => {
351+
if Instant::now() >= deadline {
352+
return Ok(None);
353+
}
354+
thread::sleep(Duration::from_millis(1));
355+
}
356+
Err(error) => return Err(error),
357+
}
358+
}
359+
}
360+
361+
fn serve_responses_allowing_missing_tail(
362+
responses: Vec<String>,
363+
) -> (String, thread::JoinHandle<io::Result<Vec<String>>>) {
364+
let listener = TcpListener::bind("127.0.0.1:0").expect("bind test server");
365+
listener
366+
.set_nonblocking(true)
367+
.expect("make test server nonblocking");
368+
let url = format!(
369+
"http://{}/pricing.json",
370+
listener.local_addr().expect("test server address")
371+
);
372+
let handle = thread::spawn(move || {
373+
let mut requests = Vec::new();
374+
for response in responses {
375+
let Some(mut stream) = accept_until_idle(&listener, Duration::from_millis(250))?
376+
else {
377+
break;
378+
};
379+
requests.push(read_request(&mut stream)?);
380+
stream.write_all(response.as_bytes())?;
381+
}
382+
Ok(requests)
383+
});
384+
(url, handle)
385+
}
386+
338387
fn close_after_request() -> (String, thread::JoinHandle<io::Result<String>>) {
339388
let listener = TcpListener::bind("127.0.0.1:0").expect("bind test server");
340389
listener
@@ -520,6 +569,40 @@ mod tests {
520569
assert_eq!(cached.body, LITELLM_BODY);
521570
}
522571

572+
#[test]
573+
fn recovers_after_a_200_models_dev_body_loads_zero_entries_before_304() {
574+
let fixture = Fixture::new();
575+
let dir = fixture.path("http-cache");
576+
let (url, server) = serve_responses_allowing_missing_tail(vec![
577+
ok_response("\"poisoned\"", ZERO_LOADED_MODELS_DEV_BODY),
578+
not_modified_response(),
579+
ok_response("\"fresh\"", MODELS_DEV_BODY),
580+
]);
581+
582+
let first = fetch_json_with_cache_dir(&url, Some(&dir), PricingEndpoint::ModelsDev)
583+
.expect("structurally valid 200 should reach the loader");
584+
assert_eq!(first, ZERO_LOADED_MODELS_DEV_BODY);
585+
assert!(PricingEndpoint::ModelsDev.validates_shape(&first));
586+
assert!(!PricingEndpoint::ModelsDev.validates(&first));
587+
588+
let second = fetch_json_with_cache_dir(&url, Some(&dir), PricingEndpoint::ModelsDev)
589+
.expect("a zero-loaded cache body must be refreshed after 304");
590+
let requests = server
591+
.join()
592+
.expect("test server")
593+
.expect("server response");
594+
595+
assert_eq!(second, MODELS_DEV_BODY);
596+
assert_eq!(requests.len(), 3, "200 -> 304 -> unconditional 200");
597+
assert!(!requests[0].to_ascii_lowercase().contains("if-none-match"));
598+
assert!(
599+
requests[1]
600+
.to_ascii_lowercase()
601+
.contains("if-none-match: \"poisoned\"")
602+
);
603+
assert!(!requests[2].to_ascii_lowercase().contains("if-none-match"));
604+
}
605+
523606
#[test]
524607
fn does_not_reuse_a_models_dev_body_for_a_litellm_304() {
525608
let fixture = Fixture::new();

0 commit comments

Comments
 (0)