Skip to content

Commit cc48cad

Browse files
jay-tauCopilot
andcommitted
feat(adapter/copilot): harden totalNanoAiu deserialization against float values + fix line-number rot
External PR ccusage#1209 code review (recommended before merge): the `total_nano_aiu` doc-comment acknowledged that a future Copilot CLI emitting the field as a JSON float (e.g. `1.5e11` or `100000000000.0`) would fail the whole `ShutdownEvent` deserialize and drop every model in that shutdown — the exact failure class fixed for `requests.cost`. The previous fix only documented the future hardening; this commit actually performs it. Replace `#[serde(default)]` on both `ShutdownData::total_nano_aiu` and `ModelMetrics::total_nano_aiu` with `#[serde(default, deserialize_with = "deserialize_nano_aiu")]`. The new helper accepts BOTH integer and float JSON forms, truncates floats to `u64`, and rejects negative / non-finite / overflow values via serde errors (the whole event is then skipped via the existing tolerate-malformed-line path — preferable to silently coercing to zero and over-suppressing the aggregate-credit guard). Three regression tests pin the behavior: - `tolerates_float_valued_per_model_total_nano_aiu`: mixes a float-valued per-model AIU with an integer-cost sibling so both rows must survive (would have dropped the whole event under u64-only). - `tolerates_float_valued_aggregate_total_nano_aiu`: the synthetic aggregate-credit entry must still emit when the session-level `totalNanoAiu` is a float. - `rejects_negative_float_total_nano_aiu`: negative values surface as a deserialize error and drop the event (NOT coerced to zero). Also addresses the reviewer's Nit about brittle line-number references rotting: replace `loader.rs:64`, `summary.rs:56-57`, and `data-loader.ts:206-214` cites in parser.rs comments with stable symbol names (`TokenCounts::total`, `UsageAccumulator::add_entry`, `skipMetricsRow`). Co-authored-by: Copilot <[email protected]>
1 parent 9644dcf commit cc48cad

1 file changed

Lines changed: 190 additions & 25 deletions

File tree

  • rust/crates/ccusage/src/adapter/copilot

‎rust/crates/ccusage/src/adapter/copilot/parser.rs‎

Lines changed: 190 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -589,18 +589,17 @@ struct ShutdownEvent {
589589
struct ShutdownData {
590590
#[serde(default)]
591591
model_metrics: HashMap<String, ModelMetrics>,
592-
// `total_nano_aiu` (and its per-model sibling on `ModelMetrics`) are
593-
// typed as `u64` rather than `f64` because "nano" units are integer by
594-
// definition — there's no meaningful fractional nano-AIU. This means
595-
// a future Copilot CLI that emits the field as a JSON float (e.g.
596-
// `1.5e11` or `100000000000.0`) would fail the whole `ShutdownEvent`
597-
// deserialize and drop every model in that shutdown — the exact
598-
// failure class fixed for `requests.cost` (which IS legitimately
599-
// fractional). Real local data only ever ships integer JSON for AIU,
600-
// so the risk is low; if it ever changes, switch to a tolerant
601-
// deserializer that accepts either form and rejects non-zero
602-
// fractional parts.
603-
#[serde(default)]
592+
// `total_nano_aiu` (and its per-model sibling on `ModelMetrics`) is typed
593+
// as `u64` because "nano" units are integer by definition — there's no
594+
// meaningful fractional nano-AIU. To stay forward-compatible against a
595+
// future Copilot CLI that emits the field as a JSON float (e.g.
596+
// `1.5e11` or `100000000000.0`), the custom deserializer
597+
// `deserialize_nano_aiu` accepts BOTH integer and float JSON forms and
598+
// truncates floats to `u64`. Without this, a float-valued shutdown
599+
// would fail the whole `ShutdownEvent` deserialize and drop every
600+
// model in that shutdown — the exact failure class fixed for
601+
// `requests.cost` (which IS legitimately fractional).
602+
#[serde(default, deserialize_with = "deserialize_nano_aiu")]
604603
total_nano_aiu: Option<u64>,
605604
}
606605

@@ -612,11 +611,69 @@ struct ModelMetrics {
612611
#[serde(default)]
613612
requests: SessionRequests,
614613
// See the `total_nano_aiu` note on `ShutdownData` above for why this
615-
// is `u64` rather than `f64`.
616-
#[serde(default)]
614+
// is `u64` rather than `f64`, and why both forms are accepted.
615+
#[serde(default, deserialize_with = "deserialize_nano_aiu")]
617616
total_nano_aiu: Option<u64>,
618617
}
619618

619+
/// Tolerant deserializer for `totalNanoAiu` that accepts both JSON integer
620+
/// (the only form observed in the local 200-file corpus) and JSON float
621+
/// (a forward-compat shape a future Copilot CLI release could emit). Floats
622+
/// are truncated to `u64`; negative values, NaN, and overflow are rejected
623+
/// with a `serde` error rather than silently coerced.
624+
///
625+
/// Pinned by `tolerates_float_valued_total_nano_aiu` and
626+
/// `tolerates_float_valued_per_model_total_nano_aiu`.
627+
fn deserialize_nano_aiu<'de, D>(deserializer: D) -> std::result::Result<Option<u64>, D::Error>
628+
where
629+
D: serde::Deserializer<'de>,
630+
{
631+
use serde::de::{Error, Unexpected};
632+
use serde::Deserialize;
633+
let value = Option::<Value>::deserialize(deserializer)?;
634+
match value {
635+
None | Some(Value::Null) => Ok(None),
636+
Some(Value::Number(n)) => {
637+
if let Some(u) = n.as_u64() {
638+
Ok(Some(u))
639+
} else if let Some(f) = n.as_f64() {
640+
if !f.is_finite() || f < 0.0 {
641+
Err(D::Error::invalid_value(
642+
Unexpected::Float(f),
643+
&"a non-negative finite number for nano-AIU",
644+
))
645+
} else if f >= (u64::MAX as f64) {
646+
Err(D::Error::invalid_value(
647+
Unexpected::Float(f),
648+
&"a nano-AIU value that fits in u64",
649+
))
650+
} else {
651+
Ok(Some(f as u64))
652+
}
653+
} else {
654+
Err(D::Error::custom(format!(
655+
"nano-AIU number not representable as u64 or f64: {n}"
656+
)))
657+
}
658+
}
659+
Some(other) => Err(D::Error::custom(format!(
660+
"nano-AIU must be a number, got {}",
661+
describe_value(&other),
662+
))),
663+
}
664+
}
665+
666+
fn describe_value(value: &Value) -> &'static str {
667+
match value {
668+
Value::Null => "null",
669+
Value::Bool(_) => "bool",
670+
Value::Number(_) => "number",
671+
Value::String(_) => "string",
672+
Value::Array(_) => "array",
673+
Value::Object(_) => "object",
674+
}
675+
}
676+
620677
#[derive(serde::Deserialize, Default)]
621678
#[serde(rename_all = "camelCase")]
622679
struct SessionUsage {
@@ -770,10 +827,10 @@ pub(super) fn parse_session_state_file(path: &Path) -> Result<Vec<CopilotUsageEn
770827

771828
/// Skip rule (plan finding from PR #957 + GPT-5.5 R12):
772829
///
773-
/// * Skip when all five token fields AND `requests.count` are zero (PR #957
774-
/// `data-loader.ts:206-214`, extended to include `reasoning_tokens` so the
775-
/// rule stays symmetric with `build_session_state_entry`'s provider-aware
776-
/// reasoning-token handling).
830+
/// * Skip when all five token fields AND `requests.count` are zero (mirrors
831+
/// PR #957's `data-loader.ts` `skipMetricsRow` predicate, extended to
832+
/// include `reasoning_tokens` so the rule stays symmetric with
833+
/// `build_session_state_entry`'s provider-aware reasoning-token handling).
777834
/// * BUT keep the row when `metrics.totalNanoAiu` is present and non-zero —
778835
/// real local data contains credit-only rows (zero tokens, zero requests,
779836
/// non-null per-model `totalNanoAiu`) that would otherwise be silently
@@ -826,7 +883,8 @@ fn build_session_state_entry(
826883

827884
// Reasoning semantics differ per provider (plan finding #2):
828885
// - OpenAI (`gpt-*`) and Anthropic (`claude-*`) include reasoning inside
829-
// `usage.outputTokens`. Summing them again in `loader.rs:64` would
886+
// `usage.outputTokens`. Adding it again into the summary's
887+
// `TokenCounts` total (`types.rs::TokenCounts::total`) would
830888
// double-count — store 0 here so the existing summation is a no-op.
831889
// - Google (`gemini-*`) reports reasoning as a separate field. Keep the
832890
// raw value so the same loader-side summation aggregates it correctly.
@@ -896,10 +954,10 @@ fn build_session_state_entry(
896954
}
897955

898956
/// Synthetic zero-token entry that carries an aggregate-only `totalNanoAiu`
899-
/// through the existing summary aggregation. `summary.rs:56-57` accumulates
900-
/// credits unconditionally while breakdowns gate on `entry.model.is_some()`,
901-
/// so an empty model here keeps `modelsUsed` clean while still surfacing the
902-
/// credit total.
957+
/// through the existing summary aggregation. `UsageAccumulator::add_entry`
958+
/// (`summary.rs`) accumulates credits unconditionally while breakdowns gate
959+
/// on `entry.model.is_some()`, so an empty model here keeps `modelsUsed`
960+
/// clean while still surfacing the credit total.
903961
fn build_aggregate_credit_entry(
904962
session_id: &str,
905963
event_id: &str,
@@ -1113,6 +1171,112 @@ mod session_state_tests {
11131171
);
11141172
}
11151173

1174+
#[test]
1175+
fn tolerates_float_valued_per_model_total_nano_aiu() {
1176+
// Forward-compat hardening (sibling to the fractional-cost fix):
1177+
// a future Copilot CLI that ships `totalNanoAiu` as a JSON float
1178+
// (e.g. `1500000000.0` after passing through a JS pipeline that
1179+
// coerces large integers to f64) must NOT fail the whole event
1180+
// deserialize and silently drop every model. The custom
1181+
// deserializer accepts both u64 and f64 forms and truncates floats
1182+
// to `u64`. Mix a float-valued per-model AIU with an integer-cost
1183+
// sibling so the test pins both the float row surviving AND the
1184+
// sibling not being collateral damage from a u64-only failure.
1185+
let (_dir, path) = write_jsonl(&[shutdown_event(
1186+
"evt-naiu-float",
1187+
"2026-05-20T16:01:29.481Z",
1188+
json!({
1189+
"claude-opus-4.7-1m-internal": {
1190+
"usage": {
1191+
"inputTokens": 90_890_709u64,
1192+
"outputTokens": 172_544u64,
1193+
"cacheReadTokens": 86_415_643u64,
1194+
"cacheWriteTokens": 4_474_731u64,
1195+
"reasoningTokens": 0u64
1196+
},
1197+
"requests": {"count": 182, "cost": 4},
1198+
"totalNanoAiu": 1_500_000_000.0_f64
1199+
},
1200+
"claude-sonnet-4.5": {
1201+
"usage": {
1202+
"inputTokens": 2_000u64,
1203+
"outputTokens": 200u64,
1204+
"cacheReadTokens": 0u64,
1205+
"cacheWriteTokens": 0u64,
1206+
"reasoningTokens": 0u64
1207+
},
1208+
"requests": {"count": 1, "cost": 1}
1209+
}
1210+
}),
1211+
)]);
1212+
1213+
let entries = parse_session_state_file(&path).unwrap();
1214+
// Both rows must survive — without the tolerant deserializer the
1215+
// float-valued AIU would fail u64 parsing and drop both models.
1216+
assert_eq!(entries.len(), 2, "got {entries:?}");
1217+
let opus = entries
1218+
.iter()
1219+
.find(|e| e.model == "claude-opus-4.7-1m-internal")
1220+
.expect("opus row");
1221+
assert_eq!(opus.nano_aiu, Some(1_500_000_000));
1222+
let sonnet = entries
1223+
.iter()
1224+
.find(|e| e.model == "claude-sonnet-4.5")
1225+
.expect("sonnet row (integer-cost sibling)");
1226+
assert_eq!(sonnet.premium_request_cost, Some(1.0));
1227+
}
1228+
1229+
#[test]
1230+
fn tolerates_float_valued_aggregate_total_nano_aiu() {
1231+
// Forward-compat hardening for the aggregate (session-level)
1232+
// `data.totalNanoAiu`: a float-valued aggregate must round-trip
1233+
// and reach the synthetic aggregate-credit entry.
1234+
let event = format!(
1235+
"{}\n",
1236+
json!({
1237+
"type": "session.shutdown",
1238+
"id": "evt-agg-float",
1239+
"timestamp": "2026-05-20T16:01:29.481Z",
1240+
"data": {
1241+
"totalNanoAiu": 1_000_000_000.0_f64,
1242+
"modelMetrics": {}
1243+
}
1244+
})
1245+
);
1246+
let (_dir, path) = write_jsonl(&[event]);
1247+
let entries = parse_session_state_file(&path).unwrap();
1248+
assert_eq!(entries.len(), 1);
1249+
assert!(entries[0].dedup_key.starts_with("credit-aggregate:"));
1250+
assert_eq!(entries[0].nano_aiu, Some(1_000_000_000));
1251+
}
1252+
1253+
#[test]
1254+
fn rejects_negative_float_total_nano_aiu() {
1255+
// `totalNanoAiu` is non-negative by definition (nano-units of a
1256+
// credit count). A negative value would be a schema bug; reject
1257+
// rather than coerce. The whole event is dropped via the
1258+
// tolerate-malformed-line path — preferable to silently round to
1259+
// zero and over-suppress the aggregate-credit guard.
1260+
let event = format!(
1261+
"{}\n",
1262+
json!({
1263+
"type": "session.shutdown",
1264+
"id": "evt-naiu-neg",
1265+
"timestamp": "2026-05-20T16:01:29.481Z",
1266+
"data": {
1267+
"totalNanoAiu": -1.0_f64,
1268+
"modelMetrics": {}
1269+
}
1270+
})
1271+
);
1272+
let (_dir, path) = write_jsonl(&[event]);
1273+
let entries = parse_session_state_file(&path).unwrap();
1274+
assert!(
1275+
entries.is_empty(),
1276+
"negative nano-AIU must reject the event (not coerce to zero), got {entries:?}"
1277+
);
1278+
}
1279+
11161280
#[test]
11171281
fn parses_session_shutdown_event() {
11181282
let (_dir, path) = write_jsonl(&[shutdown_event(
@@ -1534,8 +1698,9 @@ mod session_state_tests {
15341698

15351699
#[test]
15361700
fn gpt_reasoning_set_to_zero_to_avoid_double_count() {
1537-
// OpenAI convention: reasoning IS a subset of output. Adding them
1538-
// again at loader.rs:64 would over-count cost ~40%.
1701+
// OpenAI convention: reasoning IS a subset of output. Adding the
1702+
// raw `reasoning_tokens` into the summary again via
1703+
// `TokenCounts::total` (`types.rs`) would over-count cost ~40%.
15391704
let (_dir, path) = write_jsonl(&[shutdown_event(
15401705
"evt-gpt",
15411706
"2026-05-02T15:27:09.013Z",

0 commit comments

Comments
 (0)