Skip to content

Commit 519a249

Browse files
committed
refactor(adapter): hoist JSONL scan loop into shared helper
The pi, kimi, qwen, openclaw, and copilot parsers all repeated the same three-step boilerplate after the previous commit: iterate `byte_lines`, skip lines rejected by the `LinePrefilter`, and `serde_json::from_slice` each surviving line into a `Value`. Move that loop into a single `prefiltered_json_values` helper in `fast.rs` that yields the parsed `Value`s, and have every line-delimited adapter consume it. This removes the duplicated scan/parse glue and leaves each adapter with only its own per-record logic. No behavior change: the helper performs the identical filter-then-parse sequence and drops unparseable lines exactly as the inline `let Ok(..) else { continue }` did.
1 parent 56dceb0 commit 519a249

6 files changed

Lines changed: 60 additions & 33 deletions

File tree

‎rust/crates/ccusage/src/adapter/copilot/parser.rs‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ use serde_json::{Map, Value};
88

99
use crate::{
1010
Result, TimestampMs, TokenUsageRaw, apply_total_token_fallback,
11-
fast::{LinePrefilter, byte_lines},
11+
fast::{LinePrefilter, prefiltered_json_values},
1212
};
1313

1414
#[derive(Debug, Clone)]
@@ -58,9 +58,7 @@ struct CopilotUsageCandidate {
5858
pub(super) fn parse_otel_file(path: &Path) -> Result<Vec<CopilotUsageEntry>> {
5959
let content = fs::read(path)?;
6060
let prefilter = LinePrefilter::all(&[br#""attributes""#]);
61-
let records = byte_lines(&content)
62-
.filter(|line| prefilter.matches(line))
63-
.filter_map(|line| serde_json::from_slice::<Value>(line).ok())
61+
let records = prefiltered_json_values(&content, &prefilter)
6462
.filter_map(|value| value.as_object().cloned())
6563
.collect::<Vec<_>>();
6664
let trace_contexts = collect_trace_contexts(&records);

‎rust/crates/ccusage/src/adapter/kimi/parser.rs‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ use crate::{
1111
LoadedEntry, PricingMap, Result, TimestampMs, TokenUsageRaw, UsageEntry, UsageMessage,
1212
apply_total_token_fallback, calculate_cost_for_usage,
1313
cli::CostMode,
14-
fast::{LinePrefilter, byte_lines},
14+
fast::{LinePrefilter, prefiltered_json_values},
1515
format_date_tz, json_value_u64, missing_pricing_model_for_candidates, non_empty_json_string,
1616
};
1717

@@ -38,9 +38,7 @@ pub(super) fn read_wire_file(path: &Path) -> Result<Vec<KimiUsageEntry>> {
3838
let fallback_timestamp = file_modified_timestamp(path);
3939
let content = fs::read(path)?;
4040
let prefilter = LinePrefilter::all(&[br#""StatusUpdate""#, br#""token_usage""#]);
41-
Ok(byte_lines(&content)
42-
.filter(|line| prefilter.matches(line))
43-
.filter_map(|line| serde_json::from_slice::<Value>(line).ok())
41+
Ok(prefiltered_json_values(&content, &prefilter)
4442
.filter_map(|value| wire_line_to_entry(&value, path, &model, fallback_timestamp))
4543
.collect::<Vec<_>>())
4644
}

‎rust/crates/ccusage/src/adapter/openclaw/parser.rs‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ use crate::{
77
LoadedEntry, PricingMap, Result, TimestampMs, TokenUsageRaw, UsageEntry, UsageMessage,
88
apply_total_token_fallback, calculate_cost_for_usage,
99
cli::CostMode,
10-
fast::{LinePrefilter, byte_lines},
10+
fast::{LinePrefilter, prefiltered_json_values},
1111
format_date_tz, json_value_u64, missing_pricing_model_for_usage, non_empty_json_string,
1212
};
1313

@@ -40,13 +40,7 @@ pub(super) fn parse_session_file(
4040
let mut entries = Vec::new();
4141
let prefilter =
4242
LinePrefilter::any(&[br#""model_change""#, br#""model-snapshot""#, br#""usage""#]);
43-
for line in byte_lines(&content) {
44-
if !prefilter.matches(line) {
45-
continue;
46-
}
47-
let Ok(value) = serde_json::from_slice::<Value>(line) else {
48-
continue;
49-
};
43+
for value in prefiltered_json_values(&content, &prefilter) {
5044
let Some(record) = value.as_object() else {
5145
continue;
5246
};

‎rust/crates/ccusage/src/adapter/pi/parser.rs‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ use crate::{
77
LoadedEntry, PricingMap, Result, TokenUsageRaw, UsageEntry, UsageMessage,
88
apply_total_token_fallback, calculate_cost_for_usage,
99
cli::CostMode,
10-
fast::{LinePrefilter, byte_lines},
10+
fast::{LinePrefilter, prefiltered_json_values},
1111
format_date_tz, json_value_u64, missing_pricing_model_for_usage, non_empty_json_string,
1212
};
1313

@@ -23,13 +23,7 @@ pub(crate) fn read_session_file(
2323
let mut entries = Vec::new();
2424

2525
let prefilter = LinePrefilter::all(&[br#""usage""#, br#""message""#]);
26-
for line in byte_lines(&content) {
27-
if !prefilter.matches(line) {
28-
continue;
29-
}
30-
let Ok(value) = serde_json::from_slice::<Value>(line) else {
31-
continue;
32-
};
26+
for value in prefiltered_json_values(&content, &prefilter) {
3327
if !is_pi_message_usage(&value) {
3428
continue;
3529
}

‎rust/crates/ccusage/src/adapter/qwen/parser.rs‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ use crate::{
1414
LoadedEntry, PricingMap, Result, TimestampMs, TokenUsageRaw, UsageEntry, UsageMessage,
1515
apply_total_token_fallback, calculate_cost_for_usage,
1616
cli::{CostMode, SharedArgs},
17-
fast::{LinePrefilter, byte_lines},
17+
fast::{LinePrefilter, prefiltered_json_values},
1818
format_date_tz, format_rfc3339_millis, json_value_u64, missing_pricing_model_for_candidates,
1919
non_empty_json_string, parse_ts_timestamp, parse_tz,
2020
};
@@ -56,13 +56,7 @@ fn read_chat_file(
5656
let content = fs::read(file)?;
5757
let mut entries = Vec::new();
5858
let prefilter = LinePrefilter::all(&[br#""usageMetadata""#]);
59-
for line in byte_lines(&content) {
60-
if !prefilter.matches(line) {
61-
continue;
62-
}
63-
let Ok(value) = serde_json::from_slice::<Value>(line) else {
64-
continue;
65-
};
59+
for value in prefiltered_json_values(&content, &prefilter) {
6660
if let Some(entry) = parse_line(file, fallback, &value, tz, mode, pricing) {
6761
entries.push(entry);
6862
}

‎rust/crates/ccusage/src/fast.rs‎

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
use memchr::{memchr, memmem::Finder};
2+
use serde_json::Value;
23
use smallvec::SmallVec;
34

45
pub(crate) type FxHashMap<K, V> = rustc_hash::FxHashMap<K, V>;
@@ -103,6 +104,33 @@ pub(crate) fn byte_lines(bytes: &[u8]) -> ByteLines<'_> {
103104
ByteLines::new(bytes)
104105
}
105106

107+
/// Parse newline-delimited JSON, yielding each owned [`Value`] whose source
108+
/// line passes `prefilter`.
109+
///
110+
/// This is the shared fast-path used by the line-delimited agent adapters: it
111+
/// iterates the raw bytes with [`byte_lines`], skips lines rejected by the
112+
/// reusable [`LinePrefilter`] before paying for a parse, and silently drops
113+
/// lines that fail to deserialize (matching the per-adapter `let Ok(..) else`
114+
/// behavior). Callers keep ownership of `content` and the `prefilter`.
115+
///
116+
/// # Example
117+
///
118+
/// ```ignore
119+
/// let content = std::fs::read(path)?;
120+
/// let prefilter = LinePrefilter::all(&[b"\"usage\""]);
121+
/// for value in prefiltered_json_values(&content, &prefilter) {
122+
/// // adapter-specific handling of `value`
123+
/// }
124+
/// ```
125+
pub(crate) fn prefiltered_json_values<'a>(
126+
content: &'a [u8],
127+
prefilter: &'a LinePrefilter,
128+
) -> impl Iterator<Item = Value> + 'a {
129+
byte_lines(content)
130+
.filter(move |line| prefilter.matches(line))
131+
.filter_map(|line| serde_json::from_slice::<Value>(line).ok())
132+
}
133+
106134
pub(crate) fn suffix_string(value: &str, suffix: &str) -> String {
107135
let mut output = String::with_capacity(value.len() + suffix.len());
108136
output.push_str(value);
@@ -112,7 +140,28 @@ pub(crate) fn suffix_string(value: &str, suffix: &str) -> String {
112140

113141
#[cfg(test)]
114142
mod tests {
115-
use super::{LinePrefilter, byte_lines, suffix_string};
143+
use super::{LinePrefilter, byte_lines, prefiltered_json_values, suffix_string};
144+
145+
#[test]
146+
fn prefiltered_json_values_skips_filtered_and_malformed_lines() {
147+
let content = concat!(
148+
r#"{"usage":{"input":1}}"#,
149+
"\n",
150+
r#"{"role":"user"}"#,
151+
"\n",
152+
r#"not json but has "usage""#,
153+
"\n",
154+
r#"{"usage":{"input":2}}"#,
155+
)
156+
.as_bytes();
157+
let prefilter = LinePrefilter::all(&[b"\"usage\""]);
158+
159+
let inputs = prefiltered_json_values(content, &prefilter)
160+
.filter_map(|value| value.get("usage")?.get("input")?.as_u64())
161+
.collect::<Vec<_>>();
162+
163+
assert_eq!(inputs, [1, 2]);
164+
}
116165

117166
#[test]
118167
fn line_prefilter_all_requires_every_marker() {

0 commit comments

Comments
 (0)