Skip to content

Commit 56dceb0

Browse files
committed
perf(adapter): unify JSONL prefilter and byte_lines across agents
Migrate the pi, kimi, qwen, openclaw, and copilot adapters to the same fast line-scanning path the claude and codex adapters already use: read the file as bytes, iterate newline-delimited slices with `byte_lines`, and skip non-usage lines with a reusable `memmem` substring prefilter before paying for a full `serde_json` parse. Add a shared `LinePrefilter` helper to `fast.rs` so the per-line skip check is built once (owned `Finder` needles) and reused across every line via the SIMD-accelerated `memmem` path, instead of allocating a fresh searcher on each `str::contains` call. The helper supports both "contains all markers" and "contains any marker" modes to cover the existing per-adapter filter semantics. Behavioral notes: - pi, kimi, openclaw, copilot keep their exact existing marker substrings; only the scanning mechanism changes. - qwen previously parsed every line; it now skips lines without `"usageMetadata"`, which is a required field for any emitted entry, so the result set is unchanged. - Reading bytes instead of a UTF-8 string makes a single invalid line fail to parse and be skipped rather than aborting the whole file, matching the claude/codex behavior. Verified byte-identical `daily --json --offline` output against ccusage@latest for every agent with local logs (pi, copilot, claude, codex, gemini, opencode, amp), with logs capped at a fixed date.
1 parent cd5d58f commit 56dceb0

6 files changed

Lines changed: 131 additions & 46 deletions

File tree

‎rust/crates/ccusage/src/adapter/copilot/parser.rs‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,10 @@ use std::{
66

77
use serde_json::{Map, Value};
88

9-
use crate::{Result, TimestampMs, TokenUsageRaw, apply_total_token_fallback};
9+
use crate::{
10+
Result, TimestampMs, TokenUsageRaw, apply_total_token_fallback,
11+
fast::{LinePrefilter, byte_lines},
12+
};
1013

1114
#[derive(Debug, Clone)]
1215
pub(super) struct CopilotUsageEntry {
@@ -53,11 +56,11 @@ struct CopilotUsageCandidate {
5356
}
5457

5558
pub(super) fn parse_otel_file(path: &Path) -> Result<Vec<CopilotUsageEntry>> {
56-
let content = fs::read_to_string(path)?;
57-
let records = content
58-
.lines()
59-
.filter(|line| line.contains("\"attributes\""))
60-
.filter_map(|line| serde_json::from_str::<Value>(line).ok())
59+
let content = fs::read(path)?;
60+
let prefilter = LinePrefilter::all(&[br#""attributes""#]);
61+
let records = byte_lines(&content)
62+
.filter(|line| prefilter.matches(line))
63+
.filter_map(|line| serde_json::from_slice::<Value>(line).ok())
6164
.filter_map(|value| value.as_object().cloned())
6265
.collect::<Vec<_>>();
6366
let trace_contexts = collect_trace_contexts(&records);

‎rust/crates/ccusage/src/adapter/kimi/parser.rs‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,10 @@ use serde_json::Value;
99

1010
use crate::{
1111
LoadedEntry, PricingMap, Result, TimestampMs, TokenUsageRaw, UsageEntry, UsageMessage,
12-
apply_total_token_fallback, calculate_cost_for_usage, cli::CostMode, format_date_tz,
13-
json_value_u64, missing_pricing_model_for_candidates, non_empty_json_string,
12+
apply_total_token_fallback, calculate_cost_for_usage,
13+
cli::CostMode,
14+
fast::{LinePrefilter, byte_lines},
15+
format_date_tz, json_value_u64, missing_pricing_model_for_candidates, non_empty_json_string,
1416
};
1517

1618
const DEFAULT_MODEL: &str = "kimi-for-coding";
@@ -34,11 +36,11 @@ pub(super) struct KimiUsageEntry {
3436
pub(super) fn read_wire_file(path: &Path) -> Result<Vec<KimiUsageEntry>> {
3537
let model = read_model_from_config(path);
3638
let fallback_timestamp = file_modified_timestamp(path);
37-
let content = fs::read_to_string(path)?;
38-
Ok(content
39-
.lines()
40-
.filter(|line| line.contains("\"StatusUpdate\"") && line.contains("\"token_usage\""))
41-
.filter_map(|line| serde_json::from_str::<Value>(line).ok())
39+
let content = fs::read(path)?;
40+
let prefilter = LinePrefilter::all(&[br#""StatusUpdate""#, br#""token_usage""#]);
41+
Ok(byte_lines(&content)
42+
.filter(|line| prefilter.matches(line))
43+
.filter_map(|line| serde_json::from_slice::<Value>(line).ok())
4244
.filter_map(|value| wire_line_to_entry(&value, path, &model, fallback_timestamp))
4345
.collect::<Vec<_>>())
4446
}

‎rust/crates/ccusage/src/adapter/openclaw/parser.rs‎

Lines changed: 11 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,14 @@
1-
use std::{
2-
fs,
3-
io::{BufRead, BufReader},
4-
path::Path,
5-
sync::Arc,
6-
};
1+
use std::{fs, path::Path, sync::Arc};
72

83
use jiff::tz::TimeZone as JiffTimeZone;
94
use serde_json::{Map, Value};
105

116
use crate::{
127
LoadedEntry, PricingMap, Result, TimestampMs, TokenUsageRaw, UsageEntry, UsageMessage,
13-
apply_total_token_fallback, calculate_cost_for_usage, cli::CostMode, format_date_tz,
14-
json_value_u64, missing_pricing_model_for_usage, non_empty_json_string,
8+
apply_total_token_fallback, calculate_cost_for_usage,
9+
cli::CostMode,
10+
fast::{LinePrefilter, byte_lines},
11+
format_date_tz, json_value_u64, missing_pricing_model_for_usage, non_empty_json_string,
1512
};
1613

1714
#[derive(Debug, Clone)]
@@ -37,20 +34,17 @@ pub(super) fn parse_session_file(
3734
) -> Result<Vec<LoadedEntry>> {
3835
let session_id = extract_session_id(path);
3936
let fallback_timestamp = file_modified_timestamp(path);
40-
let input = fs::File::open(path)?;
41-
let reader = BufReader::new(input);
37+
let content = fs::read(path)?;
4238
let mut current_model = None::<String>;
4339
let mut current_provider = None::<String>;
4440
let mut entries = Vec::new();
45-
for line in reader.lines() {
46-
let line = line?;
47-
if !line.contains("\"model_change\"")
48-
&& !line.contains("\"model-snapshot\"")
49-
&& !line.contains("\"usage\"")
50-
{
41+
let prefilter =
42+
LinePrefilter::any(&[br#""model_change""#, br#""model-snapshot""#, br#""usage""#]);
43+
for line in byte_lines(&content) {
44+
if !prefilter.matches(line) {
5145
continue;
5246
}
53-
let Ok(value) = serde_json::from_str::<Value>(&line) else {
47+
let Ok(value) = serde_json::from_slice::<Value>(line) else {
5448
continue;
5549
};
5650
let Some(record) = value.as_object() else {

‎rust/crates/ccusage/src/adapter/pi/parser.rs‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,10 @@ use serde_json::Value;
55

66
use crate::{
77
LoadedEntry, PricingMap, Result, TokenUsageRaw, UsageEntry, UsageMessage,
8-
apply_total_token_fallback, calculate_cost_for_usage, cli::CostMode, format_date_tz,
9-
json_value_u64, missing_pricing_model_for_usage, non_empty_json_string,
8+
apply_total_token_fallback, calculate_cost_for_usage,
9+
cli::CostMode,
10+
fast::{LinePrefilter, byte_lines},
11+
format_date_tz, json_value_u64, missing_pricing_model_for_usage, non_empty_json_string,
1012
};
1113

1214
pub(crate) fn read_session_file(
@@ -15,16 +17,17 @@ pub(crate) fn read_session_file(
1517
mode: CostMode,
1618
pricing: Option<&PricingMap>,
1719
) -> Result<Vec<LoadedEntry>> {
18-
let content = fs::read_to_string(path)?;
20+
let content = fs::read(path)?;
1921
let project = extract_project(path);
2022
let session_id = extract_session_id(path);
2123
let mut entries = Vec::new();
2224

23-
for line in content.lines() {
24-
if !line.contains("\"usage\"") || !line.contains("\"message\"") {
25+
let prefilter = LinePrefilter::all(&[br#""usage""#, br#""message""#]);
26+
for line in byte_lines(&content) {
27+
if !prefilter.matches(line) {
2528
continue;
2629
}
27-
let Ok(value) = serde_json::from_str::<Value>(line) else {
30+
let Ok(value) = serde_json::from_slice::<Value>(line) else {
2831
continue;
2932
};
3033
if !is_pi_message_usage(&value) {

‎rust/crates/ccusage/src/adapter/qwen/parser.rs‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
use std::{
22
collections::HashSet,
3-
fs::{self, File},
4-
io::{BufRead, BufReader},
3+
fs,
54
path::Path,
65
sync::Arc,
76
time::{SystemTime, UNIX_EPOCH},
@@ -15,6 +14,7 @@ use crate::{
1514
LoadedEntry, PricingMap, Result, TimestampMs, TokenUsageRaw, UsageEntry, UsageMessage,
1615
apply_total_token_fallback, calculate_cost_for_usage,
1716
cli::{CostMode, SharedArgs},
17+
fast::{LinePrefilter, byte_lines},
1818
format_date_tz, format_rfc3339_millis, json_value_u64, missing_pricing_model_for_candidates,
1919
non_empty_json_string, parse_ts_timestamp, parse_tz,
2020
};
@@ -53,14 +53,14 @@ fn read_chat_file(
5353
shared: &SharedArgs,
5454
) -> Result<Vec<LoadedEntry>> {
5555
let fallback = file_timestamp(file, shared);
56-
let input = File::open(file)?;
57-
let reader = BufReader::new(input);
56+
let content = fs::read(file)?;
5857
let mut entries = Vec::new();
59-
for line in reader.lines() {
60-
let Ok(line) = line else {
58+
let prefilter = LinePrefilter::all(&[br#""usageMetadata""#]);
59+
for line in byte_lines(&content) {
60+
if !prefilter.matches(line) {
6161
continue;
62-
};
63-
let Ok(value) = serde_json::from_str::<Value>(&line) else {
62+
}
63+
let Ok(value) = serde_json::from_slice::<Value>(line) else {
6464
continue;
6565
};
6666
if let Some(entry) = parse_line(file, fallback, &value, tz, mode, pricing) {

‎rust/crates/ccusage/src/fast.rs‎

Lines changed: 85 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,75 @@
1-
use memchr::memchr;
1+
use memchr::{memchr, memmem::Finder};
22
use smallvec::SmallVec;
33

44
pub(crate) type FxHashMap<K, V> = rustc_hash::FxHashMap<K, V>;
55
pub(crate) type FxHashSet<T> = rustc_hash::FxHashSet<T>;
66
pub(crate) type SmallIndexVec = SmallVec<[usize; 1]>;
77

8+
/// Whether a [`LinePrefilter`] requires every marker or just one of them.
9+
#[derive(Clone, Copy)]
10+
enum PrefilterMode {
11+
/// The line must contain every configured marker.
12+
All,
13+
/// The line must contain at least one configured marker.
14+
Any,
15+
}
16+
17+
/// Reusable byte-substring prefilter for newline-delimited JSON logs.
18+
///
19+
/// JSONL adapters skip lines that cannot contain a usage record before paying
20+
/// for a full `serde_json` parse. Building the [`Finder`] needles once and
21+
/// reusing them across every line keeps that skip check on the SIMD-accelerated
22+
/// `memmem` path instead of allocating a fresh searcher per `str::contains`
23+
/// call.
24+
pub(crate) struct LinePrefilter {
25+
finders: SmallVec<[Finder<'static>; 4]>,
26+
mode: PrefilterMode,
27+
}
28+
29+
impl LinePrefilter {
30+
/// Build a prefilter that only admits lines containing *all* `markers`.
31+
///
32+
/// # Example
33+
///
34+
/// ```ignore
35+
/// let prefilter = LinePrefilter::all(&[b"\"usage\"", b"\"message\""]);
36+
/// assert!(prefilter.matches(br#"{"message":{"usage":{}}}"#));
37+
/// assert!(!prefilter.matches(br#"{"message":{}}"#));
38+
/// ```
39+
pub(crate) fn all(markers: &[&[u8]]) -> Self {
40+
Self::new(markers, PrefilterMode::All)
41+
}
42+
43+
/// Build a prefilter that admits lines containing *any* of `markers`.
44+
pub(crate) fn any(markers: &[&[u8]]) -> Self {
45+
Self::new(markers, PrefilterMode::Any)
46+
}
47+
48+
fn new(markers: &[&[u8]], mode: PrefilterMode) -> Self {
49+
// `Finder::new` borrows the needle, so take an owned copy to outlive the
50+
// caller's marker slice and allow the prefilter to be stored freely.
51+
let finders = markers
52+
.iter()
53+
.map(|marker| Finder::new(marker).into_owned())
54+
.collect();
55+
Self { finders, mode }
56+
}
57+
58+
/// Return `true` when `line` passes the filter and is worth parsing.
59+
pub(crate) fn matches(&self, line: &[u8]) -> bool {
60+
match self.mode {
61+
PrefilterMode::All => self
62+
.finders
63+
.iter()
64+
.all(|finder| finder.find(line).is_some()),
65+
PrefilterMode::Any => self
66+
.finders
67+
.iter()
68+
.any(|finder| finder.find(line).is_some()),
69+
}
70+
}
71+
}
72+
873
pub(crate) struct ByteLines<'a> {
974
bytes: &'a [u8],
1075
}
@@ -47,7 +112,25 @@ pub(crate) fn suffix_string(value: &str, suffix: &str) -> String {
47112

48113
#[cfg(test)]
49114
mod tests {
50-
use super::{byte_lines, suffix_string};
115+
use super::{LinePrefilter, byte_lines, suffix_string};
116+
117+
#[test]
118+
fn line_prefilter_all_requires_every_marker() {
119+
let prefilter = LinePrefilter::all(&[b"\"usage\"", b"\"message\""]);
120+
121+
assert!(prefilter.matches(br#"{"message":{"usage":{"input":1}}}"#));
122+
assert!(!prefilter.matches(br#"{"message":{"role":"user"}}"#));
123+
assert!(!prefilter.matches(br#"{"usage":{"input":1}}"#));
124+
}
125+
126+
#[test]
127+
fn line_prefilter_any_requires_one_marker() {
128+
let prefilter = LinePrefilter::any(&[b"\"model_change\"", b"\"usage\""]);
129+
130+
assert!(prefilter.matches(br#"{"type":"model_change"}"#));
131+
assert!(prefilter.matches(br#"{"message":{"usage":{}}}"#));
132+
assert!(!prefilter.matches(br#"{"type":"message"}"#));
133+
}
51134

52135
#[test]
53136
fn byte_lines_returns_newline_delimited_slices() {

0 commit comments

Comments
 (0)