Skip to content

Commit d9c8ff2

Browse files
committed
perf(adapter): prefilter JSONL lines with a shared multi-marker LinePrefilter
Adopt the `LinePrefilter` helper (originally proposed in #1326) so the typed `jsonl::records` path can prefilter on multiple markers with AND/OR semantics instead of a single substring. `records` now takes `Option<&LinePrefilter>`. This restores the precise prefilters that single-marker filtering had to drop: - pi: require both `"usage"` and `"message"` (was `"usage"` only) - kimi: require both `"StatusUpdate"` and `"token_usage"` (was `"token_usage"` only) - openclaw: admit any of `"model_change"` / `"model-snapshot"` / `"usage"` (previously had no prefilter at all) - qwen / copilot: unchanged markers, now via LinePrefilter `LinePrefilter` builds the `memmem` needles once and reuses them across every line. Output remains byte-for-byte identical (verified against ccusage@latest for all agents with local logs). Supersedes #1326 by folding its prefilter into the typed-parsing refactor.
1 parent b2bdf4f commit d9c8ff2

7 files changed

Lines changed: 130 additions & 53 deletions

File tree

‎rust/crates/ccusage/src/adapter/copilot/parser.rs‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,7 @@ use serde::Deserialize;
88
use serde_json::{Map, Value};
99

1010
use super::super::jsonl;
11-
use crate::{Result, TimestampMs, TokenUsageRaw, apply_total_token_fallback};
12-
13-
/// Cheap substring prefilter: every usable Copilot OTel record carries the
14-
/// `attributes` object, so lines without it are skipped before JSON parsing.
15-
const COPILOT_ATTRIBUTES_MARKER: &[u8] = b"\"attributes\"";
11+
use crate::{Result, TimestampMs, TokenUsageRaw, apply_total_token_fallback, fast::LinePrefilter};
1612

1713
/// A single parsed Copilot OpenTelemetry record. Only the fields ccusage
1814
/// consumes are declared; serde skips everything else. The `attributes` block
@@ -99,8 +95,10 @@ struct CopilotUsageCandidate {
9995

10096
pub(super) fn parse_otel_file(path: &Path) -> Result<Vec<CopilotUsageEntry>> {
10197
let content = fs::read(path)?;
102-
let records = jsonl::records::<CopilotRecord>(&content, Some(COPILOT_ATTRIBUTES_MARKER))
103-
.collect::<Vec<_>>();
98+
// Every usable Copilot OTel record carries the `attributes` object, so
99+
// lines without it are skipped before JSON parsing.
100+
let prefilter = LinePrefilter::all(&[br#""attributes""#]);
101+
let records = jsonl::records::<CopilotRecord>(&content, Some(&prefilter)).collect::<Vec<_>>();
104102
let trace_contexts = collect_trace_contexts(&records);
105103
let fallback_timestamp = file_modified_timestamp(path);
106104
let candidates = records

‎rust/crates/ccusage/src/adapter/jsonl.rs‎

Lines changed: 13 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -14,17 +14,16 @@
1414
//! `serde_json::from_slice`, so unused fields are skipped instead of being
1515
//! materialized into an intermediate `Value` tree.
1616
17-
use memchr::memmem;
1817
use serde::{Deserialize, Deserializer, de::DeserializeOwned};
1918

20-
use crate::fast::byte_lines;
19+
use crate::fast::{LinePrefilter, byte_lines};
2120

2221
/// Iterate over deserialized JSONL records contained in `content`.
2322
///
24-
/// When `marker` is provided, lines that do not contain that byte substring are
25-
/// skipped before any JSON parsing, mirroring the per-line `memmem` prefilter
26-
/// used by the Claude loader. Pick a marker that appears in every line the
27-
/// adapter would accept (for example the required `"usage"` key) so the
23+
/// When `prefilter` is provided, lines that it rejects are skipped before any
24+
/// JSON parsing, mirroring the per-line `memmem` prefilter used by the Claude
25+
/// loader. Build the [`LinePrefilter`] from markers that appear in every line
26+
/// the adapter would accept (for example the required `"usage"` key) so the
2827
/// prefilter never drops a usable record. Pass `None` to parse every line.
2928
///
3029
/// Lines that fail to deserialize into `T` are silently skipped, matching the
@@ -39,24 +38,22 @@ use crate::fast::byte_lines;
3938
/// }
4039
///
4140
/// let content = b"{\"model\":\"qwen3-coder\"}\n{}\n";
42-
/// let models: Vec<_> = jsonl::records::<Record>(content, Some(b"model"))
41+
/// let prefilter = LinePrefilter::all(&[b"model"]);
42+
/// let models: Vec<_> = jsonl::records::<Record>(content, Some(&prefilter))
4343
/// .filter_map(|record| record.model)
4444
/// .collect();
4545
/// assert_eq!(models, ["qwen3-coder"]);
4646
/// ```
4747
pub(crate) fn records<'data, T>(
4848
content: &'data [u8],
49-
marker: Option<&[u8]>,
49+
prefilter: Option<&'data LinePrefilter>,
5050
) -> impl Iterator<Item = T> + 'data
5151
where
5252
T: DeserializeOwned + 'data,
5353
{
54-
// Build the finder once and move it into the closure so the needle is
55-
// compiled a single time and reused for every line in the file.
56-
let finder = marker.map(|needle| memmem::Finder::new(needle).into_owned());
5754
byte_lines(content).filter_map(move |line| {
58-
if let Some(finder) = &finder
59-
&& finder.find(line).is_none()
55+
if let Some(prefilter) = prefilter
56+
&& !prefilter.matches(line)
6057
{
6158
return None;
6259
}
@@ -141,6 +138,7 @@ mod tests {
141138
use serde::Deserialize;
142139

143140
use super::{lenient_f64, lenient_i64, lenient_u64, non_empty_string, records};
141+
use crate::fast::LinePrefilter;
144142

145143
#[derive(Debug, PartialEq, Deserialize)]
146144
struct Record {
@@ -154,7 +152,8 @@ mod tests {
154152
fn records_skips_lines_without_marker() {
155153
let content =
156154
b"{\"model\":\"a\",\"tokens\":1}\n{\"other\":true}\n{\"model\":\"b\",\"tokens\":2}\n";
157-
let parsed = records::<Record>(content, Some(b"model")).collect::<Vec<_>>();
155+
let prefilter = LinePrefilter::all(&[b"model"]);
156+
let parsed = records::<Record>(content, Some(&prefilter)).collect::<Vec<_>>();
158157

159158
assert_eq!(
160159
parsed,

‎rust/crates/ccusage/src/adapter/kimi/parser.rs‎

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -10,19 +10,14 @@ use serde::Deserialize;
1010
use super::super::jsonl;
1111
use crate::{
1212
LoadedEntry, PricingMap, Result, TimestampMs, TokenUsageRaw, UsageEntry, UsageMessage,
13-
apply_total_token_fallback, calculate_cost_for_usage, cli::CostMode, format_date_tz,
14-
missing_pricing_model_for_candidates,
13+
apply_total_token_fallback, calculate_cost_for_usage, cli::CostMode, fast::LinePrefilter,
14+
format_date_tz, missing_pricing_model_for_candidates,
1515
};
1616

1717
const DEFAULT_MODEL: &str = "kimi-for-coding";
1818
const DEFAULT_PROVIDER: &str = "moonshot";
1919
const KIMI_FOR_CODING_K2_6_CUTOFF_MS: i64 = 1_776_698_890_072;
2020

21-
/// Cheap substring prefilter: every usable Kimi wire line carries a token usage
22-
/// payload under the `token_usage` key, so lines without it are skipped before
23-
/// JSON parsing.
24-
const KIMI_TOKEN_USAGE_MARKER: &[u8] = b"\"token_usage\"";
25-
2621
/// Kimi `config.json` document, used to read the configured display model.
2722
#[derive(Debug, Deserialize)]
2823
struct KimiConfig {
@@ -90,11 +85,12 @@ pub(super) fn read_wire_file(path: &Path) -> Result<Vec<KimiUsageEntry>> {
9085
let model = read_model_from_config(path);
9186
let fallback_timestamp = file_modified_timestamp(path);
9287
let content = fs::read(path)?;
93-
Ok(
94-
jsonl::records::<KimiWireLine>(&content, Some(KIMI_TOKEN_USAGE_MARKER))
95-
.filter_map(|line| wire_line_to_entry(&line, path, &model, fallback_timestamp))
96-
.collect::<Vec<_>>(),
97-
)
88+
// Usable Kimi wire lines are `StatusUpdate` records carrying a
89+
// `token_usage` payload, so require both substrings before JSON parsing.
90+
let prefilter = LinePrefilter::all(&[br#""StatusUpdate""#, br#""token_usage""#]);
91+
Ok(jsonl::records::<KimiWireLine>(&content, Some(&prefilter))
92+
.filter_map(|line| wire_line_to_entry(&line, path, &model, fallback_timestamp))
93+
.collect::<Vec<_>>())
9894
}
9995

10096
fn read_model_from_config(file_path: &Path) -> String {

‎rust/crates/ccusage/src/adapter/openclaw/parser.rs‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ use serde_json::Value;
77
use super::super::jsonl;
88
use crate::{
99
LoadedEntry, PricingMap, Result, TimestampMs, TokenUsageRaw, UsageEntry, UsageMessage,
10-
apply_total_token_fallback, calculate_cost_for_usage, cli::CostMode, format_date_tz,
11-
missing_pricing_model_for_usage,
10+
apply_total_token_fallback, calculate_cost_for_usage, cli::CostMode, fast::LinePrefilter,
11+
format_date_tz, missing_pricing_model_for_usage,
1212
};
1313

1414
/// A single parsed OpenClaw session line. Only the fields ccusage consumes are
@@ -164,10 +164,15 @@ pub(super) fn parse_session_file(
164164
let session_id = extract_session_id(path);
165165
let fallback_timestamp = file_modified_timestamp(path);
166166
let content = fs::read(path)?;
167+
// OpenClaw lines that matter are either model-tracking records
168+
// (`model_change`/`model-snapshot`) or assistant records carrying `usage`,
169+
// so admit lines containing any of those substrings before JSON parsing.
170+
let prefilter =
171+
LinePrefilter::any(&[br#""model_change""#, br#""model-snapshot""#, br#""usage""#]);
167172
let mut current_model = None::<String>;
168173
let mut current_provider = None::<String>;
169174
let mut entries = Vec::new();
170-
for record in jsonl::records::<OpenClawLine>(&content, None) {
175+
for record in jsonl::records::<OpenClawLine>(&content, Some(&prefilter)) {
171176
if is_model_change(&record) {
172177
let (source_model_id, source_model, source_provider) = match record.data.as_ref() {
173178
Some(source) => (

‎rust/crates/ccusage/src/adapter/pi/parser.rs‎

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,17 +6,10 @@ use serde::Deserialize;
66
use super::super::jsonl;
77
use crate::{
88
LoadedEntry, PricingMap, Result, TokenUsageRaw, UsageEntry, UsageMessage,
9-
apply_total_token_fallback, calculate_cost_for_usage, cli::CostMode, format_date_tz,
10-
missing_pricing_model_for_usage,
9+
apply_total_token_fallback, calculate_cost_for_usage, cli::CostMode, fast::LinePrefilter,
10+
format_date_tz, missing_pricing_model_for_usage,
1111
};
1212

13-
/// Cheap substring prefilter: every usable pi line carries token counts under
14-
/// the `usage` key, so lines without it are skipped before JSON parsing. The
15-
/// companion `"message"` requirement is enforced downstream by
16-
/// [`is_pi_message_usage`], which only accepts records whose message object
17-
/// carries a usage block.
18-
const PI_USAGE_MARKER: &[u8] = b"\"usage\"";
19-
2013
/// A single parsed pi session record. Only the fields ccusage consumes are
2114
/// declared; serde skips everything else.
2215
#[derive(Debug, Deserialize)]
@@ -78,9 +71,12 @@ pub(crate) fn read_session_file(
7871
let content = fs::read(path)?;
7972
let project = extract_project(path);
8073
let session_id = extract_session_id(path);
74+
// Usable pi lines carry token counts under a `usage` key nested in a
75+
// `message` object, so require both substrings before JSON parsing.
76+
let prefilter = LinePrefilter::all(&[br#""usage""#, br#""message""#]);
8177
let mut entries = Vec::new();
8278

83-
for record in jsonl::records::<PiLine>(&content, Some(PI_USAGE_MARKER)) {
79+
for record in jsonl::records::<PiLine>(&content, Some(&prefilter)) {
8480
if !is_pi_message_usage(&record) {
8581
continue;
8682
}

‎rust/crates/ccusage/src/adapter/qwen/parser.rs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,16 +14,13 @@ use crate::{
1414
LoadedEntry, PricingMap, Result, TimestampMs, TokenUsageRaw, UsageEntry, UsageMessage,
1515
apply_total_token_fallback, calculate_cost_for_usage,
1616
cli::{CostMode, SharedArgs},
17+
fast::LinePrefilter,
1718
format_date_tz, format_rfc3339_millis, missing_pricing_model_for_candidates,
1819
parse_ts_timestamp, parse_tz,
1920
};
2021

2122
const DEFAULT_QWEN_MODEL: &str = "unknown";
2223

23-
/// Cheap substring prefilter: every usable Qwen line carries token counts under
24-
/// the `usageMetadata` key, so lines without it are skipped before JSON parsing.
25-
const QWEN_USAGE_MARKER: &[u8] = b"usageMetadata";
26-
2724
/// A single parsed Qwen chat record. Only the fields ccusage consumes are
2825
/// declared; serde skips everything else.
2926
#[derive(Debug, Deserialize)]
@@ -89,8 +86,11 @@ fn read_chat_file(
8986
) -> Result<Vec<LoadedEntry>> {
9087
let fallback = file_timestamp(file, shared);
9188
let content = fs::read(file)?;
89+
// Every usable Qwen line carries token counts under the `usageMetadata`
90+
// key, so lines without it are skipped before JSON parsing.
91+
let prefilter = LinePrefilter::all(&[br#""usageMetadata""#]);
9292
let mut entries = Vec::new();
93-
for record in jsonl::records::<QwenLine>(&content, Some(QWEN_USAGE_MARKER)) {
93+
for record in jsonl::records::<QwenLine>(&content, Some(&prefilter)) {
9494
if let Some(entry) = parse_line(file, fallback, &record, tz, mode, pricing) {
9595
entries.push(entry);
9696
}

‎rust/crates/ccusage/src/fast.rs‎

Lines changed: 85 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,75 @@
1-
use memchr::memchr;
1+
use memchr::{memchr, memmem::Finder};
22
use smallvec::SmallVec;
33

44
pub(crate) type FxHashMap<K, V> = rustc_hash::FxHashMap<K, V>;
55
pub(crate) type FxHashSet<T> = rustc_hash::FxHashSet<T>;
66
pub(crate) type SmallIndexVec = SmallVec<[usize; 1]>;
77

8+
/// Whether a [`LinePrefilter`] requires every marker or just one of them.
9+
#[derive(Clone, Copy)]
10+
enum PrefilterMode {
11+
/// The line must contain every configured marker.
12+
All,
13+
/// The line must contain at least one configured marker.
14+
Any,
15+
}
16+
17+
/// Reusable byte-substring prefilter for newline-delimited JSON logs.
18+
///
19+
/// JSONL adapters skip lines that cannot contain a usage record before paying
20+
/// for a full `serde_json` parse. Building the [`Finder`] needles once and
21+
/// reusing them across every line keeps that skip check on the SIMD-accelerated
22+
/// `memmem` path instead of allocating a fresh searcher per `str::contains`
23+
/// call.
24+
pub(crate) struct LinePrefilter {
25+
finders: SmallVec<[Finder<'static>; 4]>,
26+
mode: PrefilterMode,
27+
}
28+
29+
impl LinePrefilter {
30+
/// Build a prefilter that only admits lines containing *all* `markers`.
31+
///
32+
/// # Example
33+
///
34+
/// ```ignore
35+
/// let prefilter = LinePrefilter::all(&[b"\"usage\"", b"\"message\""]);
36+
/// assert!(prefilter.matches(br#"{"message":{"usage":{}}}"#));
37+
/// assert!(!prefilter.matches(br#"{"message":{}}"#));
38+
/// ```
39+
pub(crate) fn all(markers: &[&[u8]]) -> Self {
40+
Self::new(markers, PrefilterMode::All)
41+
}
42+
43+
/// Build a prefilter that admits lines containing *any* of `markers`.
44+
pub(crate) fn any(markers: &[&[u8]]) -> Self {
45+
Self::new(markers, PrefilterMode::Any)
46+
}
47+
48+
fn new(markers: &[&[u8]], mode: PrefilterMode) -> Self {
49+
// `Finder::new` borrows the needle, so take an owned copy to outlive the
50+
// caller's marker slice and allow the prefilter to be stored freely.
51+
let finders = markers
52+
.iter()
53+
.map(|marker| Finder::new(marker).into_owned())
54+
.collect();
55+
Self { finders, mode }
56+
}
57+
58+
/// Return `true` when `line` passes the filter and is worth parsing.
59+
pub(crate) fn matches(&self, line: &[u8]) -> bool {
60+
match self.mode {
61+
PrefilterMode::All => self
62+
.finders
63+
.iter()
64+
.all(|finder| finder.find(line).is_some()),
65+
PrefilterMode::Any => self
66+
.finders
67+
.iter()
68+
.any(|finder| finder.find(line).is_some()),
69+
}
70+
}
71+
}
72+
873
pub(crate) struct ByteLines<'a> {
974
bytes: &'a [u8],
1075
}
@@ -47,7 +112,25 @@ pub(crate) fn suffix_string(value: &str, suffix: &str) -> String {
47112

48113
#[cfg(test)]
49114
mod tests {
50-
use super::{byte_lines, suffix_string};
115+
use super::{LinePrefilter, byte_lines, suffix_string};
116+
117+
#[test]
118+
fn line_prefilter_all_requires_every_marker() {
119+
let prefilter = LinePrefilter::all(&[b"\"usage\"", b"\"message\""]);
120+
121+
assert!(prefilter.matches(br#"{"message":{"usage":{"input":1}}}"#));
122+
assert!(!prefilter.matches(br#"{"message":{"role":"user"}}"#));
123+
assert!(!prefilter.matches(br#"{"usage":{"input":1}}"#));
124+
}
125+
126+
#[test]
127+
fn line_prefilter_any_requires_one_marker() {
128+
let prefilter = LinePrefilter::any(&[b"\"model_change\"", b"\"usage\""]);
129+
130+
assert!(prefilter.matches(br#"{"type":"model_change"}"#));
131+
assert!(prefilter.matches(br#"{"message":{"usage":{}}}"#));
132+
assert!(!prefilter.matches(br#"{"type":"message"}"#));
133+
}
51134

52135
#[test]
53136
fn byte_lines_returns_newline_delimited_slices() {

0 commit comments

Comments
 (0)