Skip to content

Commit 2360db6

Browse files
committed
perf(opencode): parallelize message reads and skip DB-covered files
The opencode loader read every storage/message/*.json file serially and then discarded any whose id the SQLite DB pass had already contributed. On large message trees this serial I/O dominates wall-clock, and on DB-heavy histories most of those reads are pure waste. Two I/O-focused changes, both preserving byte-identical output: - Parallelize the file pass by reusing the Claude loader's proven pattern (chunk_file_indexes_by_size + thread::scope + available_parallelism). Results are reassembled by original file index before the sequential dedup pass, so parallelism never changes which duplicate survives or the final ordering. Honors shared.single_thread. Per-file read errors now skip+debug_log instead of aborting the whole load, matching the Claude loader's swallow-and-continue behaviour. - Skip files the DB pass already covered. Message files are stored as storage/message/<sessionID>/<messageID>.json, so the file stem is the message id used for dedup. When the DB already contributed that id, the file would be discarded anyway, so it is filtered out before any read. Files whose stem is not a known id are still read and parsed normally. Benchmarks (50k synthetic message files, hyperfine --warmup 3 --runs 10): ~1.14x faster with no DB, ~5.36x faster (1.41s -> 263ms) when the DB covers 90% of messages. JSON and table output stay byte-identical across daily/monthly/weekly/session, --since/--until, and single-thread. Adds tests skips_message_files_already_covered_by_database and dedup_is_stable_across_thread_counts.
1 parent 6744254 commit 2360db6

1 file changed

Lines changed: 210 additions & 16 deletions

File tree

  • rust/crates/ccusage/src/adapter/opencode

‎rust/crates/ccusage/src/adapter/opencode/loader.rs‎

Lines changed: 210 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ use std::{
22
collections::HashSet,
33
fs,
44
path::{Path, PathBuf},
5+
thread,
56
};
67

78
use jiff::tz::TimeZone as JiffTimeZone;
@@ -11,7 +12,7 @@ use super::{
1112
paths::paths,
1213
};
1314
use crate::{
14-
LoadedEntry, PricingMap, Result,
15+
LoadedEntry, PricingMap, Result, chunk_file_indexes_by_size,
1516
cli::{CostMode, SharedArgs},
1617
collect_files_with_extension, debug_log, parse_tz,
1718
};
@@ -73,20 +74,99 @@ pub(crate) fn load_entries_from_directory(
7374
let messages_dir = opencode_dir.join("storage").join("message");
7475
let mut files = Vec::new();
7576
collect_files_with_extension(&messages_dir, "json", &mut files);
76-
for file in files {
77-
if let Some(entry) = read_message_file(&file, tz.as_ref(), shared.mode, pricing.as_ref())? {
78-
if let Some(id) = entry_id(&entry)
79-
&& !seen.insert(id.to_string())
80-
{
81-
continue;
82-
}
83-
entries.push(entry);
77+
78+
// Skip files the DB pass already covered. Message files are stored as
79+
// `storage/message/<sessionID>/<messageID>.json`, so the file stem is the
80+
// message id used for dedup. When the DB already contributed that id, the
81+
// file would be discarded by the id dedup below anyway — drop it here so we
82+
// never pay the read. Files whose stem is not a known id (or that have no
83+
// usable stem) are kept and parsed normally.
84+
if !seen.is_empty() {
85+
files.retain(|file| {
86+
file.file_stem()
87+
.and_then(|stem| stem.to_str())
88+
.is_none_or(|stem| !seen.contains(stem))
89+
});
90+
}
91+
92+
let loaded = read_message_files(&files, tz.as_ref(), shared.mode, pricing.as_ref(), shared);
93+
for entry in loaded.into_iter().flatten() {
94+
if let Some(id) = entry_id(&entry)
95+
&& !seen.insert(id.to_string())
96+
{
97+
continue;
8498
}
99+
entries.push(entry);
85100
}
86101
entries.sort_by_key(|entry| entry.timestamp);
87102
Ok(entries)
88103
}
89104

105+
/// Reads and parses the collected message files, returning one slot per input
106+
/// file in the original order so the caller's sequential dedup pass observes a
107+
/// deterministic ordering regardless of how many worker threads ran.
108+
///
109+
/// Honors [`SharedArgs::single_thread`] and falls back to a sequential read
110+
/// when only a single worker would be used. Per-file read failures are logged
111+
/// and skipped (mirroring the Claude loader's swallow-and-continue behavior)
112+
/// rather than aborting the whole load.
113+
fn read_message_files(
114+
files: &[PathBuf],
115+
tz: Option<&JiffTimeZone>,
116+
mode: CostMode,
117+
pricing: Option<&PricingMap>,
118+
shared: &SharedArgs,
119+
) -> Vec<Option<LoadedEntry>> {
120+
let worker_count = if shared.single_thread {
121+
1
122+
} else {
123+
thread::available_parallelism()
124+
.map(usize::from)
125+
.unwrap_or(1)
126+
.min(files.len())
127+
};
128+
if worker_count <= 1 {
129+
return files
130+
.iter()
131+
.map(|file| read_message_file(file, tz, mode, pricing, shared))
132+
.collect();
133+
}
134+
135+
// Balance the files across workers by byte size so a few large files do not
136+
// serialize a single worker, then reassemble results by their original
137+
// index to keep the output order identical to the sequential read.
138+
let chunks = chunk_file_indexes_by_size(files, worker_count);
139+
thread::scope(|scope| {
140+
let mut handles = Vec::with_capacity(chunks.len());
141+
for chunk in chunks {
142+
let tz = tz.cloned();
143+
handles.push(scope.spawn(move || {
144+
chunk
145+
.into_iter()
146+
.map(|index| {
147+
(
148+
index,
149+
read_message_file(&files[index], tz.as_ref(), mode, pricing, shared),
150+
)
151+
})
152+
.collect::<Vec<_>>()
153+
}));
154+
}
155+
let mut loaded = Vec::with_capacity(files.len());
156+
loaded.resize_with(files.len(), || None);
157+
for (index, entry) in handles
158+
.into_iter()
159+
.flat_map(|handle| handle.join().expect("opencode message worker panicked"))
160+
{
161+
loaded[index] = Some(entry);
162+
}
163+
loaded
164+
.into_iter()
165+
.map(|entry| entry.expect("opencode message worker returned every file"))
166+
.collect()
167+
})
168+
}
169+
90170
fn db_path(opencode_dir: &Path) -> Option<PathBuf> {
91171
let default_path = opencode_dir.join("opencode.db");
92172
if default_path.is_file() {
@@ -179,14 +259,23 @@ fn read_message_file(
179259
tz: Option<&JiffTimeZone>,
180260
mode: CostMode,
181261
pricing: Option<&PricingMap>,
182-
) -> Result<Option<LoadedEntry>> {
183-
let content = fs::read(path)?;
184-
let Ok(value) = serde_json::from_slice::<OpenCodeMessage>(&content) else {
185-
return Ok(None);
262+
shared: &SharedArgs,
263+
) -> Option<LoadedEntry> {
264+
let content = match fs::read(path) {
265+
Ok(content) => content,
266+
Err(error) => {
267+
debug_log(
268+
shared,
269+
format!(
270+
"Failed to read OpenCode message file {}: {error}",
271+
path.display()
272+
),
273+
);
274+
return None;
275+
}
186276
};
187-
Ok(message_value_to_entry(
188-
&value, None, None, tz, mode, pricing,
189-
))
277+
let value = serde_json::from_slice::<OpenCodeMessage>(&content).ok()?;
278+
message_value_to_entry(&value, None, None, tz, mode, pricing)
190279
}
191280

192281
fn entry_id(entry: &LoadedEntry) -> Option<&str> {
@@ -315,4 +404,109 @@ mod tests {
315404
assert_eq!(entries[0].data.message.usage.input_tokens, 120);
316405
assert_eq!(entries[0].cost, 0.03);
317406
}
407+
408+
#[test]
409+
fn skips_message_files_already_covered_by_database() {
410+
// Real OpenCode message files live at
411+
// `storage/message/<sessionID>/<messageID>.json`, so the file stem is
412+
// the message id. The DB pass contributes `msg-db`, so the matching
413+
// file must be dropped (DB wins) while the file that the DB does not
414+
// cover is still loaded.
415+
let fixture = fs_fixture!({
416+
"storage/message/ses_a/msg-db.json": r#"{"id":"msg-db","sessionID":"json-session","providerID":"anthropic","modelID":"claude-sonnet-4-20250514","time":{"created":1767312000000},"tokens":{"input":999,"output":999},"cost":0.99}"#,
417+
"storage/message/ses_a/msg-file.json": r#"{"id":"msg-file","sessionID":"file-session","providerID":"anthropic","modelID":"claude-sonnet-4-20250514","time":{"created":1767312000001},"tokens":{"input":50,"output":25},"cost":0.01}"#,
418+
});
419+
create_db_message(
420+
&fixture.path("opencode.db"),
421+
"msg-db",
422+
"db-session",
423+
r#"{"providerID":"anthropic","modelID":"claude-sonnet-4-20250514","time":{"created":1767312000000},"tokens":{"input":120,"output":60},"cost":0.03}"#,
424+
);
425+
426+
let shared = SharedArgs {
427+
mode: CostMode::Display,
428+
..SharedArgs::default()
429+
};
430+
let entries = load_entries_from_directory(fixture.root(), &shared).unwrap();
431+
432+
assert_eq!(entries.len(), 2);
433+
// The DB-covered id keeps the DB row, not the file's inflated tokens.
434+
let db_entry = entries
435+
.iter()
436+
.find(|entry| entry.data.message.id.as_deref() == Some("msg-db"))
437+
.expect("db-covered message present");
438+
assert_eq!(db_entry.session_id.as_ref(), "db-session");
439+
assert_eq!(db_entry.data.message.usage.input_tokens, 120);
440+
// The file the DB does not cover is still read and parsed.
441+
let file_entry = entries
442+
.iter()
443+
.find(|entry| entry.data.message.id.as_deref() == Some("msg-file"))
444+
.expect("db-uncovered message present");
445+
assert_eq!(file_entry.session_id.as_ref(), "file-session");
446+
assert_eq!(file_entry.data.message.usage.input_tokens, 50);
447+
}
448+
449+
#[test]
450+
fn dedup_is_stable_across_thread_counts() {
451+
// Build a directory with many files spread over several sessions, some
452+
// sharing ids with each other and with the DB, so the file pass has to
453+
// dedup. Parallel reads must not change which duplicate survives or the
454+
// final ordering compared to the single-threaded read.
455+
let fixture = ccusage_test_support::Fixture::new();
456+
for session in 0..4 {
457+
for message in 0..15 {
458+
let id = format!("msg-{session}-{message}");
459+
let created = 1_767_312_000_000_i64 + i64::from(session * 100 + message);
460+
let path = format!("storage/message/ses_{session}/{id}.json");
461+
let data = format!(
462+
r#"{{"id":"{id}","sessionID":"ses_{session}","providerID":"anthropic","modelID":"claude-sonnet-4-20250514","time":{{"created":{created}}},"tokens":{{"input":{input},"output":10}}}}"#,
463+
input = 100 + message,
464+
);
465+
let _ = fixture.write_file(path, data);
466+
}
467+
}
468+
// A duplicate file (same id, later timestamp) to force the file-vs-file
469+
// dedup path under both thread counts.
470+
let _ = fixture.write_file(
471+
"storage/message/ses_dup/msg-0-0.json",
472+
r#"{"id":"msg-0-0","sessionID":"ses_dup","providerID":"anthropic","modelID":"claude-sonnet-4-20250514","time":{"created":1767312999999},"tokens":{"input":7777,"output":10}}"#,
473+
);
474+
475+
create_db_message(
476+
&fixture.path("opencode.db"),
477+
"msg-1-1",
478+
"db-session",
479+
r#"{"providerID":"anthropic","modelID":"claude-sonnet-4-20250514","time":{"created":1767312000000},"tokens":{"input":120,"output":60}}"#,
480+
);
481+
482+
let single = SharedArgs {
483+
mode: CostMode::Display,
484+
single_thread: true,
485+
..SharedArgs::default()
486+
};
487+
let multi = SharedArgs {
488+
mode: CostMode::Display,
489+
single_thread: false,
490+
..SharedArgs::default()
491+
};
492+
493+
let single_entries = load_entries_from_directory(fixture.root(), &single).unwrap();
494+
let multi_entries = load_entries_from_directory(fixture.root(), &multi).unwrap();
495+
496+
let project = |entries: &[crate::LoadedEntry]| {
497+
entries
498+
.iter()
499+
.map(|entry| {
500+
(
501+
entry.timestamp.as_millis(),
502+
entry.data.message.id.clone(),
503+
entry.session_id.to_string(),
504+
entry.data.message.usage.input_tokens,
505+
)
506+
})
507+
.collect::<Vec<_>>()
508+
};
509+
510+
assert_eq!(project(&single_entries), project(&multi_entries));
511+
}
318512
}

0 commit comments

Comments
 (0)