Skip to content

Commit 8569c13

Browse files
committed
perf(adapter): parallelize copilot OTEL and database loaders
Adopt read_files_parallel in the remaining loaders that iterate over multiple sources: copilot (OTEL files) and the SQLite-backed goose, hermes, and kilo loaders, which can each see more than one database when multiple data dirs / homes / channels are configured. Each database is opened on its own read-only connection inside the worker, and the sequential dedup pass runs over the parallel results in their original path order, so the surviving record per key is identical to the single-threaded read. Within a single database the row scan stays sequential (inherent to one SQLite connection); the win is overlapping multiple sources and keeping all loaders on one shared code path. Output verified byte-identical to the previous release across report modes on synthesized multi-database fixtures with overlapping ids, with single-thread and multi-thread runs matching.
1 parent 1dce684 commit 8569c13

4 files changed

Lines changed: 63 additions & 16 deletions

File tree

‎rust/crates/ccusage/src/adapter/copilot/loader.rs‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@ use super::{
88
};
99
use crate::{
1010
LoadedEntry, Result, TokenUsageRaw, UsageEntry, UsageMessage, calculate_cost_for_usage,
11-
cli::CostMode, format_date_tz, missing_pricing_model_for_usage, parse_tz,
11+
cli::CostMode, debug_log, format_date_tz, missing_pricing_model_for_usage, parse_tz,
12+
read_files_parallel,
1213
};
1314

1415
pub(crate) fn load_entries(
@@ -27,9 +28,24 @@ fn load_entries_inner(
2728
pricing: &crate::PricingMap,
2829
) -> Result<Vec<LoadedEntry>> {
2930
let tz = parse_tz(shared.timezone.as_deref());
31+
let files = paths()?;
32+
// Read OTEL files in parallel; entries keep their original file order before
33+
// the stable sort, so output is identical to the sequential read.
34+
let loaded = read_files_parallel(&files, shared.single_thread, |path| {
35+
read_otel_file(path, tz.as_ref(), shared.mode, pricing).unwrap_or_else(|error| {
36+
debug_log(
37+
shared,
38+
format!(
39+
"Failed to read Copilot OTEL file {}: {error}",
40+
path.display()
41+
),
42+
);
43+
Vec::new()
44+
})
45+
});
3046
let mut entries = Vec::new();
31-
for path in paths()? {
32-
entries.extend(read_otel_file(&path, tz.as_ref(), shared.mode, pricing)?);
47+
for file_entries in loaded {
48+
entries.extend(file_entries);
3349
}
3450
entries.sort_by_key(|entry| entry.timestamp);
3551
Ok(entries)

‎rust/crates/ccusage/src/adapter/goose/loader.rs‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@ use std::{collections::HashSet, path::Path};
22

33
use jiff::tz::TimeZone as JiffTimeZone;
44

5-
use crate::{LoadedEntry, PricingMap, Result, cli::SharedArgs, debug_log, parse_tz};
5+
use crate::{
6+
LoadedEntry, PricingMap, Result, cli::SharedArgs, debug_log, parse_tz, read_files_parallel,
7+
};
68

79
use super::{parser::row_to_entry, paths::goose_db_paths};
810

@@ -31,10 +33,26 @@ pub(crate) fn load_entries(shared: &SharedArgs, pricing: &PricingMap) -> Result<
3133

3234
fn load_entries_inner(shared: &SharedArgs, pricing: &PricingMap) -> Result<Vec<LoadedEntry>> {
3335
let tz = parse_tz(shared.timezone.as_deref());
36+
let db_paths = goose_db_paths()?;
37+
// Load each database in parallel (a fresh read-only connection per DB), then
38+
// run the sequential per-db dedup over the original path order so the
39+
// surviving session per key matches the single-threaded read.
40+
let loaded = read_files_parallel(&db_paths, shared.single_thread, |db_path| {
41+
load_entries_from_db(db_path, tz.as_ref(), pricing, shared).unwrap_or_else(|error| {
42+
debug_log(
43+
shared,
44+
format!(
45+
"Failed to load Goose database {}: {error}",
46+
db_path.display()
47+
),
48+
);
49+
Vec::new()
50+
})
51+
});
3452
let mut entries = Vec::new();
3553
let mut seen = HashSet::new();
36-
for db_path in goose_db_paths()? {
37-
for entry in load_entries_from_db(&db_path, tz.as_ref(), pricing, shared)? {
54+
for (db_path, db_entries) in db_paths.iter().zip(loaded) {
55+
for entry in db_entries {
3856
let key = format!("{}:{}", db_path.display(), entry.session_id);
3957
if seen.insert(key) {
4058
entries.push(entry);

‎rust/crates/ccusage/src/adapter/hermes/loader.rs‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
use std::{collections::HashSet, path::Path};
22

3-
use crate::{LoadedEntry, PricingMap, Result, cli::SharedArgs};
3+
use crate::{LoadedEntry, PricingMap, Result, cli::SharedArgs, read_files_parallel};
44

55
use super::{
66
parser::{HermesEntry, read_session_row, to_loaded_entry},
@@ -15,10 +15,17 @@ pub(crate) fn load_entries(shared: &SharedArgs, pricing: &PricingMap) -> Result<
1515

1616
fn load_entries_inner(shared: &SharedArgs, pricing: &PricingMap) -> Result<Vec<LoadedEntry>> {
1717
let tz = crate::parse_tz(shared.timezone.as_deref());
18+
let db_paths = hermes_state_db_paths()?;
19+
// Load each state database in parallel (a fresh read-only connection per DB),
20+
// then run the sequential session dedup over the original path order so the
21+
// surviving session matches the single-threaded read.
22+
let loaded = read_files_parallel(&db_paths, shared.single_thread, |db_path| {
23+
load_state_db_entries(db_path, shared)
24+
});
1825
let mut entries = Vec::new();
1926
let mut seen_sessions = HashSet::new();
20-
for db_path in hermes_state_db_paths()? {
21-
for entry in load_state_db_entries(&db_path, shared) {
27+
for db_entries in loaded {
28+
for entry in db_entries {
2229
if !seen_sessions.insert(entry.session_id.clone()) {
2330
continue;
2431
}

‎rust/crates/ccusage/src/adapter/kilo/loader.rs‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
1-
use std::{collections::HashSet, path::Path};
1+
use std::{collections::HashSet, path::Path, path::PathBuf};
22

33
use jiff::tz::TimeZone as JiffTimeZone;
44

5-
use crate::{LoadedEntry, PricingMap, Result, cli::SharedArgs, debug_log, parse_tz};
5+
use crate::{
6+
LoadedEntry, PricingMap, Result, cli::SharedArgs, debug_log, parse_tz, read_files_parallel,
7+
};
68

79
use super::{
810
parser::{KiloMessage, message_value_to_entry},
@@ -17,13 +19,17 @@ pub(crate) fn load_entries(shared: &SharedArgs, pricing: &PricingMap) -> Result<
1719

1820
fn load_entries_inner(shared: &SharedArgs, pricing: &PricingMap) -> Result<Vec<LoadedEntry>> {
1921
let tz = parse_tz(shared.timezone.as_deref());
22+
let db_paths: Vec<PathBuf> = paths()?.iter().filter_map(|path| db_path(path)).collect();
23+
// Load each database in parallel (a fresh read-only connection per DB), then
24+
// run the sequential id dedup over the original path order so the surviving
25+
// record per id matches the single-threaded read.
26+
let loaded = read_files_parallel(&db_paths, shared.single_thread, |db_path| {
27+
load_entries_from_database(db_path, tz.as_ref(), shared, pricing)
28+
});
2029
let mut entries = Vec::new();
2130
let mut seen = HashSet::new();
22-
for path in paths()? {
23-
let Some(db_path) = db_path(&path) else {
24-
continue;
25-
};
26-
for entry in load_entries_from_database(&db_path, tz.as_ref(), shared, pricing) {
31+
for db_entries in loaded {
32+
for entry in db_entries {
2733
if let Some(id) = entry.data.message.id.as_deref()
2834
&& !seen.insert(id.to_string())
2935
{

0 commit comments

Comments
 (0)