@@ -2,6 +2,7 @@ use std::{
22 collections:: HashSet ,
33 fs,
44 path:: { Path , PathBuf } ,
5+ thread,
56} ;
67
78use jiff:: tz:: TimeZone as JiffTimeZone ;
@@ -11,7 +12,7 @@ use super::{
1112 paths:: paths,
1213} ;
1314use crate :: {
14- LoadedEntry , PricingMap , Result ,
15+ LoadedEntry , PricingMap , Result , chunk_file_indexes_by_size ,
1516 cli:: { CostMode , SharedArgs } ,
1617 collect_files_with_extension, debug_log, parse_tz,
1718} ;
@@ -73,20 +74,99 @@ pub(crate) fn load_entries_from_directory(
7374 let messages_dir = opencode_dir. join ( "storage" ) . join ( "message" ) ;
7475 let mut files = Vec :: new ( ) ;
7576 collect_files_with_extension ( & messages_dir, "json" , & mut files) ;
76- for file in files {
77- if let Some ( entry) = read_message_file ( & file, tz. as_ref ( ) , shared. mode , pricing. as_ref ( ) ) ? {
78- if let Some ( id) = entry_id ( & entry)
79- && !seen. insert ( id. to_string ( ) )
80- {
81- continue ;
82- }
83- entries. push ( entry) ;
77+
78+ // Skip files the DB pass already covered. Message files are stored as
79+ // `storage/message/<sessionID>/<messageID>.json`, so the file stem is the
80+ // message id used for dedup. When the DB already contributed that id, the
81+ // file would be discarded by the id dedup below anyway — drop it here so we
82+ // never pay the read. Files whose stem is not a known id (or that have no
83+ // usable stem) are kept and parsed normally.
84+ if !seen. is_empty ( ) {
85+ files. retain ( |file| {
86+ file. file_stem ( )
87+ . and_then ( |stem| stem. to_str ( ) )
88+ . is_none_or ( |stem| !seen. contains ( stem) )
89+ } ) ;
90+ }
91+
92+ let loaded = read_message_files ( & files, tz. as_ref ( ) , shared. mode , pricing. as_ref ( ) , shared) ;
93+ for entry in loaded. into_iter ( ) . flatten ( ) {
94+ if let Some ( id) = entry_id ( & entry)
95+ && !seen. insert ( id. to_string ( ) )
96+ {
97+ continue ;
8498 }
99+ entries. push ( entry) ;
85100 }
86101 entries. sort_by_key ( |entry| entry. timestamp ) ;
87102 Ok ( entries)
88103}
89104
105+ /// Reads and parses the collected message files, returning one slot per input
106+ /// file in the original order so the caller's sequential dedup pass observes a
107+ /// deterministic ordering regardless of how many worker threads ran.
108+ ///
109+ /// Honors [`SharedArgs::single_thread`] and falls back to a sequential read
110+ /// when only a single worker would be used. Per-file read failures are logged
111+ /// and skipped (mirroring the Claude loader's swallow-and-continue behavior)
112+ /// rather than aborting the whole load.
113+ fn read_message_files (
114+ files : & [ PathBuf ] ,
115+ tz : Option < & JiffTimeZone > ,
116+ mode : CostMode ,
117+ pricing : Option < & PricingMap > ,
118+ shared : & SharedArgs ,
119+ ) -> Vec < Option < LoadedEntry > > {
120+ let worker_count = if shared. single_thread {
121+ 1
122+ } else {
123+ thread:: available_parallelism ( )
124+ . map ( usize:: from)
125+ . unwrap_or ( 1 )
126+ . min ( files. len ( ) )
127+ } ;
128+ if worker_count <= 1 {
129+ return files
130+ . iter ( )
131+ . map ( |file| read_message_file ( file, tz, mode, pricing, shared) )
132+ . collect ( ) ;
133+ }
134+
135+ // Balance the files across workers by byte size so a few large files do not
136+ // serialize a single worker, then reassemble results by their original
137+ // index to keep the output order identical to the sequential read.
138+ let chunks = chunk_file_indexes_by_size ( files, worker_count) ;
139+ thread:: scope ( |scope| {
140+ let mut handles = Vec :: with_capacity ( chunks. len ( ) ) ;
141+ for chunk in chunks {
142+ let tz = tz. cloned ( ) ;
143+ handles. push ( scope. spawn ( move || {
144+ chunk
145+ . into_iter ( )
146+ . map ( |index| {
147+ (
148+ index,
149+ read_message_file ( & files[ index] , tz. as_ref ( ) , mode, pricing, shared) ,
150+ )
151+ } )
152+ . collect :: < Vec < _ > > ( )
153+ } ) ) ;
154+ }
155+ let mut loaded = Vec :: with_capacity ( files. len ( ) ) ;
156+ loaded. resize_with ( files. len ( ) , || None ) ;
157+ for ( index, entry) in handles
158+ . into_iter ( )
159+ . flat_map ( |handle| handle. join ( ) . expect ( "opencode message worker panicked" ) )
160+ {
161+ loaded[ index] = Some ( entry) ;
162+ }
163+ loaded
164+ . into_iter ( )
165+ . map ( |entry| entry. expect ( "opencode message worker returned every file" ) )
166+ . collect ( )
167+ } )
168+ }
169+
90170fn db_path ( opencode_dir : & Path ) -> Option < PathBuf > {
91171 let default_path = opencode_dir. join ( "opencode.db" ) ;
92172 if default_path. is_file ( ) {
@@ -179,14 +259,23 @@ fn read_message_file(
179259 tz : Option < & JiffTimeZone > ,
180260 mode : CostMode ,
181261 pricing : Option < & PricingMap > ,
182- ) -> Result < Option < LoadedEntry > > {
183- let content = fs:: read ( path) ?;
184- let Ok ( value) = serde_json:: from_slice :: < OpenCodeMessage > ( & content) else {
185- return Ok ( None ) ;
262+ shared : & SharedArgs ,
263+ ) -> Option < LoadedEntry > {
264+ let content = match fs:: read ( path) {
265+ Ok ( content) => content,
266+ Err ( error) => {
267+ debug_log (
268+ shared,
269+ format ! (
270+ "Failed to read OpenCode message file {}: {error}" ,
271+ path. display( )
272+ ) ,
273+ ) ;
274+ return None ;
275+ }
186276 } ;
187- Ok ( message_value_to_entry (
188- & value, None , None , tz, mode, pricing,
189- ) )
277+ let value = serde_json:: from_slice :: < OpenCodeMessage > ( & content) . ok ( ) ?;
278+ message_value_to_entry ( & value, None , None , tz, mode, pricing)
190279}
191280
192281fn entry_id ( entry : & LoadedEntry ) -> Option < & str > {
@@ -315,4 +404,109 @@ mod tests {
315404 assert_eq ! ( entries[ 0 ] . data. message. usage. input_tokens, 120 ) ;
316405 assert_eq ! ( entries[ 0 ] . cost, 0.03 ) ;
317406 }
407+
408+ #[ test]
409+ fn skips_message_files_already_covered_by_database ( ) {
410+ // Real OpenCode message files live at
411+ // `storage/message/<sessionID>/<messageID>.json`, so the file stem is
412+ // the message id. The DB pass contributes `msg-db`, so the matching
413+ // file must be dropped (DB wins) while the file that the DB does not
414+ // cover is still loaded.
415+ let fixture = fs_fixture ! ( {
416+ "storage/message/ses_a/msg-db.json" : r#"{"id":"msg-db","sessionID":"json-session","providerID":"anthropic","modelID":"claude-sonnet-4-20250514","time":{"created":1767312000000},"tokens":{"input":999,"output":999},"cost":0.99}"# ,
417+ "storage/message/ses_a/msg-file.json" : r#"{"id":"msg-file","sessionID":"file-session","providerID":"anthropic","modelID":"claude-sonnet-4-20250514","time":{"created":1767312000001},"tokens":{"input":50,"output":25},"cost":0.01}"# ,
418+ } ) ;
419+ create_db_message (
420+ & fixture. path ( "opencode.db" ) ,
421+ "msg-db" ,
422+ "db-session" ,
423+ r#"{"providerID":"anthropic","modelID":"claude-sonnet-4-20250514","time":{"created":1767312000000},"tokens":{"input":120,"output":60},"cost":0.03}"# ,
424+ ) ;
425+
426+ let shared = SharedArgs {
427+ mode : CostMode :: Display ,
428+ ..SharedArgs :: default ( )
429+ } ;
430+ let entries = load_entries_from_directory ( fixture. root ( ) , & shared) . unwrap ( ) ;
431+
432+ assert_eq ! ( entries. len( ) , 2 ) ;
433+ // The DB-covered id keeps the DB row, not the file's inflated tokens.
434+ let db_entry = entries
435+ . iter ( )
436+ . find ( |entry| entry. data . message . id . as_deref ( ) == Some ( "msg-db" ) )
437+ . expect ( "db-covered message present" ) ;
438+ assert_eq ! ( db_entry. session_id. as_ref( ) , "db-session" ) ;
439+ assert_eq ! ( db_entry. data. message. usage. input_tokens, 120 ) ;
440+ // The file the DB does not cover is still read and parsed.
441+ let file_entry = entries
442+ . iter ( )
443+ . find ( |entry| entry. data . message . id . as_deref ( ) == Some ( "msg-file" ) )
444+ . expect ( "db-uncovered message present" ) ;
445+ assert_eq ! ( file_entry. session_id. as_ref( ) , "file-session" ) ;
446+ assert_eq ! ( file_entry. data. message. usage. input_tokens, 50 ) ;
447+ }
448+
449+ #[ test]
450+ fn dedup_is_stable_across_thread_counts ( ) {
451+ // Build a directory with many files spread over several sessions, some
452+ // sharing ids with each other and with the DB, so the file pass has to
453+ // dedup. Parallel reads must not change which duplicate survives or the
454+ // final ordering compared to the single-threaded read.
455+ let fixture = ccusage_test_support:: Fixture :: new ( ) ;
456+ for session in 0 ..4 {
457+ for message in 0 ..15 {
458+ let id = format ! ( "msg-{session}-{message}" ) ;
459+ let created = 1_767_312_000_000_i64 + i64:: from ( session * 100 + message) ;
460+ let path = format ! ( "storage/message/ses_{session}/{id}.json" ) ;
461+ let data = format ! (
462+ r#"{{"id":"{id}","sessionID":"ses_{session}","providerID":"anthropic","modelID":"claude-sonnet-4-20250514","time":{{"created":{created}}},"tokens":{{"input":{input},"output":10}}}}"# ,
463+ input = 100 + message,
464+ ) ;
465+ let _ = fixture. write_file ( path, data) ;
466+ }
467+ }
468+ // A duplicate file (same id, later timestamp) to force the file-vs-file
469+ // dedup path under both thread counts.
470+ let _ = fixture. write_file (
471+ "storage/message/ses_dup/msg-0-0.json" ,
472+ r#"{"id":"msg-0-0","sessionID":"ses_dup","providerID":"anthropic","modelID":"claude-sonnet-4-20250514","time":{"created":1767312999999},"tokens":{"input":7777,"output":10}}"# ,
473+ ) ;
474+
475+ create_db_message (
476+ & fixture. path ( "opencode.db" ) ,
477+ "msg-1-1" ,
478+ "db-session" ,
479+ r#"{"providerID":"anthropic","modelID":"claude-sonnet-4-20250514","time":{"created":1767312000000},"tokens":{"input":120,"output":60}}"# ,
480+ ) ;
481+
482+ let single = SharedArgs {
483+ mode : CostMode :: Display ,
484+ single_thread : true ,
485+ ..SharedArgs :: default ( )
486+ } ;
487+ let multi = SharedArgs {
488+ mode : CostMode :: Display ,
489+ single_thread : false ,
490+ ..SharedArgs :: default ( )
491+ } ;
492+
493+ let single_entries = load_entries_from_directory ( fixture. root ( ) , & single) . unwrap ( ) ;
494+ let multi_entries = load_entries_from_directory ( fixture. root ( ) , & multi) . unwrap ( ) ;
495+
496+ let project = |entries : & [ crate :: LoadedEntry ] | {
497+ entries
498+ . iter ( )
499+ . map ( |entry| {
500+ (
501+ entry. timestamp . as_millis ( ) ,
502+ entry. data . message . id . clone ( ) ,
503+ entry. session_id . to_string ( ) ,
504+ entry. data . message . usage . input_tokens ,
505+ )
506+ } )
507+ . collect :: < Vec < _ > > ( )
508+ } ;
509+
510+ assert_eq ! ( project( & single_entries) , project( & multi_entries) ) ;
511+ }
318512}
0 commit comments