Skip to content

Commit 409b4a5

Browse files
fix(codex): dedupe replayed usage (#1435)
Codex fork and subagent logs replay the parent's `token_count` history with rewritten timestamps. The same-second heuristic missed replay bursts spanning multiple seconds and nested forks, so the copied history was aggregated again. Match forked session event prefixes against the immutable parent stream before aggregation, bounded at the fork timestamp so usage the parent recorded after the fork cannot mask the child's own events. When the parent log is unavailable, or when the prefix does not line up with the start of the parent stream, fall back to skipping the rewritten-second burst. Also ignore `last_token_usage` snapshots that Codex re-emits without advancing `total_token_usage`. Fixes #1434. Addresses #950, #988, #1288, #1337, and #1349. Co-authored-by: Miles Cranmer <[email protected]> Co-authored-by: ryoppippi <[email protected]>
1 parent 739e88f commit 409b4a5

5 files changed

Lines changed: 874 additions & 175 deletions

File tree

‎rust/crates/ccusage/src/adapter/codex/aggregate.rs‎

Lines changed: 140 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ use crate::{
1616
format_date_tz, parse_ts_timestamp, parse_tz, wants_json, week_start,
1717
};
1818

19-
use super::{parser, paths};
19+
use super::{parser, paths, replay::CodexReplayPlan};
2020

2121
type CodexEventKey = (
2222
u64,
@@ -37,6 +37,15 @@ struct CodexAggregation {
3737
seen: FxHashSet<CodexEventKey>,
3838
}
3939

40+
/// Read-only inputs every file of one aggregation run shares.
41+
struct CodexAggregateRun<'a> {
42+
sessions_dir: &'a Path,
43+
files: &'a [PathBuf],
44+
shared: &'a SharedArgs,
45+
kind: AgentReportKind,
46+
replay_plan: &'a CodexReplayPlan,
47+
}
48+
4049
pub(crate) fn load_groups(
4150
shared: &SharedArgs,
4251
kind: AgentReportKind,
@@ -53,12 +62,28 @@ fn load_groups_from_sources(
5362
shared: &SharedArgs,
5463
kind: AgentReportKind,
5564
) -> Result<BTreeMap<String, CodexGroup>> {
65+
let file_groups = paths::collect_deduped_codex_usage_files(sources);
66+
let replay_plan = CodexReplayPlan::new(
67+
file_groups
68+
.iter()
69+
.map(|group| (group.dir.as_path(), group.files.as_slice())),
70+
shared.single_thread,
71+
);
5672
let mut groups = BTreeMap::new();
5773
let seen = create_dedupe_shards();
58-
for group in paths::collect_deduped_codex_usage_files(sources) {
74+
for group in &file_groups {
5975
merge_groups(
6076
&mut groups,
61-
aggregate_files_with_dedupe(&group.dir, &group.files, shared, kind, &seen)?,
77+
aggregate_files_with_dedupe(
78+
&CodexAggregateRun {
79+
sessions_dir: &group.dir,
80+
files: &group.files,
81+
shared,
82+
kind,
83+
replay_plan: &replay_plan,
84+
},
85+
&seen,
86+
)?,
6287
);
6388
}
6489
Ok(groups)
@@ -70,82 +95,67 @@ pub(super) fn load_groups_from_directory(
7095
kind: AgentReportKind,
7196
) -> Result<BTreeMap<String, CodexGroup>> {
7297
let files = paths::collect_codex_usage_files(sessions_dir);
98+
let replay_plan =
99+
CodexReplayPlan::new([(sessions_dir, files.as_slice())], shared.single_thread);
100+
let run = CodexAggregateRun {
101+
sessions_dir,
102+
files: &files,
103+
shared,
104+
kind,
105+
replay_plan: &replay_plan,
106+
};
73107
if shared.single_thread {
74-
return aggregate_files_local(sessions_dir, &files, shared, kind);
108+
return aggregate_files_local(&run);
75109
}
76110
let seen = create_dedupe_shards();
77-
aggregate_files_parallel(sessions_dir, &files, shared, kind, &seen)
111+
aggregate_files_parallel(&run, &seen)
78112
}
79113

80114
fn aggregate_files_with_dedupe(
81-
sessions_dir: &Path,
82-
files: &[PathBuf],
83-
shared: &SharedArgs,
84-
kind: AgentReportKind,
115+
run: &CodexAggregateRun<'_>,
85116
seen: &CodexDedupeShards,
86117
) -> Result<BTreeMap<String, CodexGroup>> {
87-
if shared.single_thread {
88-
return aggregate_files(sessions_dir, files, shared, kind, seen);
118+
if run.shared.single_thread {
119+
return aggregate_files(run, seen);
89120
}
90-
aggregate_files_parallel(sessions_dir, files, shared, kind, seen)
121+
aggregate_files_parallel(run, seen)
91122
}
92123

93124
fn aggregate_files(
94-
sessions_dir: &Path,
95-
files: &[PathBuf],
96-
shared: &SharedArgs,
97-
kind: AgentReportKind,
125+
run: &CodexAggregateRun<'_>,
98126
seen: &CodexDedupeShards,
99127
) -> Result<BTreeMap<String, CodexGroup>> {
100128
let mut groups = BTreeMap::new();
101-
let timezone = parse_tz(shared.timezone.as_deref()).or_else(|| Some(JiffTimeZone::system()));
102-
for file in files {
103-
aggregate_file(
104-
sessions_dir,
105-
file,
106-
kind,
107-
timezone.as_ref(),
108-
shared,
109-
seen,
110-
&mut groups,
111-
)?;
129+
let timezone =
130+
parse_tz(run.shared.timezone.as_deref()).or_else(|| Some(JiffTimeZone::system()));
131+
for file in run.files {
132+
aggregate_file(run, file, timezone.as_ref(), seen, &mut groups)?;
112133
}
113134
Ok(groups)
114135
}
115136

116137
fn aggregate_files_parallel(
117-
sessions_dir: &Path,
118-
files: &[PathBuf],
119-
shared: &SharedArgs,
120-
kind: AgentReportKind,
138+
run: &CodexAggregateRun<'_>,
121139
seen: &CodexDedupeShards,
122140
) -> Result<BTreeMap<String, CodexGroup>> {
123141
let worker_count = thread::available_parallelism()
124142
.map(usize::from)
125143
.unwrap_or(1)
126-
.min(files.len());
144+
.min(run.files.len());
127145
if worker_count <= 1 {
128-
return aggregate_files(sessions_dir, files, shared, kind, seen);
146+
return aggregate_files(run, seen);
129147
}
130148

131-
let chunks = crate::chunk_file_indexes_by_size(files, worker_count);
149+
let chunks = crate::chunk_file_indexes_by_size(run.files, worker_count);
132150
thread::scope(|scope| {
133151
let mut handles = Vec::with_capacity(chunks.len());
134152
for chunk in chunks {
135153
handles.push(scope.spawn(move || {
136154
let mut groups = BTreeMap::new();
137-
let timezone =
138-
parse_tz(shared.timezone.as_deref()).or_else(|| Some(JiffTimeZone::system()));
155+
let timezone = parse_tz(run.shared.timezone.as_deref())
156+
.or_else(|| Some(JiffTimeZone::system()));
139157
for index in chunk {
140-
aggregate_file(
141-
sessions_dir,
142-
&files[index],
143-
kind,
144-
timezone.as_ref(),
145-
shared,
146-
seen,
147-
&mut groups,
148-
)?;
158+
aggregate_file(run, &run.files[index], timezone.as_ref(), seen, &mut groups)?;
149159
}
150160
Result::<BTreeMap<String, CodexGroup>>::Ok(groups)
151161
}));
@@ -165,63 +175,49 @@ fn aggregate_files_parallel(
165175
}
166176

167177
fn aggregate_file(
168-
sessions_dir: &Path,
178+
run: &CodexAggregateRun<'_>,
169179
file: &Path,
170-
kind: AgentReportKind,
171180
timezone: Option<&JiffTimeZone>,
172-
shared: &SharedArgs,
173181
seen: &CodexDedupeShards,
174182
groups: &mut BTreeMap<String, CodexGroup>,
175183
) -> Result<()> {
176-
parser::visit_codex_session_file(sessions_dir, file, |event| {
177-
add_event_to_groups(&event, kind, timezone, shared, seen, groups)
178-
})
184+
parser::visit_codex_session_file(
185+
run.sessions_dir,
186+
file,
187+
run.replay_plan.replay_prefix(file),
188+
|event| add_event_to_groups(&event, run.kind, timezone, run.shared, seen, groups),
189+
)
179190
}
180191

181-
fn aggregate_files_local(
182-
sessions_dir: &Path,
183-
files: &[PathBuf],
184-
shared: &SharedArgs,
185-
kind: AgentReportKind,
186-
) -> Result<BTreeMap<String, CodexGroup>> {
187-
Ok(aggregate_files_local_with_seen(sessions_dir, files, shared, kind)?.groups)
192+
fn aggregate_files_local(run: &CodexAggregateRun<'_>) -> Result<BTreeMap<String, CodexGroup>> {
193+
Ok(aggregate_files_local_with_seen(run)?.groups)
188194
}
189195

190-
fn aggregate_files_local_with_seen(
191-
sessions_dir: &Path,
192-
files: &[PathBuf],
193-
shared: &SharedArgs,
194-
kind: AgentReportKind,
195-
) -> Result<CodexAggregation> {
196+
fn aggregate_files_local_with_seen(run: &CodexAggregateRun<'_>) -> Result<CodexAggregation> {
196197
let mut aggregation = CodexAggregation {
197198
groups: BTreeMap::new(),
198199
seen: FxHashSet::default(),
199200
};
200-
let timezone = parse_tz(shared.timezone.as_deref()).or_else(|| Some(JiffTimeZone::system()));
201-
for file in files {
202-
aggregate_file_local(
203-
sessions_dir,
204-
file,
205-
kind,
206-
timezone.as_ref(),
207-
shared,
208-
&mut aggregation,
209-
)?;
201+
let timezone =
202+
parse_tz(run.shared.timezone.as_deref()).or_else(|| Some(JiffTimeZone::system()));
203+
for file in run.files {
204+
aggregate_file_local(run, file, timezone.as_ref(), &mut aggregation)?;
210205
}
211206
Ok(aggregation)
212207
}
213208

214209
fn aggregate_file_local(
215-
sessions_dir: &Path,
210+
run: &CodexAggregateRun<'_>,
216211
file: &Path,
217-
kind: AgentReportKind,
218212
timezone: Option<&JiffTimeZone>,
219-
shared: &SharedArgs,
220213
aggregation: &mut CodexAggregation,
221214
) -> Result<()> {
222-
parser::visit_codex_session_file(sessions_dir, file, |event| {
223-
add_event_to_groups_local(&event, kind, timezone, shared, aggregation)
224-
})
215+
parser::visit_codex_session_file(
216+
run.sessions_dir,
217+
file,
218+
run.replay_plan.replay_prefix(file),
219+
|event| add_event_to_groups_local(&event, run.kind, timezone, run.shared, aggregation),
220+
)
225221
}
226222

227223
fn add_event_to_groups(
@@ -496,6 +492,70 @@ mod tests {
496492
adapter::codex::paths::CodexUsageSource, model_aliases::set_model_aliases_for_tests,
497493
};
498494

495+
#[test]
496+
fn skips_forked_parent_prefix_rewritten_across_seconds() {
497+
fn token_count(timestamp: &str, input: u64, total_input: u64) -> String {
498+
json!({
499+
"timestamp": timestamp,
500+
"type": "event_msg",
501+
"payload": {
502+
"type": "token_count",
503+
"info": {
504+
"model": "gpt-5.2",
505+
"last_token_usage": {
506+
"input_tokens": input,
507+
"output_tokens": 1,
508+
"total_tokens": input + 1,
509+
},
510+
"total_token_usage": {
511+
"input_tokens": total_input,
512+
"output_tokens": 1,
513+
"total_tokens": total_input + 1,
514+
},
515+
},
516+
},
517+
})
518+
.to_string()
519+
}
520+
521+
let fixture = fs_fixture!({
522+
"sessions/parent.jsonl": [
523+
json!({
524+
"type": "session_meta",
525+
"payload": {"id": "parent"},
526+
})
527+
.to_string(),
528+
token_count("2026-07-10T08:01:00.000Z", 100, 100),
529+
token_count("2026-07-10T08:02:00.000Z", 200, 300),
530+
]
531+
.join("\n"),
532+
"sessions/child.jsonl": [
533+
json!({
534+
"type": "session_meta",
535+
"payload": {"id": "child", "forked_from_id": "parent"},
536+
})
537+
.to_string(),
538+
token_count("2026-07-10T09:00:00.100Z", 100, 100),
539+
token_count("2026-07-10T09:00:01.100Z", 200, 300),
540+
token_count("2026-07-10T09:00:02.100Z", 50, 350),
541+
]
542+
.join("\n"),
543+
});
544+
let shared = SharedArgs {
545+
timezone: Some("UTC".to_string()),
546+
single_thread: true,
547+
..SharedArgs::default()
548+
};
549+
550+
let groups =
551+
load_groups_from_directory(&fixture.path("sessions"), &shared, AgentReportKind::Daily)
552+
.unwrap();
553+
554+
let group = groups.get("2026-07-10").unwrap();
555+
assert_eq!(group.input_tokens, 350);
556+
assert_eq!(group.total_tokens, 353);
557+
}
558+
499559
#[test]
500560
fn dedupes_copied_token_usage_across_session_files() {
501561
let usage_line = json!({

0 commit comments

Comments
 (0)