Skip to content

Commit 2bc5a29

Browse files
fix(codex): skip history outside date window
Co-authored-by: zhangxaochen <[email protected]>
1 parent b2809fa commit 2bc5a29

6 files changed

Lines changed: 708 additions & 59 deletions

File tree

‎rust/adapters/codex/src/aggregate.rs‎

Lines changed: 187 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -75,21 +75,41 @@ fn load_groups_from_sources(
7575
kind: AgentReportKind,
7676
) -> Result<BTreeMap<String, CodexGroup>> {
7777
let file_groups = paths::collect_deduped_codex_usage_files(sources);
78-
let replay_plan = CodexReplayPlan::new(
79-
file_groups
80-
.iter()
81-
.map(|group| (group.dir.as_path(), group.files.as_slice())),
82-
shared.single_thread,
83-
);
78+
let files_by_group = file_groups
79+
.iter()
80+
.map(|group| paths::filter_codex_usage_files(&group.dir, &group.files, shared))
81+
.collect::<Vec<_>>();
82+
let replay_plan = if shared.since.is_some() || shared.until.is_some() {
83+
CodexReplayPlan::for_bounded_files(
84+
file_groups
85+
.iter()
86+
.zip(&files_by_group)
87+
.map(|(group, files)| (group.dir.as_path(), files.as_slice())),
88+
file_groups
89+
.iter()
90+
.map(|group| (group.dir.as_path(), group.files.as_slice())),
91+
shared.single_thread,
92+
)
93+
} else {
94+
CodexReplayPlan::new(
95+
file_groups
96+
.iter()
97+
.map(|group| (group.dir.as_path(), group.files.as_slice())),
98+
shared.single_thread,
99+
)
100+
};
84101
let mut groups = BTreeMap::new();
85102
let seen = create_dedupe_shards();
86-
for group in &file_groups {
103+
for (group, files) in file_groups.iter().zip(&files_by_group) {
104+
if files.is_empty() {
105+
continue;
106+
}
87107
merge_groups(
88108
&mut groups,
89109
aggregate_files_with_dedupe(
90110
&CodexAggregateRun {
91111
sessions_dir: &group.dir,
92-
files: &group.files,
112+
files: &files,
93113
shared,
94114
kind,
95115
replay_plan: &replay_plan,
@@ -107,9 +127,17 @@ pub(super) fn load_groups_from_directory(
107127
shared: &SharedArgs,
108128
kind: AgentReportKind,
109129
) -> Result<BTreeMap<String, CodexGroup>> {
110-
let files = paths::collect_codex_usage_files(sessions_dir);
111-
let replay_plan =
112-
CodexReplayPlan::new([(sessions_dir, files.as_slice())], shared.single_thread);
130+
let all_files = paths::collect_codex_usage_files(sessions_dir);
131+
let files = paths::filter_codex_usage_files(sessions_dir, &all_files, shared);
132+
let replay_plan = if shared.since.is_some() || shared.until.is_some() {
133+
CodexReplayPlan::for_bounded_files(
134+
[(sessions_dir, files.as_slice())],
135+
[(sessions_dir, all_files.as_slice())],
136+
shared.single_thread,
137+
)
138+
} else {
139+
CodexReplayPlan::new([(sessions_dir, all_files.as_slice())], shared.single_thread)
140+
};
113141
let run = CodexAggregateRun {
114142
sessions_dir,
115143
files: &files,
@@ -678,7 +706,7 @@ mod tests {
678706

679707
use crate::{
680708
CodexModelUsage, PricingMap, cli::CodexSpeed, model_aliases::set_model_aliases_for_tests,
681-
paths::CodexUsageSource,
709+
paths::CodexUsageSource, replay,
682710
};
683711

684712
#[test]
@@ -747,6 +775,153 @@ mod tests {
747775
);
748776
}
749777

778+
#[test]
779+
fn skips_historical_files_before_parsing_but_keeps_long_running_sessions() {
780+
let usage_line = |timestamp: &str, input_tokens: u64| {
781+
json!({
782+
"timestamp": timestamp,
783+
"type": "event_msg",
784+
"payload": {
785+
"type": "token_count",
786+
"info": {
787+
"model": "gpt-5",
788+
"last_token_usage": {
789+
"input_tokens": input_tokens,
790+
"cached_input_tokens": 0,
791+
"output_tokens": 1,
792+
"total_tokens": input_tokens + 1,
793+
},
794+
},
795+
},
796+
})
797+
.to_string()
798+
};
799+
let historical = [
800+
json!({
801+
"timestamp": "2025-01-01T08:00:00.000Z",
802+
"type": "session_meta",
803+
"payload": {"id": "historical"},
804+
})
805+
.to_string(),
806+
usage_line("2026-03-15T08:01:00.000Z", 999),
807+
]
808+
.join("\n");
809+
let long_running = [
810+
json!({
811+
"timestamp": "2026-01-01T08:00:00.000Z",
812+
"type": "session_meta",
813+
"payload": {"id": "long-running"},
814+
})
815+
.to_string(),
816+
usage_line("2026-01-01T08:01:00.000Z", 10),
817+
usage_line("2026-03-15T08:01:00.000Z", 100),
818+
]
819+
.join("\n");
820+
let fixture = fs_fixture!({
821+
"sessions/2025/01/01/historical.jsonl": &historical,
822+
"sessions/2026/01/01/long-running.jsonl": &long_running,
823+
});
824+
let historical_path = fixture.path("sessions/2025/01/01/historical.jsonl");
825+
let long_running_path = fixture.path("sessions/2026/01/01/long-running.jsonl");
826+
crate::paths::set_file_modified(
827+
&historical_path,
828+
parse_ts_timestamp("2025-01-01T08:01:00.000Z").unwrap(),
829+
);
830+
crate::paths::set_file_modified(
831+
&long_running_path,
832+
parse_ts_timestamp("2026-03-15T08:01:00.000Z").unwrap(),
833+
);
834+
let shared = SharedArgs {
835+
since: Some("20260315".to_string()),
836+
until: Some("20260315".to_string()),
837+
timezone: Some("UTC".to_string()),
838+
..SharedArgs::default()
839+
};
840+
841+
for single_thread in [true, false] {
842+
let _ = replay::take_observed_file_reads();
843+
let bounded = load_groups_from_directory(
844+
&fixture.path("sessions"),
845+
&SharedArgs {
846+
single_thread,
847+
..shared.clone()
848+
},
849+
AgentReportKind::Daily,
850+
)
851+
.unwrap();
852+
let group = &bounded["2026-03-15"];
853+
assert_eq!(group.input_tokens, 100);
854+
assert_eq!(group.output_tokens, 1);
855+
assert_eq!(group.total_tokens, 101);
856+
if single_thread {
857+
let reads = replay::take_observed_file_reads();
858+
assert!(reads.contains(&long_running_path));
859+
assert!(!reads.contains(&historical_path));
860+
}
861+
862+
let unbounded = load_groups_from_directory(
863+
&fixture.path("sessions"),
864+
&SharedArgs {
865+
single_thread,
866+
..SharedArgs::default()
867+
},
868+
AgentReportKind::Daily,
869+
)
870+
.unwrap();
871+
assert_eq!(unbounded["2026-03-15"].input_tokens, 1_099);
872+
}
873+
}
874+
875+
#[test]
876+
fn retains_next_utc_path_day_for_local_until_boundary() {
877+
let late_utc = [
878+
json!({
879+
"timestamp": "2026-03-16T06:29:00.000Z",
880+
"type": "session_meta",
881+
"payload": {"id": "late-utc"},
882+
})
883+
.to_string(),
884+
json!({
885+
"timestamp": "2026-03-16T06:30:00.000Z",
886+
"type": "event_msg",
887+
"payload": {
888+
"type": "token_count",
889+
"info": {
890+
"model": "gpt-5",
891+
"last_token_usage": {
892+
"input_tokens": 100,
893+
"cached_input_tokens": 0,
894+
"output_tokens": 1,
895+
"total_tokens": 101,
896+
},
897+
},
898+
},
899+
})
900+
.to_string(),
901+
]
902+
.join("\n");
903+
let fixture = fs_fixture!({
904+
"sessions/2026/03/16/late-utc.jsonl": &late_utc,
905+
});
906+
let late_utc_path = fixture.path("sessions/2026/03/16/late-utc.jsonl");
907+
let shared = SharedArgs {
908+
since: Some("20260315".to_string()),
909+
until: Some("20260315".to_string()),
910+
timezone: Some("America/Los_Angeles".to_string()),
911+
single_thread: true,
912+
..SharedArgs::default()
913+
};
914+
915+
let _ = replay::take_observed_file_reads();
916+
let groups =
917+
load_groups_from_directory(&fixture.path("sessions"), &shared, AgentReportKind::Daily)
918+
.unwrap();
919+
920+
assert_eq!(groups["2026-03-15"].input_tokens, 100);
921+
assert_eq!(groups["2026-03-15"].total_tokens, 101);
922+
assert!(replay::take_observed_file_reads().contains(&late_utc_path));
923+
}
924+
750925
#[test]
751926
fn dedupes_copied_token_usage_across_session_files() {
752927
let usage_line = json!({

0 commit comments

Comments
 (0)