Skip to content

Commit 527ec3a

Browse files
fix(codex): skip history outside date window (#1665)
* fix(codex): skip history outside date window Co-authored-by: zhangxaochen <[email protected]> * fix(codex): export bounded loader detection * style(codex): satisfy workspace clippy * style(codex): format utc path date * refactor(codex): remove unused loader wrapper * docs(codex): correct loader surface Replace the removed loader wrapper in the internal adapter inventory.\n\nThe unified adapter uses load_codex_events_with_detection, which is\nthe current loader entry point. * fix(codex): resolve bounded replay parents Build bounded replay parent indexes from authoritative session IDs.\n\nThis preserves deterministic first-wins resolution while full usage parsing\nremains limited to selected parent streams. * fix(codex): skip self parent candidates Keep all files sharing a session ID in deterministic order so a child excluded as a self-parent can fall back to the next snapshot. Add bounded loading coverage for duplicate IDs in both loading modes. * fix(codex): resolve self parent candidates Apply the deterministic candidate-list lookup to unbounded replay plans as well as bounded plans. Exclude each child path before selecting its first parent snapshot, preserving the missing-parent fallback for self-parent sessions. Cover duplicate self-parent snapshots and the no-alternative case in both loading modes. --------- Co-authored-by: zhangxaochen <[email protected]>
1 parent b2809fa commit 527ec3a

8 files changed

Lines changed: 1096 additions & 95 deletions

File tree

‎rust/adapters/codex/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ chunking, and ordered parallel reads.
3131
- `aggregate::aggregate_events`
3232
- `aggregate::filter_events_by_date`
3333
- `aggregate::load_groups`
34-
- `loader::load_codex_events`
34+
- `loader::load_codex_events_with_detection`
3535
- `loader::load_codex_events_from_directory`
3636
- `report::calculate_codex_model_cost`
3737
- `report::calculate_group_cost`

‎rust/adapters/codex/src/aggregate.rs‎

Lines changed: 197 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -75,21 +75,41 @@ fn load_groups_from_sources(
7575
kind: AgentReportKind,
7676
) -> Result<BTreeMap<String, CodexGroup>> {
7777
let file_groups = paths::collect_deduped_codex_usage_files(sources);
78-
let replay_plan = CodexReplayPlan::new(
79-
file_groups
80-
.iter()
81-
.map(|group| (group.dir.as_path(), group.files.as_slice())),
82-
shared.single_thread,
83-
);
78+
let files_by_group = file_groups
79+
.iter()
80+
.map(|group| paths::filter_codex_usage_files(&group.dir, &group.files, shared))
81+
.collect::<Vec<_>>();
82+
let replay_plan = if shared.since.is_some() || shared.until.is_some() {
83+
CodexReplayPlan::for_bounded_files(
84+
file_groups
85+
.iter()
86+
.zip(&files_by_group)
87+
.map(|(group, files)| (group.dir.as_path(), files.as_slice())),
88+
file_groups
89+
.iter()
90+
.map(|group| (group.dir.as_path(), group.files.as_slice())),
91+
shared.single_thread,
92+
)
93+
} else {
94+
CodexReplayPlan::new(
95+
file_groups
96+
.iter()
97+
.map(|group| (group.dir.as_path(), group.files.as_slice())),
98+
shared.single_thread,
99+
)
100+
};
84101
let mut groups = BTreeMap::new();
85102
let seen = create_dedupe_shards();
86-
for group in &file_groups {
103+
for (group, files) in file_groups.iter().zip(&files_by_group) {
104+
if files.is_empty() {
105+
continue;
106+
}
87107
merge_groups(
88108
&mut groups,
89109
aggregate_files_with_dedupe(
90110
&CodexAggregateRun {
91111
sessions_dir: &group.dir,
92-
files: &group.files,
112+
files,
93113
shared,
94114
kind,
95115
replay_plan: &replay_plan,
@@ -107,9 +127,17 @@ pub(super) fn load_groups_from_directory(
107127
shared: &SharedArgs,
108128
kind: AgentReportKind,
109129
) -> Result<BTreeMap<String, CodexGroup>> {
110-
let files = paths::collect_codex_usage_files(sessions_dir);
111-
let replay_plan =
112-
CodexReplayPlan::new([(sessions_dir, files.as_slice())], shared.single_thread);
130+
let all_files = paths::collect_codex_usage_files(sessions_dir);
131+
let files = paths::filter_codex_usage_files(sessions_dir, &all_files, shared);
132+
let replay_plan = if shared.since.is_some() || shared.until.is_some() {
133+
CodexReplayPlan::for_bounded_files(
134+
[(sessions_dir, files.as_slice())],
135+
[(sessions_dir, all_files.as_slice())],
136+
shared.single_thread,
137+
)
138+
} else {
139+
CodexReplayPlan::new([(sessions_dir, all_files.as_slice())], shared.single_thread)
140+
};
113141
let run = CodexAggregateRun {
114142
sessions_dir,
115143
files: &files,
@@ -678,7 +706,7 @@ mod tests {
678706

679707
use crate::{
680708
CodexModelUsage, PricingMap, cli::CodexSpeed, model_aliases::set_model_aliases_for_tests,
681-
paths::CodexUsageSource,
709+
paths::CodexUsageSource, replay,
682710
};
683711

684712
#[test]
@@ -747,6 +775,163 @@ mod tests {
747775
);
748776
}
749777

778+
#[test]
779+
fn skips_historical_files_before_parsing_but_keeps_long_running_sessions() {
780+
let usage_line = |timestamp: &str, input_tokens: u64| {
781+
json!({
782+
"timestamp": timestamp,
783+
"type": "event_msg",
784+
"payload": {
785+
"type": "token_count",
786+
"info": {
787+
"model": "gpt-5",
788+
"last_token_usage": {
789+
"input_tokens": input_tokens,
790+
"cached_input_tokens": 0,
791+
"output_tokens": 1,
792+
"total_tokens": input_tokens + 1,
793+
},
794+
},
795+
},
796+
})
797+
.to_string()
798+
};
799+
let historical = [
800+
json!({
801+
"timestamp": "2025-01-01T08:00:00.000Z",
802+
"type": "session_meta",
803+
"payload": {"id": "historical"},
804+
})
805+
.to_string(),
806+
usage_line("2026-03-15T08:01:00.000Z", 999),
807+
]
808+
.join("\n");
809+
let long_running = [
810+
json!({
811+
"timestamp": "2026-01-01T08:00:00.000Z",
812+
"type": "session_meta",
813+
"payload": {"id": "long-running"},
814+
})
815+
.to_string(),
816+
usage_line("2026-01-01T08:01:00.000Z", 10),
817+
usage_line("2026-03-15T08:01:00.000Z", 100),
818+
]
819+
.join("\n");
820+
let fixture = fs_fixture!({
821+
"sessions/2025/01/01/historical.jsonl": &historical,
822+
"sessions/2026/01/01/long-running.jsonl": &long_running,
823+
});
824+
let historical_path = fixture.path("sessions/2025/01/01/historical.jsonl");
825+
let long_running_path = fixture.path("sessions/2026/01/01/long-running.jsonl");
826+
crate::paths::set_file_modified(
827+
&historical_path,
828+
parse_ts_timestamp("2025-01-01T08:01:00.000Z").unwrap(),
829+
);
830+
crate::paths::set_file_modified(
831+
&long_running_path,
832+
parse_ts_timestamp("2026-03-15T08:01:00.000Z").unwrap(),
833+
);
834+
let shared = SharedArgs {
835+
since: Some("20260315".to_string()),
836+
until: Some("20260315".to_string()),
837+
timezone: Some("UTC".to_string()),
838+
..SharedArgs::default()
839+
};
840+
841+
for single_thread in [true, false] {
842+
let _ = replay::take_observed_file_read_events();
843+
let bounded = load_groups_from_directory(
844+
&fixture.path("sessions"),
845+
&SharedArgs {
846+
single_thread,
847+
..shared.clone()
848+
},
849+
AgentReportKind::Daily,
850+
)
851+
.unwrap();
852+
let group = &bounded["2026-03-15"];
853+
assert_eq!(group.input_tokens, 100);
854+
assert_eq!(group.output_tokens, 1);
855+
assert_eq!(group.total_tokens, 101);
856+
if single_thread {
857+
let reads = replay::take_observed_file_read_events();
858+
assert!(reads.iter().any(|read| matches!(
859+
read,
860+
replay::ObservedFileRead::MetadataProbe(path) if path == &long_running_path
861+
)));
862+
assert!(reads.iter().any(|read| matches!(
863+
read,
864+
replay::ObservedFileRead::MetadataProbe(path) if path == &historical_path
865+
)));
866+
assert!(!reads.iter().any(|read| matches!(
867+
read,
868+
replay::ObservedFileRead::ParentUsage(path) if path == &historical_path
869+
)));
870+
}
871+
872+
let unbounded = load_groups_from_directory(
873+
&fixture.path("sessions"),
874+
&SharedArgs {
875+
single_thread,
876+
..SharedArgs::default()
877+
},
878+
AgentReportKind::Daily,
879+
)
880+
.unwrap();
881+
assert_eq!(unbounded["2026-03-15"].input_tokens, 1_099);
882+
}
883+
}
884+
885+
#[test]
886+
fn retains_next_utc_path_day_for_local_until_boundary() {
887+
let late_utc = [
888+
json!({
889+
"timestamp": "2026-03-16T06:29:00.000Z",
890+
"type": "session_meta",
891+
"payload": {"id": "late-utc"},
892+
})
893+
.to_string(),
894+
json!({
895+
"timestamp": "2026-03-16T06:30:00.000Z",
896+
"type": "event_msg",
897+
"payload": {
898+
"type": "token_count",
899+
"info": {
900+
"model": "gpt-5",
901+
"last_token_usage": {
902+
"input_tokens": 100,
903+
"cached_input_tokens": 0,
904+
"output_tokens": 1,
905+
"total_tokens": 101,
906+
},
907+
},
908+
},
909+
})
910+
.to_string(),
911+
]
912+
.join("\n");
913+
let fixture = fs_fixture!({
914+
"sessions/2026/03/16/late-utc.jsonl": &late_utc,
915+
});
916+
let late_utc_path = fixture.path("sessions/2026/03/16/late-utc.jsonl");
917+
let shared = SharedArgs {
918+
since: Some("20260315".to_string()),
919+
until: Some("20260315".to_string()),
920+
timezone: Some("America/Los_Angeles".to_string()),
921+
single_thread: true,
922+
..SharedArgs::default()
923+
};
924+
925+
let _ = replay::take_observed_file_reads();
926+
let groups =
927+
load_groups_from_directory(&fixture.path("sessions"), &shared, AgentReportKind::Daily)
928+
.unwrap();
929+
930+
assert_eq!(groups["2026-03-15"].input_tokens, 100);
931+
assert_eq!(groups["2026-03-15"].total_tokens, 101);
932+
assert!(replay::take_observed_file_reads().contains(&late_utc_path));
933+
}
934+
750935
#[test]
751936
fn dedupes_copied_token_usage_across_session_files() {
752937
let usage_line = json!({

‎rust/adapters/codex/src/lib.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,9 @@ mod types;
1313
use crate::{PricingMap, Result, cli::AgentCommandArgs, log_level, print_json_or_jq, wants_json};
1414

1515
pub use aggregate::{aggregate_events, filter_events_by_date, load_groups};
16-
pub use loader::load_codex_events;
1716
#[doc(hidden)]
1817
pub use loader::load_codex_events_from_directory;
18+
pub use loader::load_codex_events_with_detection;
1919
pub use report::{
2020
calculate_codex_model_cost, calculate_group_cost, codex_model_missing_pricing,
2121
non_cached_input_tokens,

0 commit comments

Comments
 (0)