Skip to content

Commit 0035388

Browse files
committed
fix(codex): stream session files
Read Codex JSONL sessions through a buffered line reader instead of loading each file into memory at once. This keeps codex reports usable when a session file grows to multiple gigabytes while preserving the existing line-level parsing, model tracking, and usage aggregation behavior.
1 parent 78b93c2 commit 0035388

1 file changed

Lines changed: 62 additions & 11 deletions

File tree

‎rust/crates/ccusage/src/codex_loader.rs‎

Lines changed: 62 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
use std::{
22
env, fs,
3+
io::{BufRead, BufReader},
34
path::{Path, PathBuf},
45
thread,
56
};
@@ -8,10 +9,7 @@ use compact_str::CompactString;
89
use serde_json::Value;
910

1011
use crate::{
11-
chunk_file_indexes_by_size,
12-
cli::SharedArgs,
13-
cli_error, collect_usage_files,
14-
fast::{byte_lines, FxHashSet},
12+
chunk_file_indexes_by_size, cli::SharedArgs, cli_error, collect_usage_files, fast::FxHashSet,
1513
home, non_empty_json_string, progress, CodexRawUsage, CodexTokenUsageEvent, Result,
1614
TimestampMs,
1715
};
@@ -137,22 +135,43 @@ fn read_codex_session_file(sessions_dir: &Path, path: &Path) -> Vec<CodexTokenUs
137135
pub(crate) fn visit_codex_session_file(
138136
sessions_dir: &Path,
139137
path: &Path,
140-
mut visit: impl FnMut(CodexTokenUsageEvent) -> Result<()>,
138+
visit: impl FnMut(CodexTokenUsageEvent) -> Result<()>,
141139
) -> Result<()> {
142-
let Ok(content) = fs::read(path) else {
140+
let Ok(file) = fs::File::open(path) else {
143141
return Ok(());
144142
};
143+
let fallback_timestamp = file_modified_timestamp(path);
144+
visit_codex_session_reader(
145+
sessions_dir,
146+
path,
147+
BufReader::new(file),
148+
&fallback_timestamp,
149+
visit,
150+
)
151+
}
152+
153+
fn visit_codex_session_reader(
154+
sessions_dir: &Path,
155+
path: &Path,
156+
mut reader: impl BufRead,
157+
fallback_timestamp: &str,
158+
mut visit: impl FnMut(CodexTokenUsageEvent) -> Result<()>,
159+
) -> Result<()> {
145160
let session_id = codex_session_id(sessions_dir, path);
146161
let mut previous_totals: Option<CodexRawUsage> = None;
147162
let mut current_model: Option<String> = None;
148163
let mut current_model_is_fallback = false;
149-
let fallback_timestamp = file_modified_timestamp(path);
164+
let mut line = Vec::new();
150165

151-
for line in byte_lines(&content) {
152-
if !codex_line_may_contain_usage(line) {
166+
loop {
167+
line.clear();
168+
if reader.read_until(b'\n', &mut line)? == 0 {
169+
break;
170+
}
171+
if !codex_line_may_contain_usage(&line) {
153172
continue;
154173
}
155-
let Ok(value) = serde_json::from_slice::<Value>(line) else {
174+
let Ok(value) = serde_json::from_slice::<Value>(&line) else {
156175
continue;
157176
};
158177
let entry_type = value.get("type").and_then(Value::as_str);
@@ -167,7 +186,7 @@ pub(crate) fn visit_codex_session_file(
167186
add_codex_exec_event(
168187
&session_id,
169188
&value,
170-
&fallback_timestamp,
189+
fallback_timestamp,
171190
&mut current_model,
172191
&mut current_model_is_fallback,
173192
&mut visit,
@@ -641,4 +660,36 @@ mod tests {
641660

642661
fs::remove_dir_all(dir).unwrap();
643662
}
663+
664+
#[test]
665+
fn parses_codex_usage_from_streaming_reader() {
666+
let content = [
667+
r#"{"timestamp":"2026-01-02T00:00:00.000Z","type":"turn_context","payload":{"model":"gpt-5.2-codex"}}"#,
668+
r#"{"timestamp":"2026-01-02T00:01:00.000Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":42,"cached_input_tokens":10,"output_tokens":7,"reasoning_output_tokens":1,"total_tokens":50}}}}"#,
669+
]
670+
.join("\n");
671+
let mut events = Vec::new();
672+
673+
visit_codex_session_reader(
674+
Path::new("sessions"),
675+
Path::new("sessions/project/session.jsonl"),
676+
std::io::Cursor::new(content.into_bytes()),
677+
"1970-01-01T00:00:00.000Z",
678+
|event| {
679+
events.push(event);
680+
Ok(())
681+
},
682+
)
683+
.unwrap();
684+
685+
assert_eq!(events.len(), 1);
686+
assert_eq!(events[0].session_id, "project/session");
687+
assert_eq!(events[0].timestamp, "2026-01-02T00:01:00.000Z");
688+
assert_eq!(events[0].model.as_deref(), Some("gpt-5.2-codex"));
689+
assert_eq!(events[0].input_tokens, 42);
690+
assert_eq!(events[0].cached_input_tokens, 10);
691+
assert_eq!(events[0].output_tokens, 7);
692+
assert_eq!(events[0].reasoning_output_tokens, 1);
693+
assert_eq!(events[0].total_tokens, 50);
694+
}
644695
}

0 commit comments

Comments
 (0)