Skip to content

Commit 60389a0

Browse files
miiiiiiichclaude
andcommitted
Codex: skip the fork replay burst structurally
Layer a parse-time skip under the content-based keys (GH-36) — the approach ccusage took for the same bug (ccusage/ccusage#950) — with a broader trigger: a second session_meta marks a fork-shaped rollout (catches Desktop thread branches that carry no thread_spawn marker), and event lines stamped in the same second as that meta are the replay burst. - Replayed history now contributes nothing even when the parent rollout is outside the scan (deleted or past the mtime window), where cross-file dedup has no original to collapse into and previously the replay was counted once at the fork instant. - Durations (unkeyed) and session touches from replayed lines stop counting, closing two known remainders. - Content-based keys stay as the safety net for burst lines that leak past the fork second, whole-file copies, and re-emissions. Real-data check: same-instant A/B with and without the skip yields byte-identical Codex totals on an all-parents-in-scan corpus — the skip only changes outcomes where originals are missing from the scan. Co-Authored-By: Claude Fable 5 <[email protected]>
1 parent 84def1d commit 60389a0

2 files changed

Lines changed: 72 additions & 11 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
1717
(usage vectors + running cumulative; `turn_id` for effort), re-emitted
1818
`token_count` events whose cumulative did not advance collapse too, and
1919
keyed duplicates keep the earliest observed timestamp, so scan order
20-
can't decide attribution when both the original and the replay are
21-
scanned. Known remainder, scoped as follow-up: session touches and task
22-
durations from replayed lines are still counted, and a replay whose
23-
original falls outside the scan window keeps the fork-instant timestamp.
20+
can't decide attribution. Fork-shaped rollouts (second `session_meta`)
21+
additionally skip the replay burst at the fork instant wholesale — so
22+
replayed history contributes nothing (tokens, rate-limit samples,
23+
efforts, durations, session touches) even when the parent rollout is
24+
outside the scan window, and Desktop thread branches without a
25+
`thread_spawn` marker are covered too.
2426

2527
## [0.10.0] - 2026-07-13
2628

‎src/collector/codex.rs‎

Lines changed: 66 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,8 @@ fn parse_file(path: &Path, local_offset: UtcOffset) -> Option<FileEvents> {
7979
let mut current_session_id = fallback_session_id(path);
8080
let mut current_model = None;
8181
let mut current_project = None;
82+
let mut session_meta_count = 0usize;
83+
let mut replay_second: Option<i64> = None;
8284
let reader = BufReader::new(file);
8385

8486
for (line_index, line) in reader.lines().enumerate() {
@@ -97,20 +99,44 @@ fn parse_file(path: &Path, local_offset: UtcOffset) -> Option<FileEvents> {
9799

98100
let timestamp = parse_timestamp(value.get("timestamp"));
99101
if value.get("type").and_then(Value::as_str) == Some("session_meta") {
102+
session_meta_count += 1;
103+
if session_meta_count == 2 {
104+
replay_second = timestamp.map(OffsetDateTime::unix_timestamp);
105+
}
100106
current_session_id = string_path(&value, &["payload", "id"]).or(current_session_id);
101107
current_model = session_model(&value).or(current_model);
102108
current_project =
103109
string_path(&value, &["payload", "cwd"]).map(|cwd| project_from_cwd(&cwd));
104110
}
111+
// A fork/spawn child rollout opens with its own session_meta, then the
112+
// ancestors' copied session_metas, then the ancestor history replayed
113+
// in one write burst — every replayed line stamped at the fork
114+
// instant (GH-36). Skip event collection for that burst wholesale:
115+
// this works even when the parent rollout itself is outside the scan
116+
// (deleted, or past the mtime window), where cross-file dedup has no
117+
// original to collapse into. Burst lines that leak past the fork
118+
// second are still caught by the content-based keys when the parent
119+
// is in the scan. State tracking (session id, model, project) stays
120+
// live so post-burst lines parse with the right context.
121+
let in_replay_burst = session_meta_count >= 2
122+
&& match (replay_second, timestamp) {
123+
(Some(second), Some(ts)) => ts.unix_timestamp() == second,
124+
_ => false,
125+
};
105126
if value.get("type").and_then(Value::as_str) == Some("turn_context") {
106127
current_model = string_path(&value, &["payload", "model"]).or(current_model);
107-
collect_effort_event(
108-
&value,
109-
timestamp,
110-
current_session_id.as_ref(),
111-
line_index,
112-
&mut events,
113-
);
128+
if !in_replay_burst {
129+
collect_effort_event(
130+
&value,
131+
timestamp,
132+
current_session_id.as_ref(),
133+
line_index,
134+
&mut events,
135+
);
136+
}
137+
}
138+
if in_replay_burst {
139+
continue;
114140
}
115141

116142
if let (Some(timestamp), Some(session_id)) = (timestamp, current_session_id.as_ref()) {
@@ -733,6 +759,8 @@ mod tests {
733759
r#"{"timestamp":"2026-06-01T00:00:03Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":100,"cached_input_tokens":40,"output_tokens":10,"total_tokens":110},"total_token_usage":{"input_tokens":100,"cached_input_tokens":40,"output_tokens":10,"total_tokens":110}},"rate_limits":{"primary":{"used_percent":10.0,"window_minutes":300,"resets_at":1750000000}}}}"#,
734760
"\n",
735761
r#"{"timestamp":"2026-06-01T00:00:09Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":120,"cached_input_tokens":0,"output_tokens":20,"total_tokens":140},"total_token_usage":{"input_tokens":220,"cached_input_tokens":40,"output_tokens":30,"total_tokens":250}},"rate_limits":{"primary":{"used_percent":12.5,"window_minutes":300,"resets_at":1750000000}}}}"#,
762+
"\n",
763+
r#"{"timestamp":"2026-06-01T00:00:10Z","type":"event_msg","payload":{"type":"task_complete","duration_ms":111}}"#,
736764
"\n"
737765
)
738766
}
@@ -753,9 +781,13 @@ mod tests {
753781
"\n",
754782
r#"{"timestamp":"2026-06-01T01:00:00Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":120,"cached_input_tokens":0,"output_tokens":20,"total_tokens":140},"total_token_usage":{"input_tokens":220,"cached_input_tokens":40,"output_tokens":30,"total_tokens":250}},"rate_limits":{"primary":{"used_percent":12.5,"window_minutes":300,"resets_at":1750000000}}}}"#,
755783
"\n",
784+
r#"{"timestamp":"2026-06-01T01:00:00Z","type":"event_msg","payload":{"type":"task_complete","duration_ms":111}}"#,
785+
"\n",
756786
r#"{"timestamp":"2026-06-01T01:00:05Z","type":"turn_context","payload":{"model":"gpt-5.5","effort":"xhigh","turn_id":"t9"}}"#,
757787
"\n",
758788
r#"{"timestamp":"2026-06-01T01:00:07Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":90,"cached_input_tokens":10,"output_tokens":40,"total_tokens":130},"total_token_usage":{"input_tokens":310,"cached_input_tokens":50,"output_tokens":70,"total_tokens":380}},"rate_limits":{"primary":{"used_percent":15.0,"window_minutes":300,"resets_at":1750018000}}}}"#,
789+
"\n",
790+
r#"{"timestamp":"2026-06-01T01:00:08Z","type":"event_msg","payload":{"type":"task_complete","duration_ms":222}}"#,
759791
"\n"
760792
)
761793
}
@@ -771,6 +803,9 @@ mod tests {
771803
assert_eq!(total, 380); // 110 + 140 + 130
772804
assert_eq!(collection.rate_limit_samples.len(), 3);
773805
assert_eq!(collection.effort_events.len(), 2);
806+
// Replayed task_complete lines are skipped with the burst, so the
807+
// (unkeyed) duration events cannot double-count.
808+
assert_eq!(collection.duration_events.len(), 2);
774809

775810
// Replayed events keep the parent's ORIGINAL timestamps — day/hour
776811
// attribution must not shift to the fork instant, whatever the file
@@ -835,6 +870,30 @@ mod tests {
835870
assert_fork_replay_counted_once(&collection);
836871
}
837872

873+
/// The replay burst is skipped structurally, so it contributes nothing
874+
/// even when the parent rollout is NOT in the scan (deleted, or outside
875+
/// the mtime window) — the case cross-file dedup alone cannot cover.
876+
#[test]
877+
fn fork_replay_skipped_without_parent_in_scan() {
878+
let temp = TempDir::new().expect("test tempdir should be created");
879+
let day = temp.path().join("2026/06/01");
880+
fs::create_dir_all(&day).expect("test dirs should be created");
881+
fs::write(day.join("rollout-child.jsonl"), fork_child_lines())
882+
.expect("fixture should be written");
883+
884+
let collection = collect(temp.path(), None, false, UtcOffset::UTC);
885+
886+
// Only the child's own turn survives; the replayed parent history is
887+
// dropped instead of being counted at the fork instant.
888+
assert_eq!(collection.usage_events.len(), 1);
889+
assert_eq!(collection.usage_events[0].usage.token_volume(), 130);
890+
assert_eq!(collection.rate_limit_samples.len(), 1);
891+
assert_eq!(collection.effort_events.len(), 1);
892+
assert_eq!(collection.effort_events[0].effort, "xhigh");
893+
assert_eq!(collection.duration_events.len(), 1);
894+
assert_eq!(collection.duration_events[0].duration_ms, 222);
895+
}
896+
838897
/// A re-emitted `token_count` whose cumulative did not advance reports no
839898
/// new consumption — summing it again would overcount, so it collapses.
840899
#[test]

0 commit comments

Comments
 (0)