Skip to content

Commit facce31

Browse files
committed
fix(codex): follow a rewritten replay burst past a second tick
When a forked session's parent log is not in the scanned set, the parser cannot subtract the exact replayed prefix and falls back to skipping the burst Codex rewrote to the fork instant. That fallback bucketed events by their recorded second, so any burst written across a second tick was truncated and its remainder counted as the child's own usage. One real subagent log on hand reports 47,175,282 tokens through the fallback against 19,820,638 when the parent log is present, a 2.4x over-count from 315 replayed records that landed in the second after the tick. Follow the run instead of the second: skip while successive events stay within a second of each other. Measured across the fork logs on hand, a rewritten burst spans 10 to 40ms and the child's own first turn follows a pause of 5.8 to 15.3 seconds, so a second sits two orders of magnitude above the burst and well below the pause. Verified against 210 real fork sessions whose parent log could be located: the fallback now returns the same totals as exact prefix subtraction for every one of them, and a full scan is unchanged down to the cost figure. `skips_missing_parent_replay_when_duplicate_snapshot_is_suppressed` placed the child's own turn 800ms after the burst, which only read as the child's own under second bucketing. Its subject is snapshot suppression, so the fixture now uses a pause a real log would show and keeps testing that. Closes the gap left by #1457.
1 parent 24a3ed0 commit facce31

2 files changed

Lines changed: 102 additions & 21 deletions

File tree

‎rust/adapters/codex/src/loader.rs‎

Lines changed: 55 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1721,6 +1721,58 @@ mod tests {
17211721
);
17221722
}
17231723

1724+
#[test]
1725+
fn skips_a_rewritten_burst_that_straddles_a_second_boundary() {
1726+
let fixture = fs_fixture!({
1727+
// Codex writes the replayed history in a few milliseconds, so the
1728+
// burst lands on either side of a second tick whenever the fork
1729+
// happens late in a second. Measured against real logs, such a burst
1730+
// spans tens of milliseconds while the child's own first turn follows
1731+
// seconds later, so the run is what identifies it, not the second it
1732+
// was stamped with.
1733+
"child.jsonl": [
1734+
replay_metadata("2026-07-10T09:00:00.985Z", "child", Some("missing-parent")),
1735+
replay_token_count("2026-07-10T09:00:00.986Z", 100),
1736+
replay_token_count("2026-07-10T09:00:00.999Z", 200),
1737+
replay_token_count("2026-07-10T09:00:01.000Z", 300),
1738+
replay_token_count("2026-07-10T09:00:01.009Z", 400),
1739+
// The child's own first turn, after a real pause.
1740+
replay_token_count("2026-07-10T09:00:08.000Z", 500),
1741+
]
1742+
.join("\n"),
1743+
});
1744+
1745+
assert_eq!(
1746+
replay_input_tokens_by_session(fixture.root()),
1747+
[("child".to_string(), 500)]
1748+
);
1749+
}
1750+
1751+
#[test]
1752+
fn keeps_fork_local_usage_recorded_after_a_rewritten_burst() {
1753+
let fixture = fs_fixture!({
1754+
"child.jsonl": [
1755+
replay_metadata("2026-07-10T09:00:00.000Z", "child", Some("missing-parent")),
1756+
replay_token_count("2026-07-10T09:00:00.100Z", 100),
1757+
replay_token_count("2026-07-10T09:00:00.200Z", 200),
1758+
// Every later turn is the child's own, however many there are.
1759+
replay_token_count("2026-07-10T09:00:30.000Z", 300),
1760+
replay_token_count("2026-07-10T09:01:00.000Z", 400),
1761+
replay_token_count("2026-07-10T09:01:30.000Z", 500),
1762+
]
1763+
.join("\n"),
1764+
});
1765+
1766+
assert_eq!(
1767+
replay_input_tokens_by_session(fixture.root()),
1768+
[
1769+
("child".to_string(), 300),
1770+
("child".to_string(), 400),
1771+
("child".to_string(), 500),
1772+
]
1773+
);
1774+
}
1775+
17241776
#[test]
17251777
fn keeps_child_usage_matching_parent_event_after_fork() {
17261778
let fixture = fs_fixture!({
@@ -1841,9 +1893,11 @@ mod tests {
18411893
"payload": {"id": "child", "forked_from_id": "missing-parent"},
18421894
})
18431895
.to_string(),
1896+
// The replay repeats a snapshot, which must not end the burst.
18441897
token_count("2026-07-10T08:00:00.100Z", 100, 100),
18451898
token_count("2026-07-10T08:00:00.200Z", 100, 100),
1846-
token_count("2026-07-10T08:00:01.000Z", 50, 50),
1899+
// The child's own turn, after a real pause.
1900+
token_count("2026-07-10T08:00:08.000Z", 50, 50),
18471901
]
18481902
.join("\n"),
18491903
});

‎rust/adapters/codex/src/parser.rs‎

Lines changed: 47 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,9 @@ use memchr::memmem::Finder;
1010
use serde::Deserialize;
1111
use serde_json::Value;
1212

13-
use crate::{CodexRawUsage, CodexServiceTier, CodexTokenUsageEvent, Result, TimestampMs};
13+
use crate::{
14+
CodexRawUsage, CodexServiceTier, CodexTokenUsageEvent, Result, TimestampMs, parse_ts_timestamp,
15+
};
1416

1517
use super::types::{
1618
CodexInfo, CodexLogEntry, CodexModelMetadata, CodexPayload, CodexResultFields,
@@ -72,19 +74,37 @@ enum CodexReplayState<'a> {
7274
index: usize,
7375
},
7476
/// The parent stream could not anchor the replay, so skip the leading burst
75-
/// that Codex rewrote into a single second.
76-
SkippingSecond([u8; 19]),
77+
/// that Codex rewrote to the fork instant. Carries the last skipped event so
78+
/// the run can be followed however long it takes to write.
79+
SkippingRewrittenBurst(TimestampMs),
7780
/// Past the replayed history: every remaining event is the child's own usage.
7881
Done,
7982
}
8083

81-
fn detect_replay_second(path: &Path) -> Option<[u8; 19]> {
84+
/// Longest pause tolerated inside a burst of replayed usage.
85+
///
86+
/// Codex rewrites a replayed history to the fork instant and writes it in one
87+
/// go, so the burst is dense while the child's own first turn follows a real
88+
/// pause. Across the fork logs this was measured against, bursts spanned 10 to
89+
/// 40ms and the pause that followed ran from 5.8 to 15.3 seconds, so a second
90+
/// sits two orders of magnitude above the one and well below the other.
91+
///
92+
/// Bucketing by the recorded second instead would split any burst written across
93+
/// a second tick, leaving the remainder counted as the child's own usage.
94+
const CODEX_REWRITTEN_BURST_PAUSE_MS: i64 = 1_000;
95+
96+
/// Start of the burst of replayed usage at the head of `path`, if it has one.
97+
///
98+
/// A session that opens with two usage events written back to back replayed a
99+
/// history it did not spend; one that pauses between them was recording its own
100+
/// turns from the start.
101+
fn detect_rewritten_burst(path: &Path) -> Option<TimestampMs> {
82102
let Ok(file) = fs::File::open(path) else {
83103
return None;
84104
};
85105
let mut reader = BufReader::new(file);
86106
let mut line = Vec::new();
87-
let mut first_second: Option<[u8; 19]> = None;
107+
let mut first: Option<TimestampMs> = None;
88108

89109
loop {
90110
line.clear();
@@ -111,19 +131,19 @@ fn detect_replay_second(path: &Path) -> Option<[u8; 19]> {
111131
{
112132
continue;
113133
}
114-
let Some(timestamp) = codex_session_timestamp(value.timestamp.as_ref()) else {
115-
continue;
116-
};
117-
let Some(second) = timestamp
118-
.as_bytes()
119-
.get(..19)
120-
.and_then(|second| second.try_into().ok())
134+
let Some(timestamp) = codex_session_timestamp(value.timestamp.as_ref())
135+
.as_deref()
136+
.and_then(parse_ts_timestamp)
121137
else {
122138
continue;
123139
};
124-
match first_second {
125-
None => first_second = Some(second),
126-
Some(first) => return (first == second).then_some(first),
140+
match first {
141+
None => first = Some(timestamp),
142+
Some(first) => {
143+
return (timestamp.as_millis() - first.as_millis()
144+
<= CODEX_REWRITTEN_BURST_PAUSE_MS)
145+
.then_some(first);
146+
}
127147
}
128148
}
129149
}
@@ -176,14 +196,21 @@ pub(super) fn visit_codex_session_file(
176196
}
177197
// Nothing matched, so the parent stream cannot anchor this
178198
// replay: the log is unavailable, or Codex rewrote the copied
179-
// history. Fall back to the same-second burst instead.
199+
// history. Fall back to the rewritten burst instead.
180200
replay = (index == 0)
181-
.then(|| detect_replay_second(path))
201+
.then(|| detect_rewritten_burst(path))
182202
.flatten()
183-
.map_or(CodexReplayState::Done, CodexReplayState::SkippingSecond);
203+
.map_or(
204+
CodexReplayState::Done,
205+
CodexReplayState::SkippingRewrittenBurst,
206+
);
184207
}
185-
CodexReplayState::SkippingSecond(second) => {
186-
if event.timestamp.as_bytes().get(..19) == Some(second.as_slice()) {
208+
CodexReplayState::SkippingRewrittenBurst(previous) => {
209+
if let Some(timestamp) = parse_ts_timestamp(&event.timestamp)
210+
&& timestamp.as_millis() - previous.as_millis()
211+
<= CODEX_REWRITTEN_BURST_PAUSE_MS
212+
{
213+
replay = CodexReplayState::SkippingRewrittenBurst(timestamp);
187214
return Ok(());
188215
}
189216
replay = CodexReplayState::Done;

0 commit comments

Comments
 (0)