Skip to content

Commit 9b1faae

Browse files
Retain execution and instruction receipts
1 parent 95d38b9 commit 9b1faae

29 files changed

Lines changed: 2019 additions & 125 deletions
Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
diff --git a/codex-rs/rollout/src/lib.rs b/codex-rs/rollout/src/lib.rs
2+
--- a/codex-rs/rollout/src/lib.rs
3+
+++ b/codex-rs/rollout/src/lib.rs
4+
@@ -144,6 +144,7 @@
5+
pub use persistence_metrics::RolloutPersistenceTelemetry;
6+
pub use persistence_metrics::measure_and_filter_rollout_items;
7+
pub use policy::into_persisted_rollout_items;
8+
+pub use policy::persisted_event_msg;
9+
pub use policy::persisted_rollout_item;
10+
pub use policy::persisted_rollout_items;
11+
pub use policy::should_persist_response_item;
12+
diff --git a/codex-rs/rollout/src/policy.rs b/codex-rs/rollout/src/policy.rs
13+
--- a/codex-rs/rollout/src/policy.rs
14+
+++ b/codex-rs/rollout/src/policy.rs
15+
@@ -124,7 +124,8 @@
16+
}
17+
}
18+
19+
-fn persisted_event_msg(
20+
+/// Returns the authoritative durable representation of an event.
21+
+pub fn persisted_event_msg(
22+
ev: &EventMsg,
23+
history_mode: ThreadHistoryMode,
24+
) -> Option<Cow<'_, EventMsg>> {
25+
@@ -232,7 +233,13 @@
26+
history_mode: ThreadHistoryMode,
27+
) -> Option<Cow<'a, EventMsg>> {
28+
match history_mode {
29+
- ThreadHistoryMode::Legacy => {
30+
+ // The SDK also reads execution completions from legacy in-memory histories.
31+
+ ThreadHistoryMode::Legacy
32+
+ if !matches!(
33+
+ event.item,
34+
+ TurnItem::CommandExecution(_) | TurnItem::McpToolCall(_) | TurnItem::FileChange(_)
35+
+ ) =>
36+
+ {
37+
// Legacy rollouts keep only items with no lossless raw ResponseItem or legacy
38+
// equivalent.
39+
(matches!(
40+
@@ -247,7 +254,7 @@
41+
))
42+
.then_some(Cow::Borrowed(ev))
43+
}
44+
- ThreadHistoryMode::Paginated => match &event.item {
45+
+ ThreadHistoryMode::Legacy | ThreadHistoryMode::Paginated => match &event.item {
46+
TurnItem::CommandExecution(CommandExecutionItem {
47+
aggregated_output: Some(aggregated_output),
48+
..

‎packages/codex/patches/0020-sdk-turn-receipts.patch‎

Lines changed: 634 additions & 0 deletions
Large diffs are not rendered by default.

‎packages/codex/patches/series.json‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,15 @@
7171
{
7272
"path": "0018-api-key-rate-limits.patch",
7373
"sha256": "7214955c12f507f0534a7813205eeba656265aa6d919aa3704ec5a47f3ea7d64"
74+
},
75+
{
76+
"path": "0019-retained-execution-completions.patch",
77+
"sha256": "c96852d1582d17b99e5ffaafa110c471fd3af697a7b384afa15e86955897af10"
78+
},
79+
{
80+
"path": "0020-sdk-turn-receipts.patch",
81+
"sha256": "de462b9b82de958fa52b350cf17da9c8a6a1200005ba6a12c64d0cb4d0d70d94"
7482
}
7583
],
76-
"result_tree": "031267b70bbf9fda8d45193acc5d41776c136c5a"
84+
"result_tree": "e08df069309c6ed01d5dc0160f74df8ab90b7240"
7785
}

‎packages/codex/rust/application_instructions.rs‎

Lines changed: 121 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,23 @@
11
//! One application instruction owner, published through Core's native history.
22
3-
use std::sync::{Mutex, PoisonError};
3+
use std::sync::{LazyLock, Mutex, PoisonError};
44

55
use anyhow::{Context, Result, ensure};
66
use codex_core_api::CodexThread;
77
use codex_extension_api::{
88
ContextContributor, ExtensionFuture, PreviousWorldStateSection, RenderedWorldStateFragment,
9-
WorldStateContributionInput, WorldStateSectionContribution,
9+
SdkApplicationInstructions, SdkTurnReceiptSeed, WorldStateContributionInput,
10+
WorldStateSectionContribution,
1011
};
11-
use codex_history::RolloutItem;
12+
use codex_history::{CodexHarnessMetadata, ResponseItemEnvelope, RolloutItem};
1213
use codex_protocol::models::{
1314
ContentItem, ContentItemKind, InternalChatMessageMetadataPassthrough, ResponseItem,
1415
};
1516
use codex_protocol::protocol::EventMsg;
1617
use codex_protocol::{ResponseItemId, ThreadId};
18+
use serde::{Deserialize, Serialize};
1719
use serde_json::Value;
20+
use sha2::{Digest, Sha256};
1821

1922
const SECTION: &str = "sdk_application";
2023
const ACCEPTED: &str = "sdk_application.accepted";
@@ -24,10 +27,31 @@ const MARKERS: (&str, &str) = ("<application_instructions>", "</application_inst
2427
const REVISION: &str = "Native instruction revision: ";
2528
const NOTICE: &str = "The following application instructions replace all earlier application instructions. An empty value clears them.\n";
2629

30+
static CODEX_SOURCE_TREE: LazyLock<String> = LazyLock::new(|| {
31+
let manifest: Value = serde_json::from_str(include_str!("../patches/series.json"))
32+
.expect("embedded Codex patch manifest must be valid");
33+
manifest["result_tree"]
34+
.as_str()
35+
.expect("embedded Codex manifest must record the reviewed tree")
36+
.to_owned()
37+
});
38+
39+
#[derive(Clone, Deserialize, Serialize)]
40+
pub(crate) struct InstructionFile {
41+
path: String,
42+
source: String,
43+
sha256: String,
44+
}
45+
46+
fn sha256(text: &str) -> String {
47+
format!("{:x}", Sha256::digest(text.as_bytes()))
48+
}
49+
2750
#[derive(Clone)]
2851
pub struct Accepted {
2952
id: ResponseItemId,
3053
pub(crate) text: String,
54+
files: Vec<InstructionFile>,
3155
}
3256

3357
#[derive(Default)]
@@ -111,6 +135,7 @@ fn accepted(item: &ResponseItem) -> Result<Option<Accepted>> {
111135
Ok(Some(Accepted {
112136
id: id.clone(),
113137
text: text.to_owned(),
138+
files: Vec::new(),
114139
}))
115140
}
116141

@@ -123,12 +148,27 @@ pub fn validate(history: &[RolloutItem], expected: &str) -> Result<Accepted> {
123148
Ok(latest)
124149
}
125150

151+
fn accepted_envelope(envelope: &ResponseItemEnvelope) -> Result<Option<Accepted>> {
152+
let Some(mut accepted) = accepted(&envelope.item)? else {
153+
return Ok(None);
154+
};
155+
if let Some(files) = envelope
156+
.metadata
157+
.as_ref()
158+
.and_then(|metadata| metadata.sdk_instruction_files.clone())
159+
{
160+
accepted.files =
161+
serde_json::from_value(files).context("invalid native instruction file receipt")?;
162+
}
163+
Ok(Some(accepted))
164+
}
165+
126166
/// Only original accepted items change authority. Compaction copies never do.
127167
pub(crate) fn latest(history: &[RolloutItem]) -> Result<Accepted> {
128168
let mut originals: Vec<(&ResponseItem, Accepted)> = Vec::new();
129169
for row in history {
130170
if let RolloutItem::ResponseItem(item) = row {
131-
if let Some(accepted) = accepted(&item.item)? {
171+
if let Some(accepted) = accepted_envelope(item)? {
132172
ensure!(
133173
!originals
134174
.iter()
@@ -227,7 +267,7 @@ pub(crate) fn take_initial_receipt(
227267
history: &mut Vec<RolloutItem>,
228268
latest: &Accepted,
229269
root: ThreadId,
230-
) -> Option<ResponseItem> {
270+
) -> Option<ResponseItemEnvelope> {
231271
let mut receipt_index = None;
232272
for (index, row) in history.iter().enumerate() {
233273
match row {
@@ -236,16 +276,23 @@ pub(crate) fn take_initial_receipt(
236276
if event.thread_id == Some(root) => {}
237277
RolloutItem::ResponseItem(envelope)
238278
if receipt_index.is_none()
239-
&& envelope.metadata.is_none()
240-
&& envelope.item.id() == Some(&latest.id) =>
279+
&& envelope.item.id() == Some(&latest.id)
280+
&& envelope.metadata.as_ref().is_none_or(|metadata| {
281+
metadata.sdk_instruction_files.is_some()
282+
&& *metadata
283+
== CodexHarnessMetadata {
284+
sdk_instruction_files: metadata.sdk_instruction_files.clone(),
285+
..Default::default()
286+
}
287+
}) =>
241288
{
242289
receipt_index = Some(index);
243290
}
244291
_ => return None,
245292
}
246293
}
247294
match history.remove(receipt_index?) {
248-
RolloutItem::ResponseItem(envelope) => Some(envelope.item),
295+
RolloutItem::ResponseItem(envelope) => Some(envelope),
249296
_ => unreachable!("receipt_index identifies a response item"),
250297
}
251298
}
@@ -254,6 +301,7 @@ pub(crate) fn publication(text: &str) -> ResponseItem {
254301
let accepted = Accepted {
255302
id: ResponseItemId::new(ID_PREFIX),
256303
text: text.to_owned(),
304+
files: Vec::new(),
257305
};
258306
ResponseItem::Message {
259307
id: Some(accepted.id.clone()),
@@ -301,28 +349,62 @@ fn receipt<'a>(
301349

302350
/// The caller holds the existing native operation and idle mutation gates.
303351
pub async fn publish(thread: &CodexThread, text: String) -> Result<()> {
304-
publish_item(thread, publication(&text)).await
352+
publish_with_files(thread, text, Vec::new()).await
353+
}
354+
355+
pub(crate) async fn publish_with_files(
356+
thread: &CodexThread,
357+
text: String,
358+
files: Vec<InstructionFile>,
359+
) -> Result<()> {
360+
let item = publication(&text);
361+
let mut envelope = thread.annotate_client_response_item(item);
362+
if !files.is_empty() {
363+
envelope
364+
.metadata
365+
.get_or_insert_with(Default::default)
366+
.sdk_instruction_files = Some(serde_json::to_value(files)?);
367+
}
368+
publish_item(thread, envelope).await
305369
}
306370

307371
/// Publish an existing validated receipt on restore without allocating a revision.
308-
pub(crate) async fn publish_item(thread: &CodexThread, item: ResponseItem) -> Result<()> {
372+
pub(crate) async fn publish_item(
373+
thread: &CodexThread,
374+
envelope: ResponseItemEnvelope,
375+
) -> Result<()> {
309376
let owner = thread
310377
.thread_extension_data()
311378
.get::<Instructions>()
312379
.context("application instruction updates require a managed Session")?;
313-
let accepted = accepted(&item)?.context("missing application instructions publication")?;
314-
thread.inject_response_items(vec![item.clone()]).await?;
380+
let accepted =
381+
accepted_envelope(&envelope)?.context("missing application instructions publication")?;
382+
thread
383+
.inject_response_item_envelopes(vec![envelope.clone()])
384+
.await?;
315385
thread.flush_rollout().await?;
316386
let live = thread.conversation_history_snapshot().await;
317-
receipt(live.items(), &item)?;
387+
receipt(live.items(), &envelope.item)?;
318388
let stored = thread.load_history(true).await?;
319-
receipt(
320-
stored.items.iter().filter_map(|row| match row {
321-
RolloutItem::ResponseItem(item) => Some(&item.item),
322-
_ => None,
323-
}),
324-
&item,
325-
)?;
389+
let mut stored_items = stored.items.iter().filter_map(|row| match row {
390+
RolloutItem::ResponseItem(item) => Some(item),
391+
_ => None,
392+
});
393+
receipt(stored_items.clone().map(|item| &item.item), &envelope.item)?;
394+
let recorded = stored_items
395+
.find(|item| item.item.id() == envelope.item.id())
396+
.context("missing native application instruction receipt")?;
397+
ensure!(
398+
recorded
399+
.metadata
400+
.as_ref()
401+
.and_then(|value| value.sdk_instruction_files.as_ref())
402+
== envelope
403+
.metadata
404+
.as_ref()
405+
.and_then(|value| value.sdk_instruction_files.as_ref()),
406+
"native application instruction file receipt was not confirmed"
407+
);
326408
*owner.0.lock().unwrap_or_else(PoisonError::into_inner) = Some(accepted);
327409
Ok(())
328410
}
@@ -335,12 +417,27 @@ impl ContextContributor for Contributor {
335417
input: WorldStateContributionInput<'a>,
336418
) -> ExtensionFuture<'a, Vec<WorldStateSectionContribution>> {
337419
Box::pin(async move {
338-
let Some(owner) = input.thread_store.get::<Instructions>() else {
339-
return Vec::new();
340-
};
341420
// Initial publication must become durable before this cache can
342421
// contribute. Otherwise Core's injection preparation emits it twice.
343-
let Some(accepted) = owner.current() else {
422+
let accepted = input
423+
.thread_store
424+
.get::<Instructions>()
425+
.and_then(|owner| owner.current());
426+
input.step_store.insert(SdkTurnReceiptSeed {
427+
codex_revision: include_str!("../CODEX_REVISION").trim(),
428+
codex_source_tree: CODEX_SOURCE_TREE.as_str(),
429+
sha256,
430+
strict_executor_writes: false,
431+
application: accepted.as_ref().map(|accepted| SdkApplicationInstructions {
432+
revision: accepted.id.to_string(),
433+
rendered: rendered(accepted),
434+
developer_sha256: sha256(&accepted.text),
435+
files: accepted.files.iter().map(|file| {
436+
serde_json::json!({"path": file.path, "source": file.source, "sha256": file.sha256})
437+
}).collect(),
438+
}),
439+
});
440+
let Some(accepted) = accepted else {
344441
return Vec::new();
345442
};
346443
let exact = rendered(&accepted);

‎packages/codex/rust/engine.rs‎

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1363,8 +1363,18 @@ async fn read_child(children: &mut VecDeque<(ThreadId, ChildReader)>) -> (ChildE
13631363

13641364
fn native_event(
13651365
source: Option<(ThreadId, u64)>,
1366-
event: Event,
1366+
mut event: Event,
1367+
session_reader: bool,
13671368
) -> anyhow::Result<serde_json::Value> {
1369+
// Session observations and stored items use Core's same retention limit.
1370+
// Keep the full native event for callers of the low-level raw reader.
1371+
if session_reader
1372+
&& matches!(event.msg, EventMsg::ItemCompleted(_))
1373+
&& let Some(std::borrow::Cow::Owned(retained)) =
1374+
codex_rollout::persisted_event_msg(&event.msg, ThreadHistoryMode::Paginated)
1375+
{
1376+
event.msg = retained;
1377+
}
13681378
let mut value = serde_json::to_value(&event)?;
13691379
if let Some((id, instance)) = source {
13701380
value["subagent_id"] = serde_json::json!(id);
@@ -1469,6 +1479,7 @@ impl ThreadState {
14691479
self: &Arc<Self>,
14701480
source: ChildEventSource,
14711481
mut event: Event,
1482+
session_reader: bool,
14721483
) -> anyhow::Result<serde_json::Value> {
14731484
let mut current = false;
14741485
let mut turn_failed = None;
@@ -1505,7 +1516,7 @@ impl ThreadState {
15051516
} else {
15061517
None
15071518
};
1508-
let mut event = native_event(Some((source.0, source.1)), event)?;
1519+
let mut event = native_event(Some((source.0, source.1)), event, session_reader)?;
15091520
if let Some(grant) = question {
15101521
let id = ThreadId::new().to_string();
15111522
self.child_questions.lock().await.insert(
@@ -1562,7 +1573,7 @@ impl ThreadState {
15621573
let mut events = self.events.lock().await;
15631574
ensure!(!*closing.borrow(), "Codex thread is closed");
15641575
if let Some(event) = events.initial.take() {
1565-
return native_event(None, event);
1576+
return native_event(None, event, admission_ready);
15661577
}
15671578
loop {
15681579
if let Some(terminal) = events.session_terminal.as_mut() {
@@ -1579,7 +1590,7 @@ impl ThreadState {
15791590
{
15801591
settlement.delivered(&event.msg);
15811592
}
1582-
let mut value = native_event(None, event)?;
1593+
let mut value = native_event(None, event, admission_ready)?;
15831594
receipt.attach_provider_error(&mut value);
15841595
if !value["msg"]["error"].is_null() {
15851596
receipt.attach_policy_failure(&mut value);
@@ -1593,7 +1604,7 @@ impl ThreadState {
15931604
return graph_event(event);
15941605
}
15951606
if let Some((source, event)) = events.pending.pop_front() {
1596-
return self.child_event(source, event).await;
1607+
return self.child_event(source, event, admission_ready).await;
15971608
}
15981609
let ThreadEvents {
15991610
extensions,
@@ -1627,9 +1638,9 @@ impl ThreadState {
16271638
.get::<crate::turn_settlement::Settlement>() {
16281639
settlement.delivered(&event.msg);
16291640
}
1630-
return native_event(None, event);
1641+
return native_event(None, event, admission_ready);
16311642
},
1632-
(source, event) = read_child(children) => return self.child_event(source, event).await,
1643+
(source, event) = read_child(children) => return self.child_event(source, event, admission_ready).await,
16331644
Some(event) = graph.recv() => {
16341645
self.discover_children(&mut events).await?;
16351646
return graph_event(event);
@@ -1645,7 +1656,7 @@ impl ThreadState {
16451656
event = extensions.recv() => {
16461657
let (target, event) = event?;
16471658
if target.is_empty() || target == self.id.to_string() {
1648-
return native_event(None, event);
1659+
return native_event(None, event, admission_ready);
16491660
}
16501661
if let Ok(id) = ThreadId::from_string(&target) {
16511662
if !events.children.iter().any(|(child_id, _)| *child_id == id) {

0 commit comments

Comments
 (0)