Skip to content

Commit d2f6580

Browse files
Reconcile interrupted local tool calls
1 parent 7ceb5da commit d2f6580

12 files changed

Lines changed: 558 additions & 38 deletions

File tree

‎packages/codex/rust/lib.rs‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1372,13 +1372,21 @@ impl NativeThread {
13721372
})
13731373
}
13741374

1375-
fn _validate_local_executor_policy<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
1375+
fn _reconcile_local_calls<'py>(
1376+
&self,
1377+
py: Python<'py>,
1378+
receipts_json: String,
1379+
) -> PyResult<Bound<'py, PyAny>> {
13761380
let inner = self.current()?;
13771381
async_bridge::future_into_py(py, async move {
13781382
let _context_operation = inner.engine.operation(false).map_err(py_error)?;
1379-
local_reconciliation::validate_executor_policy(&inner)
1380-
.await
1381-
.map_err(py_error)
1383+
async_bridge::retain_writer_operation(async move {
1384+
let _context_operation = _context_operation;
1385+
local_reconciliation::reconcile(&inner, &receipts_json)
1386+
.await
1387+
.map_err(py_error)
1388+
})
1389+
.await
13821390
})
13831391
}
13841392

‎packages/codex/rust/local_reconciliation.rs‎

Lines changed: 194 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,90 @@
1-
//! Read and validate applied executor policies for restore.
1+
//! Import application-confirmed outcomes for original local calls without replay.
2+
3+
use std::collections::{HashMap, HashSet};
24

35
use anyhow::{Context, Result, ensure};
46
use codex_history::RolloutItem;
7+
use codex_protocol::models::{
8+
ContentItemKind, FunctionCallOutputBody, FunctionCallOutputPayload,
9+
InternalChatMessageMetadataPassthrough, ResponseItem,
10+
};
511
use codex_protocol::protocol::sdk_executor_policy_receipt;
12+
use serde::Deserialize;
613
use serde_json::Value;
714

815
use crate::engine::ThreadState;
916

17+
const APPLIED: &str = "sdk_reconciliation.applied";
18+
const NOT_APPLIED: &str = "sdk_reconciliation.not_applied";
19+
20+
#[derive(Clone, Copy, Deserialize)]
21+
#[serde(rename_all = "snake_case")]
22+
enum Outcome {
23+
Applied,
24+
NotApplied,
25+
}
26+
27+
impl Outcome {
28+
fn tag(self) -> &'static str {
29+
match self {
30+
Self::Applied => APPLIED,
31+
Self::NotApplied => NOT_APPLIED,
32+
}
33+
}
34+
}
35+
36+
#[derive(Deserialize)]
37+
#[serde(deny_unknown_fields)]
38+
struct ReconciledCall {
39+
thread_id: String,
40+
turn_id: String,
41+
call_id: String,
42+
outcome: Outcome,
43+
result: String,
44+
}
45+
46+
fn output(item: &ResponseItem) -> Option<(&str, &FunctionCallOutputPayload)> {
47+
match item {
48+
ResponseItem::FunctionCallOutput {
49+
call_id: Some(call_id),
50+
output,
51+
..
52+
}
53+
| ResponseItem::CustomToolCallOutput {
54+
call_id, output, ..
55+
} => Some((call_id, output)),
56+
_ => None,
57+
}
58+
}
59+
60+
fn tag(item: &ResponseItem) -> Option<&str> {
61+
let metadata = match item {
62+
ResponseItem::FunctionCallOutput {
63+
internal_chat_message_metadata_passthrough,
64+
..
65+
}
66+
| ResponseItem::CustomToolCallOutput {
67+
internal_chat_message_metadata_passthrough,
68+
..
69+
} => internal_chat_message_metadata_passthrough.as_ref()?,
70+
_ => return None,
71+
};
72+
let tags = metadata.content_item_kinds.as_deref()?;
73+
match tags {
74+
[ContentItemKind(tag)] if tag == APPLIED || tag == NOT_APPLIED => Some(tag),
75+
_ => None,
76+
}
77+
}
78+
79+
fn matches_receipt(item: &ResponseItem, receipt: &ReconciledCall) -> bool {
80+
item.turn_id() == Some(receipt.turn_id.as_str())
81+
&& tag(item) == Some(receipt.outcome.tag())
82+
&& output(item).is_some_and(|(id, payload)| {
83+
id == receipt.call_id
84+
&& matches!(&payload.body, FunctionCallOutputBody::Text(text) if text == &receipt.result)
85+
})
86+
}
87+
1088
pub(crate) async fn executor_policies(state: &ThreadState) -> Result<Vec<Value>> {
1189
let manager = state.engine.manager.environment_manager();
1290
let local_network = if state
@@ -35,17 +113,19 @@ pub(crate) async fn executor_policies(state: &ThreadState) -> Result<Vec<Value>>
35113
.collect()
36114
}
37115

38-
pub(crate) async fn validate_executor_policy(state: &ThreadState) -> Result<()> {
116+
pub(crate) async fn reconcile(state: &ThreadState, encoded: &str) -> Result<()> {
117+
let receipts: Vec<ReconciledCall> = serde_json::from_str(encoded)
118+
.map_err(|_| anyhow::anyhow!("Invalid local reconciliation receipt"))?;
39119
let tools = state
40120
.thread
41121
.thread_extension_data()
42122
.get::<crate::python_tools::PythonTools>()
43-
.context("Local resume requires an SDK-owned Session")?;
123+
.context("Reconciliation requires an SDK-owned Session")?;
44124
let store = state
45125
.engine
46126
.local_checklist_store
47127
.as_ref()
48-
.context("Local resume requires a local storage writer")?;
128+
.context("Reconciliation requires a local storage writer")?;
49129
let _mutation = state.mutations.lock().await;
50130
tools.check_idle(&state.thread).await?;
51131
let history = store.load_owned_history(state.id).await?;
@@ -76,5 +156,115 @@ pub(crate) async fn validate_executor_policy(state: &ThreadState) -> Result<()>
76156
"Local resume requires the recorded executor write and network policy"
77157
);
78158
}
159+
let guard = state
160+
.thread
161+
.thread_extension_data()
162+
.get::<crate::recovery_guard::Guard>()
163+
.context("Reconciliation requires the SDK replay guard")?;
164+
165+
let mut seen = HashMap::new();
166+
let mut outputs = HashMap::new();
167+
let mut protected = HashSet::new();
168+
let mut context_turn = None;
169+
for record in &history {
170+
match record {
171+
RolloutItem::TurnContext(context) => {
172+
context_turn = context.turn_id.as_deref().or(context_turn);
173+
}
174+
RolloutItem::ResponseItem(envelope) => {
175+
let item = &envelope.item;
176+
let Some(turn) = item.turn_id().or(context_turn) else {
177+
continue;
178+
};
179+
match item {
180+
ResponseItem::FunctionCall { call_id, .. }
181+
| ResponseItem::CustomToolCall { call_id, .. } => {
182+
let entry = seen.entry((turn, call_id.as_str())).or_insert(Some(item));
183+
if *entry != Some(item) {
184+
*entry = None;
185+
}
186+
}
187+
_ => {
188+
if let Some((call_id, _)) = output(item) {
189+
let entry = outputs.entry((turn, call_id)).or_insert(Some(item));
190+
if *entry != Some(item) {
191+
*entry = None;
192+
}
193+
if tag(item).is_some() {
194+
protected.insert(call_id.to_owned());
195+
}
196+
}
197+
}
198+
}
199+
}
200+
_ => {}
201+
}
202+
}
203+
let mut requested = HashSet::new();
204+
let mut append = Vec::new();
205+
for receipt in &receipts {
206+
let key = (receipt.turn_id.as_str(), receipt.call_id.as_str());
207+
ensure!(
208+
receipt.thread_id == thread_id && requested.insert(key),
209+
"Reconciliation receipt is duplicate or belongs to a different thread"
210+
);
211+
let call = seen
212+
.get(&key)
213+
.copied()
214+
.flatten()
215+
.context("Reconciliation does not identify one retained invocation")?;
216+
if let Some(existing) = outputs.get(&key) {
217+
ensure!(
218+
existing.is_some_and(|existing| matches_receipt(existing, receipt)),
219+
"Reconciliation conflicts with a retained result"
220+
);
221+
continue;
222+
}
223+
let output = FunctionCallOutputPayload {
224+
body: FunctionCallOutputBody::Text(receipt.result.clone()),
225+
success: Some(matches!(receipt.outcome, Outcome::Applied)),
226+
};
227+
let metadata = Some(InternalChatMessageMetadataPassthrough {
228+
turn_id: Some(receipt.turn_id.clone()),
229+
content_item_kinds: Some(vec![ContentItemKind(receipt.outcome.tag().to_owned())]),
230+
..Default::default()
231+
});
232+
append.push(match call {
233+
ResponseItem::FunctionCall {
234+
name, namespace, ..
235+
} => ResponseItem::FunctionCallOutput {
236+
id: None,
237+
call_id: Some(receipt.call_id.clone()),
238+
name: Some(name.clone()),
239+
namespace: namespace.clone(),
240+
output,
241+
internal_chat_message_metadata_passthrough: metadata,
242+
},
243+
ResponseItem::CustomToolCall { name, .. } => ResponseItem::CustomToolCallOutput {
244+
id: None,
245+
call_id: receipt.call_id.clone(),
246+
name: Some(name.clone()),
247+
output,
248+
internal_chat_message_metadata_passthrough: metadata,
249+
},
250+
_ => unreachable!("the call index retains only supported invocations"),
251+
});
252+
protected.insert(receipt.call_id.clone());
253+
}
254+
if !append.is_empty() {
255+
state.thread.inject_response_items(append).await?;
256+
state.thread.flush_rollout().await?;
257+
let durable = store.load_owned_history(state.id).await?;
258+
for receipt in &receipts {
259+
ensure!(
260+
durable.iter().any(|record| matches!(
261+
record,
262+
RolloutItem::ResponseItem(envelope) if matches_receipt(&envelope.item, receipt)
263+
)),
264+
"Reconciliation result was not confirmed in native storage"
265+
);
266+
}
267+
}
268+
guard.set_reconciled(protected);
79269
Ok(())
80270
}

‎packages/codex/rust/recovery_guard.rs‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -49,10 +49,18 @@ pub struct Guard {
4949
#[derive(Default)]
5050
struct State {
5151
current: Option<Arc<HashSet<String>>>,
52+
reconciled: Option<Arc<HashSet<String>>>,
5253
origins: HashMap<String, Origin>,
5354
}
5455

5556
impl Guard {
57+
pub(crate) fn set_reconciled(&self, ids: HashSet<String>) {
58+
self.state
59+
.lock()
60+
.expect("recovery guard poisoned")
61+
.reconciled = (!ids.is_empty()).then(|| Arc::new(ids));
62+
}
63+
5664
pub(crate) fn has_active_recovery(&self) -> bool {
5765
self.state
5866
.lock()
@@ -158,12 +166,14 @@ impl ModelRequestContributor for RecoveryContributor {
158166
.expect("recovery index poisoned")
159167
.get(input.thread_id)?
160168
.upgrade()?;
161-
let original_ids = guard
162-
.state
163-
.lock()
164-
.expect("recovery guard poisoned")
165-
.current
166-
.clone()?;
169+
let state = guard.state.lock().expect("recovery guard poisoned");
170+
let original_ids = match (&state.current, &state.reconciled) {
171+
(Some(current), Some(reconciled)) => {
172+
Arc::new(current.union(reconciled).cloned().collect())
173+
}
174+
(Some(ids), None) | (None, Some(ids)) => Arc::clone(ids),
175+
(None, None) => return None,
176+
};
167177
Some(Box::new(Request(original_ids)))
168178
}
169179
}

‎packages/codex/tests/test_history.py‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -298,6 +298,48 @@ def test_canonical_execution_replaces_legacy_receipt_and_keeps_model_records():
298298
assert old[0]["error"] == {"message": "legacy result"}
299299

300300

301+
def test_reconciliation_binds_to_the_original_turn_even_when_the_call_id_is_reused():
302+
def call(turn):
303+
return [
304+
event("task_started", turn),
305+
response("function_call", call_id="shared", name="write", arguments="{}"),
306+
]
307+
308+
def reconciliation(original_turn):
309+
return response(
310+
"function_call_output",
311+
call_id="shared",
312+
output="application-confirmed no effect",
313+
internal_chat_message_metadata_passthrough={
314+
"turn_id": original_turn,
315+
"content_item_kinds": ["sdk_reconciliation.not_applied"],
316+
},
317+
)
318+
319+
_, items = project_history(
320+
[
321+
*call("original"),
322+
event("turn_aborted", "original"),
323+
*call("next"),
324+
reconciliation("unknown"),
325+
reconciliation("original"),
326+
response("function_call_output", call_id="shared", output="actual result"),
327+
event("task_complete", "next"),
328+
],
329+
"root",
330+
)
331+
assert [(item["type"], item["turn_id"], item["status"]) for item in items] == [
332+
("function_call", "original", "failed"),
333+
("function_call", "next", "completed"),
334+
("function_call_output", "original", "failed"),
335+
("function_call_output", "next", "completed"),
336+
]
337+
assert items[2]["type"] == "function_call_output"
338+
assert items[2]["reconciliation"] == "not_applied"
339+
assert items[2]["output"] == "application-confirmed no effect"
340+
assert "reconciliation" not in items[3]
341+
342+
301343
def test_latest_sampled_context_receipt_survives_inherited_turn_completion():
302344
def receipt(model):
303345
return {

‎src/agents/__init__.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,11 @@
6060
ApiRateLimits as ApiRateLimits,
6161
ApiRateLimitWindow as ApiRateLimitWindow,
6262
)
63+
from .recovery import (
64+
LocalRecoveryRequired as LocalRecoveryRequired,
65+
ReconciledCall as ReconciledCall,
66+
UncertainCall as UncertainCall,
67+
)
6368
from .schema import (
6469
NativeOutput as NativeOutput,
6570
OutputSpec as OutputSpec,
@@ -148,6 +153,9 @@
148153
"OutputValidationError": ".schema",
149154
"CheckpointStatus": ".state",
150155
"LocalStorage": ".state",
156+
"LocalRecoveryRequired": ".recovery",
157+
"ReconciledCall": ".recovery",
158+
"UncertainCall": ".recovery",
151159
"ApiRateLimits": ".rate_limits",
152160
"ApiRateLimitWindow": ".rate_limits",
153161
"InputImage": ".input",

‎src/agents/_events.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -535,6 +535,7 @@ class FunctionCallOutputItem(TypedDict):
535535
status: FunctionCallStatus
536536
output: str | list[InputText | InputImage] | None
537537
error: str | None
538+
reconciliation: NotRequired[Literal["applied", "not_applied"]]
538539

539540

540541
class McpCallItem(TypedDict):

0 commit comments

Comments
 (0)