Skip to content

Commit a9de927

Browse files
fix(sessions): fail closed when an accepted terminal output cannot be restored
Structured and custom terminal outputs and compaction-aware Sessions are supported public paths, but they were left with the old non-resumable behavior. Retrying one of those states still entered the model and ran the agent lifecycle hooks again, after the tool effect and the terminal-output hooks had already completed once. That is the reported failure, only in the cases the previous boundary excluded. Every post-acceptance terminal append now takes the same checkpoint, and the checkpoint records which of two outcomes a resume gets: - a losslessly persisted string settles the original output, as before; - anything else persists an explicit terminal-unrecoverable state. That covers an output the codec cannot round-trip and a batch whose post-append maintenance the replay cannot reproduce. An unrecoverable state rejects the resume with an actionable UserError before the pending Session write, sandbox preparation, and any agent, model, tool, guardrail, or lifecycle hook work. It is a fail-closed branch rather than another recovery mode, and it stays closed on every later resume because the marker does not depend on the pending write.
1 parent 71ed7f8 commit a9de927

6 files changed

Lines changed: 261 additions & 73 deletions

File tree

‎src/agents/run.py‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -140,14 +140,15 @@
140140
persist_session_items_for_guardrail_trip,
141141
prepare_input_with_session,
142142
reconcile_nested_history_owned_session_item_refs,
143-
recoverable_terminal_step,
143+
record_terminal_checkpoint,
144144
resume_pending_session_write,
145145
resumed_turn_items,
146146
resumed_write_owner,
147147
save_result_to_session,
148148
save_resumed_turn_items,
149149
session_items_for_turn,
150-
terminal_checkpoint_owner,
150+
settleable_terminal_step,
151+
terminal_state_is_unrecoverable,
151152
update_run_state_after_resume,
152153
)
153154
from .run_internal.tool_use_tracker import (
@@ -640,9 +641,16 @@ async def _run_impl(
640641
)
641642
context = context_wrapper.context
642643

644+
if terminal_state_is_unrecoverable(run_state):
645+
# Fail closed before any Session, sandbox, guardrail, model, tool, or hook
646+
# work: this run already produced an output that cannot be produced again.
647+
raise UserError(
648+
"This RunState accepted a final output that cannot be restored, so the "
649+
"run cannot be resumed. Start a new run instead of retrying this state."
650+
)
643651
# Capture before reconciling: settling the pending write clears the checkpoint
644-
# that proves the append was the only work still owed for this run.
645-
resumed_terminal_step = recoverable_terminal_step(run_state)
652+
# that proves the output was accepted before the append failed.
653+
resumed_terminal_step = settleable_terminal_step(run_state)
646654
await resume_pending_session_write(run_state, session, wrapper=context_wrapper)
647655
max_turns = run_state._max_turns
648656
else:
@@ -1394,15 +1402,16 @@ def _mark_response_hooks_started() -> None:
13941402
current_agent,
13951403
run_config,
13961404
)
1405+
# Reached only after every output guardrail succeeded, so
1406+
# this append is the post-acceptance one.
13971407
if (
1398-
terminal_checkpoint_owner(
1408+
record_terminal_checkpoint(
13991409
run_state, session, settle_terminal_output=True
14001410
)
14011411
is not None
14021412
):
1403-
# Every output guardrail passed for this exact output. Publish
1404-
# them before the append can raise so a settled retry reports
1405-
# the original evidence instead of evaluating a new one.
1413+
# Publish the results before the append can raise so a settled
1414+
# retry reports the original evidence instead of a new one.
14061415
run_state._output_guardrail_results = list(
14071416
output_guardrail_results
14081417
)

‎src/agents/run_internal/agent_runner_helpers.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,9 +42,9 @@
4242
ProcessedResponse,
4343
)
4444
from .session_persistence import (
45+
record_terminal_checkpoint,
4546
save_result_to_session,
4647
save_resumed_turn_items,
47-
terminal_checkpoint_owner,
4848
)
4949
from .tool_use_tracker import AgentToolUseTracker, serialize_tool_use_tracker
5050
from .turn_preparation import get_model
@@ -628,8 +628,8 @@ async def save_final_turn_items_after_guardrails(
628628
return 0
629629
# An accepted terminal output is appended after the output guardrails, so this is the last
630630
# fallible step of the run. Only the caller that saw every guardrail succeed opts in to
631-
# letting the resumed state own the batch and settle the same output later.
632-
terminal_write_state = terminal_checkpoint_owner(
631+
# checkpointing it, and the checkpoint records whether a resume may settle it or must reject.
632+
terminal_write_state = record_terminal_checkpoint(
633633
run_state, session, settle_terminal_output=settle_terminal_output
634634
)
635635
if run_state is not None and run_state._current_turn_persisted_item_count > 0:

‎src/agents/run_internal/run_loop.py‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -177,15 +177,16 @@
177177
persist_session_items_for_guardrail_trip,
178178
prepare_input_with_session,
179179
reconcile_nested_history_owned_session_item_refs,
180-
recoverable_terminal_step,
180+
record_terminal_checkpoint,
181181
resume_pending_session_write,
182182
resumed_turn_items,
183183
resumed_write_owner,
184184
rewind_session_items,
185185
save_result_to_session,
186186
save_resumed_turn_items,
187187
session_items_for_turn,
188-
terminal_checkpoint_owner,
188+
settleable_terminal_step,
189+
terminal_state_is_unrecoverable,
189190
update_run_state_after_resume,
190191
)
191192
from .streaming import stream_step_items_to_queue, stream_step_result_to_queue
@@ -624,12 +625,14 @@ async def _finalize_streamed_final_output(
624625
raise redacted_persistence_error from None
625626

626627
streamed_result.output_guardrail_results.extend(output_guardrail_results)
628+
# Reached only after every output guardrail succeeded, so this append is the
629+
# post-acceptance one.
627630
if (
628-
terminal_checkpoint_owner(streamed_result._state, session, settle_terminal_output=True)
631+
record_terminal_checkpoint(streamed_result._state, session, settle_terminal_output=True)
629632
is not None
630633
):
631-
# Every output guardrail passed for this exact output. Publish them before the append can
632-
# raise so a settled retry reports the original evidence instead of evaluating a new one.
634+
# Publish the results before the append can raise so a settled retry reports the original
635+
# evidence instead of evaluating a new one.
633636
streamed_result._state._output_guardrail_results = list(
634637
streamed_result.output_guardrail_results
635638
)
@@ -941,9 +944,16 @@ async def start_streaming(
941944

942945
resumed_terminal_step: NextStepFinalOutput | None = None
943946
if is_resumed_state and run_state is not None:
947+
if terminal_state_is_unrecoverable(run_state):
948+
# Fail closed before any Session, sandbox, guardrail, model, tool, or hook
949+
# work: this run already produced an output that cannot be produced again.
950+
raise UserError(
951+
"This RunState accepted a final output that cannot be restored, so the "
952+
"run cannot be resumed. Start a new run instead of retrying this state."
953+
)
944954
# Capture before reconciling: settling the pending write clears the checkpoint
945-
# that proves the append was the only work still owed for this run.
946-
resumed_terminal_step = recoverable_terminal_step(run_state)
955+
# that proves the output was accepted before the append failed.
956+
resumed_terminal_step = settleable_terminal_step(run_state)
947957
await resume_pending_session_write(run_state, session, wrapper=context_wrapper)
948958
streamed_result._current_turn_persisted_item_count = (
949959
run_state._current_turn_persisted_item_count

‎src/agents/run_internal/session_persistence.py‎

Lines changed: 79 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -81,9 +81,10 @@
8181
"save_result_to_session",
8282
"save_resumed_turn_items",
8383
"resume_pending_session_write",
84+
"record_terminal_checkpoint",
8485
"resumed_write_owner",
85-
"terminal_checkpoint_owner",
86-
"recoverable_terminal_step",
86+
"settleable_terminal_step",
87+
"terminal_state_is_unrecoverable",
8788
"update_run_state_after_resume",
8889
"rewind_session_items",
8990
"wait_for_session_cleanup",
@@ -723,44 +724,83 @@ async def save_result_to_session(
723724
return saved_run_items_count
724725

725726

726-
def terminal_checkpoint_owner(
727+
class TerminalOutputUnrecoverable:
728+
"""Marker for a terminal output accepted before a Session append that cannot be restored.
729+
730+
The value itself is never persisted, so a state carrying this marker is resumable only in
731+
the sense that it can be rejected: the run already produced its output and already ran its
732+
terminal hooks, and neither can be reproduced without repeating them.
733+
"""
734+
735+
__slots__ = ()
736+
737+
def __repr__(self) -> str:
738+
return "<terminal output unrecoverable>"
739+
740+
def __copy__(self) -> TerminalOutputUnrecoverable:
741+
return self
742+
743+
def __deepcopy__(self, memo: dict[int, Any]) -> TerminalOutputUnrecoverable:
744+
return self
745+
746+
747+
TERMINAL_OUTPUT_UNRECOVERABLE = TerminalOutputUnrecoverable()
748+
749+
750+
def _terminal_output_is_restorable(output: Any, session: Session | None) -> bool:
751+
"""Return whether an accepted terminal output can be settled on a later resume.
752+
753+
Two things have to survive to the next resume for that to be safe:
754+
755+
- the output itself. Only a plain string round-trips through the RunState codec unchanged;
756+
richer values fall back to ``json.dumps(..., default=str)`` and would change type.
757+
- the rest of the batch's persistence. A compaction-aware Session also records a deferred
758+
compaction, which the checkpoint does not carry and the replay would skip.
759+
"""
760+
if type(output) is not str:
761+
return False
762+
return not is_openai_responses_compaction_aware_session(session)
763+
764+
765+
def record_terminal_checkpoint(
727766
run_state: RunState[Any, Any] | None,
728767
session: Session | None,
729768
*,
730769
settle_terminal_output: bool,
731770
) -> RunState[Any, Any] | None:
732-
"""Return the state that may own a deferred terminal append.
733-
734-
A later resume settles this batch by replaying the append alone, so the checkpoint may only
735-
be armed when the append really is all that is left of the run:
736-
737-
- ``settle_terminal_output`` says every output guardrail completed. Only the successful
738-
finalize path can state that, so callers opt in instead of it being inferred from the step:
739-
a guardrail that raised must surface on the next resume rather than be settled away.
740-
- the accepted output is a plain string. Richer values fall back to
741-
``json.dumps(..., default=str)`` and would change type across the codec, so they keep the
742-
existing non-resumable behavior.
743-
- the Session owes no post-append maintenance. A compaction-aware Session also records a
744-
deferred compaction for the batch, which the checkpoint does not carry, so settling one
745-
would drop maintenance the first attempt would have performed.
771+
"""Record how an accepted terminal output may be resumed, and return the state that owns it.
772+
773+
Every post-acceptance terminal append gets a checkpoint, so no accepted output can be
774+
silently replayed by a later resume. What differs is what the checkpoint promises: a
775+
restorable output is kept so the resume can settle it, and anything else is replaced by
776+
``TERMINAL_OUTPUT_UNRECOVERABLE`` so the resume fails closed instead.
777+
778+
``settle_terminal_output`` says every output guardrail completed. Only the successful
779+
finalize path can state that, so callers opt in rather than it being inferred from the step:
780+
a guardrail that raised must surface on the next resume, not be checkpointed over.
781+
782+
The step is replaced rather than mutated because the live run still holds the same
783+
``NextStepFinalOutput`` for the result it is building.
746784
"""
747785
if not settle_terminal_output or run_state is None:
748786
return None
749787
step = run_state._current_step
750-
if not isinstance(step, NextStepFinalOutput) or type(step.output) is not str:
751-
return None
752-
if is_openai_responses_compaction_aware_session(session):
788+
if not isinstance(step, NextStepFinalOutput):
753789
return None
790+
if _terminal_output_is_restorable(step.output, session):
791+
run_state._current_step = NextStepFinalOutput(output=step.output)
792+
else:
793+
run_state._current_step = NextStepFinalOutput(output=TERMINAL_OUTPUT_UNRECOVERABLE)
754794
return run_state
755795

756796

757-
def recoverable_terminal_step(run_state: RunState[Any, Any] | None) -> NextStepFinalOutput | None:
797+
def settleable_terminal_step(run_state: RunState[Any, Any] | None) -> NextStepFinalOutput | None:
758798
"""Return the accepted terminal step a resumed run may settle without another model call.
759799
760800
The step is only settleable while the append it is waiting on is still outstanding. The
761-
pending write is that record, and it is only armed by ``terminal_checkpoint_owner``, so its
762-
presence is what proves the append was all that remained. Reconciling clears it, so callers
763-
capture this before settling it.
801+
pending write is that record, and only ``record_terminal_checkpoint`` arms it, so its
802+
presence is what proves the output was accepted. Reconciling clears it, so callers capture
803+
this before settling it.
764804
"""
765805
if run_state is None or run_state._pending_session_write is None:
766806
return None
@@ -770,6 +810,18 @@ def recoverable_terminal_step(run_state: RunState[Any, Any] | None) -> NextStepF
770810
return step
771811

772812

813+
def terminal_state_is_unrecoverable(run_state: RunState[Any, Any] | None) -> bool:
814+
"""Return whether this state accepted a terminal output that can no longer be produced.
815+
816+
Unlike a settleable step this does not depend on the pending write. Settling the append
817+
repairs history but cannot bring the output back, so the state stays closed to resumes.
818+
"""
819+
step = run_state._current_step if run_state is not None else None
820+
return isinstance(step, NextStepFinalOutput) and isinstance(
821+
step.output, TerminalOutputUnrecoverable
822+
)
823+
824+
773825
def resumed_write_owner(
774826
run_state: RunState[Any, Any] | None,
775827
session: Session | None,
@@ -778,14 +830,14 @@ def resumed_write_owner(
778830
) -> RunState[Any, Any] | None:
779831
"""Return the state that owns a resumed write, so a failed append settles on the next resume.
780832
781-
A run-again or interruption step always owns its write. A terminal step only does under the
782-
stricter rule in ``terminal_checkpoint_owner``, because settling one ends the run.
833+
A run-again or interruption step always owns its write. A terminal step owns its write once
834+
``record_terminal_checkpoint`` has recorded how that output may be resumed.
783835
"""
784836
if run_state is None:
785837
return None
786838
if isinstance(run_state._current_step, NextStepRunAgain | NextStepInterruption):
787839
return run_state
788-
return terminal_checkpoint_owner(
840+
return record_terminal_checkpoint(
789841
run_state, session, settle_terminal_output=settle_terminal_output
790842
)
791843

‎src/agents/run_state.py‎

Lines changed: 36 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -226,7 +226,8 @@ def _default_run_state_validation_error(
226226
),
227227
"1.17": (
228228
"Persists Docker container labels and current-response generated-item ownership across "
229-
"resume flows, including pending resumed Session writes and accepted terminal outputs."
229+
"resume flows, including pending resumed Session writes and accepted terminal "
230+
"outputs, recoverable or not."
230231
),
231232
}
232233
SUPPORTED_SCHEMA_VERSIONS = frozenset(SCHEMA_VERSION_SUMMARIES)
@@ -1982,8 +1983,15 @@ def _serialize_processed_response(
19821983
def _serialize_current_step(self) -> dict[str, Any] | None:
19831984
"""Serialize the current resumable step."""
19841985
# Import at runtime to avoid circular import
1985-
from .run_internal.run_steps import NextStepInterruption, NextStepRunAgain
1986-
from .run_internal.session_persistence import recoverable_terminal_step
1986+
from .run_internal.run_steps import (
1987+
NextStepFinalOutput,
1988+
NextStepInterruption,
1989+
NextStepRunAgain,
1990+
)
1991+
from .run_internal.session_persistence import (
1992+
TerminalOutputUnrecoverable,
1993+
settleable_terminal_step,
1994+
)
19871995

19881996
agent_identity_keys_by_id = (
19891997
_build_agent_identity_keys_by_id(cast(Agent[Any], self._starting_agent))
@@ -1994,12 +2002,21 @@ def _serialize_current_step(self) -> dict[str, Any] | None:
19942002
if isinstance(self._current_step, NextStepRunAgain):
19952003
return {"type": "next_step_run_again"}
19962004

1997-
terminal_step = recoverable_terminal_step(self)
1998-
if terminal_step is not None:
1999-
return {
2000-
"type": "next_step_final_output",
2001-
"data": {"output": terminal_step.output},
2002-
}
2005+
if isinstance(self._current_step, NextStepFinalOutput):
2006+
if isinstance(self._current_step.output, TerminalOutputUnrecoverable):
2007+
return {
2008+
"type": "next_step_final_output",
2009+
"data": {"unrecoverable": True},
2010+
}
2011+
settleable = settleable_terminal_step(self)
2012+
if settleable is not None:
2013+
return {
2014+
"type": "next_step_final_output",
2015+
"data": {"output": settleable.output},
2016+
}
2017+
# A terminal step the checkpoint never recorded carries no resume promise, so there
2018+
# is nothing durable to write for it.
2019+
return None
20032020

20042021
if self._current_step is None or not isinstance(self._current_step, NextStepInterruption):
20052022
return None
@@ -4319,13 +4336,20 @@ async def _build_run_state_from_json(
43194336
state._current_step = NextStepRunAgain()
43204337
elif current_step_data and current_step_data.get("type") == "next_step_final_output":
43214338
from .run_internal.run_steps import NextStepFinalOutput
4339+
from .run_internal.session_persistence import TERMINAL_OUTPUT_UNRECOVERABLE
43224340

4323-
terminal_output = current_step_data.get("data", {}).get("output")
4341+
terminal_data = current_step_data.get("data", {})
4342+
terminal_output = terminal_data.get("output")
4343+
unrecoverable = terminal_data.get("unrecoverable") is True
43244344
# An older label never wrote this step, so accepting one would grant a snapshot a
43254345
# resume shortcut the schema it claims does not have.
4326-
if (schema_major, schema_minor) < (1, 17) or not isinstance(terminal_output, str):
4346+
if (schema_major, schema_minor) < (1, 17) or not (
4347+
unrecoverable or isinstance(terminal_output, str)
4348+
):
43274349
raise validation_error_factory("Run state terminal output step is invalid", UserError)
4328-
state._current_step = NextStepFinalOutput(output=terminal_output)
4350+
state._current_step = NextStepFinalOutput(
4351+
output=TERMINAL_OUTPUT_UNRECOVERABLE if unrecoverable else terminal_output
4352+
)
43294353
elif current_step_data and current_step_data.get("type") == "next_step_interruption":
43304354
interruptions: list[ToolApprovalItem] = []
43314355
interruptions_data = current_step_data.get("data", {}).get(

0 commit comments

Comments
 (0)