Skip to content

Commit 06ff1f8

Browse files
fix(sessions): arm a terminal checkpoint only from the successful finalize path
Three review findings landed on the same rule, so this replaces it rather than subtracting another case from it. The checkpoint replays one Session append, so it may only be armed when that append really is all the run has left. Ownership is now decided by terminal_checkpoint_owner() and opted into by the caller, instead of being inferred from the step: - Only the finalize path that saw every output guardrail succeed passes settle_terminal_output. The tripwire, guardrail-error, and max-turns saves reach the same helper and must not arm a checkpoint, because an error there has to surface on the next resume rather than be settled away. - A compaction-aware Session also owes a deferred compaction for the batch, which the checkpoint does not carry. Those runs keep the existing non-resumable behavior instead of settling a batch whose maintenance the replay would skip. resumed_write_owner() now resolves the owning state at the call sites, so save_resumed_turn_items() forwards an already resolved owner and its signature is unchanged.
1 parent d9b541f commit 06ff1f8

5 files changed

Lines changed: 206 additions & 54 deletions

File tree

‎src/agents/run.py‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -136,17 +136,18 @@
136136
from .run_internal.session_persistence import (
137137
_session_get_items,
138138
admit_pending_input,
139-
checkpointable_terminal_step,
140139
commit_server_pending_input,
141140
persist_session_items_for_guardrail_trip,
142141
prepare_input_with_session,
143142
reconcile_nested_history_owned_session_item_refs,
144143
recoverable_terminal_step,
145144
resume_pending_session_write,
146145
resumed_turn_items,
146+
resumed_write_owner,
147147
save_result_to_session,
148148
save_resumed_turn_items,
149149
session_items_for_turn,
150+
terminal_checkpoint_owner,
150151
update_run_state_after_resume,
151152
)
152153
from .run_internal.tool_use_tracker import (
@@ -1191,7 +1192,7 @@ def _mark_response_hooks_started() -> None:
11911192
):
11921193
run_state._current_turn_persisted_item_count = (
11931194
await save_resumed_turn_items(
1194-
run_state=run_state,
1195+
run_state=resumed_write_owner(run_state, session),
11951196
session=session,
11961197
items=turn_session_items,
11971198
persisted_count=(
@@ -1393,10 +1394,15 @@ def _mark_response_hooks_started() -> None:
13931394
current_agent,
13941395
run_config,
13951396
)
1396-
if checkpointable_terminal_step(run_state) is not None:
1397-
# These passed for this exact output. Publish them before the
1398-
# append can raise so a settled retry reports the original
1399-
# evidence instead of evaluating a new final output.
1397+
if (
1398+
terminal_checkpoint_owner(
1399+
run_state, session, settle_terminal_output=True
1400+
)
1401+
is not None
1402+
):
1403+
# Every output guardrail passed for this exact output. Publish
1404+
# them before the append can raise so a settled retry reports
1405+
# the original evidence instead of evaluating a new one.
14001406
run_state._output_guardrail_results = list(
14011407
output_guardrail_results
14021408
)
@@ -1409,6 +1415,8 @@ def _mark_response_hooks_started() -> None:
14091415
response_id=turn_result.model_response.response_id,
14101416
store=store_setting,
14111417
wrapper=context_wrapper,
1418+
# Reached only after every output guardrail succeeded.
1419+
settle_terminal_output=True,
14121420
)
14131421
current_step = getattr(run_state, "_current_step", None)
14141422
approvals_from_state = approvals_from_step(current_step)

‎src/agents/run_internal/agent_runner_helpers.py‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -42,9 +42,9 @@
4242
ProcessedResponse,
4343
)
4444
from .session_persistence import (
45-
checkpointable_terminal_step,
4645
save_result_to_session,
4746
save_resumed_turn_items,
47+
terminal_checkpoint_owner,
4848
)
4949
from .tool_use_tracker import AgentToolUseTracker, serialize_tool_use_tracker
5050
from .turn_preparation import get_model
@@ -619,17 +619,18 @@ async def save_final_turn_items_after_guardrails(
619619
reasoning_item_id_policy: ReasoningItemIdPolicy | None = None,
620620
store: bool | None = None,
621621
wrapper: RunContextWrapper[Any] | None = None,
622+
settle_terminal_output: bool = False,
622623
) -> int:
623624
"""Persist deferred final-turn items without skipping a partially persisted resumed turn."""
624625
if not session_persistence_enabled or not items:
625626
return 0
626627
if input_guardrails_triggered(input_guardrail_results):
627628
return 0
628629
# An accepted terminal output is appended after the output guardrails, so this is the last
629-
# fallible step of the run. Let the resumed state own the batch when it can settle the same
630-
# output later instead of re-entering the model.
631-
terminal_write_state = (
632-
run_state if checkpointable_terminal_step(run_state) is not None else None
630+
# fallible step of the run. Only the caller that saw every guardrail succeed opts in to
631+
# letting the resumed state own the batch and settle the same output later.
632+
terminal_write_state = terminal_checkpoint_owner(
633+
run_state, session, settle_terminal_output=settle_terminal_output
633634
)
634635
if run_state is not None and run_state._current_turn_persisted_item_count > 0:
635636
run_state._current_turn_persisted_item_count = await save_resumed_turn_items(

‎src/agents/run_internal/run_loop.py‎

Lines changed: 40 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -173,18 +173,19 @@
173173
from .session_persistence import (
174174
_session_get_items,
175175
admit_pending_input,
176-
checkpointable_terminal_step,
177176
commit_server_pending_input,
178177
persist_session_items_for_guardrail_trip,
179178
prepare_input_with_session,
180179
reconcile_nested_history_owned_session_item_refs,
181180
recoverable_terminal_step,
182181
resume_pending_session_write,
183182
resumed_turn_items,
183+
resumed_write_owner,
184184
rewind_session_items,
185185
save_result_to_session,
186186
save_resumed_turn_items,
187187
session_items_for_turn,
188+
terminal_checkpoint_owner,
188189
update_run_state_after_resume,
189190
)
190191
from .streaming import stream_step_items_to_queue, stream_step_result_to_queue
@@ -387,6 +388,7 @@ async def _save_resumed_stream_items(
387388
items: list[RunItem],
388389
response_id: str | None,
389390
store: bool | None = None,
391+
settle_terminal_output: bool = False,
390392
) -> None:
391393
if not await _should_persist_stream_items(
392394
session=session,
@@ -395,7 +397,9 @@ async def _save_resumed_stream_items(
395397
):
396398
return
397399
streamed_result._current_turn_persisted_item_count = await save_resumed_turn_items(
398-
run_state=run_state,
400+
run_state=resumed_write_owner(
401+
run_state, session, settle_terminal_output=settle_terminal_output
402+
),
399403
session=session,
400404
items=items,
401405
persisted_count=streamed_result._current_turn_persisted_item_count,
@@ -497,7 +501,8 @@ async def _finalize_streamed_final_output(
497501
run_config: RunConfig,
498502
output: Any,
499503
context_wrapper: RunContextWrapper[TContext],
500-
save_items: Callable[[list[RunItem], str | None, bool | None], Awaitable[None]],
504+
save_items: Callable[[list[RunItem], str | None, bool | None, bool], Awaitable[None]],
505+
session: Session | None,
501506
items: list[RunItem],
502507
model_response: ModelResponse | None,
503508
processed_response: ProcessedResponse | None,
@@ -562,7 +567,7 @@ async def _finalize_streamed_final_output(
562567
)
563568
if retained_items:
564569
try:
565-
await save_items(retained_items, response_id, store_setting)
570+
await save_items(retained_items, response_id, store_setting, False)
566571
except BaseException as persistence_error:
567572
safe_error = _safe_redacted_persistence_error(persistence_error)
568573
if (
@@ -587,7 +592,7 @@ async def _finalize_streamed_final_output(
587592
agent,
588593
run_config,
589594
)
590-
await save_items(final_turn_items, response_id, store_setting)
595+
await save_items(final_turn_items, response_id, store_setting, False)
591596
except BaseException as persistence_error:
592597
if guardrail_error_is_redacted:
593598
safe_persistence_error = _safe_redacted_persistence_error(persistence_error)
@@ -619,9 +624,12 @@ async def _finalize_streamed_final_output(
619624
raise redacted_persistence_error from None
620625

621626
streamed_result.output_guardrail_results.extend(output_guardrail_results)
622-
if checkpointable_terminal_step(streamed_result._state) is not None:
623-
# These passed for this exact output. Publish them before the append can raise so a
624-
# settled retry reports the original evidence instead of evaluating a new final output.
627+
if (
628+
terminal_checkpoint_owner(streamed_result._state, session, settle_terminal_output=True)
629+
is not None
630+
):
631+
# Every output guardrail passed for this exact output. Publish them before the append can
632+
# raise so a settled retry reports the original evidence instead of evaluating a new one.
625633
streamed_result._state._output_guardrail_results = list(
626634
streamed_result.output_guardrail_results
627635
)
@@ -636,11 +644,12 @@ async def _finalize_streamed_final_output(
636644
# Saved as one ordered batch so the session mirrors the model response. Doing it in two
637645
# halves would both reorder the turn and, because the first save advances the turn's
638646
# persisted-item count, make the second one a no-op.
647+
# Reached only after every output guardrail succeeded.
639648
if on_persisted_after_guardrails is None:
640-
await save_items(final_turn_items, response_id, store_setting)
649+
await save_items(final_turn_items, response_id, store_setting, True)
641650
else:
642651
try:
643-
await save_items(final_turn_items, response_id, store_setting)
652+
await save_items(final_turn_items, response_id, store_setting, True)
644653
except asyncio.CancelledError as persistence_error:
645654
if streamed_result._cancel_mode == "immediate":
646655
raise
@@ -1126,7 +1135,10 @@ def _mark_response_hooks_started() -> None:
11261135
streamed_result._original_input_for_persistence = session_items_snapshot
11271136

11281137
async def _save_resumed_items(
1129-
items: list[RunItem], response_id: str | None, store_setting: bool | None
1138+
items: list[RunItem],
1139+
response_id: str | None,
1140+
store_setting: bool | None,
1141+
settle_terminal_output: bool = False,
11301142
) -> None:
11311143
await _save_resumed_stream_items(
11321144
session=session,
@@ -1136,11 +1148,18 @@ async def _save_resumed_items(
11361148
items=items,
11371149
response_id=response_id,
11381150
store=store_setting,
1151+
settle_terminal_output=settle_terminal_output,
11391152
)
11401153

11411154
async def _save_stream_items_with_count(
1142-
items: list[RunItem], response_id: str | None, store_setting: bool | None
1155+
items: list[RunItem],
1156+
response_id: str | None,
1157+
store_setting: bool | None,
1158+
settle_terminal_output: bool = False,
11431159
) -> None:
1160+
# The main loop never records a terminal step on the state, so there is no terminal
1161+
# append for a later resume to own.
1162+
del settle_terminal_output
11441163
await _save_stream_items(
11451164
session=session,
11461165
server_conversation_tracker=server_conversation_tracker,
@@ -1167,8 +1186,13 @@ async def _save_stream_items_without_count(
11671186
)
11681187

11691188
async def _save_max_turns_items(
1170-
items: list[RunItem], response_id: str | None, store_setting: bool | None
1189+
items: list[RunItem],
1190+
response_id: str | None,
1191+
store_setting: bool | None,
1192+
settle_terminal_output: bool = False,
11711193
) -> None:
1194+
# A max-turns handler output is not a resumed terminal append.
1195+
del settle_terminal_output
11721196
if not await _should_persist_stream_items(
11731197
session=session,
11741198
server_conversation_tracker=server_conversation_tracker,
@@ -1457,6 +1481,7 @@ async def _save_max_turns_items(
14571481
output=turn_result.next_step.output,
14581482
context_wrapper=context_wrapper,
14591483
save_items=_save_resumed_items,
1484+
session=session,
14601485
items=list(turn_session_items),
14611486
model_response=turn_result.model_response,
14621487
processed_response=(
@@ -1655,6 +1680,7 @@ def _record_max_turns_handler_output(
16551680
output=validated_output,
16561681
context_wrapper=context_wrapper,
16571682
save_items=_save_max_turns_items,
1683+
session=session,
16581684
items=[synthesized_item] if include_in_history else [],
16591685
model_response=None,
16601686
processed_response=None,
@@ -1920,6 +1946,7 @@ def _record_max_turns_handler_output(
19201946
output=turn_result.next_step.output,
19211947
context_wrapper=context_wrapper,
19221948
save_items=_save_stream_items_with_count,
1949+
session=session,
19231950
items=turn_session_items,
19241951
model_response=turn_result.model_response,
19251952
processed_response=turn_result.processed_response,

‎src/agents/run_internal/session_persistence.py‎

Lines changed: 61 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,8 @@
8181
"save_result_to_session",
8282
"save_resumed_turn_items",
8383
"resume_pending_session_write",
84-
"checkpointable_terminal_step",
84+
"resumed_write_owner",
85+
"terminal_checkpoint_owner",
8586
"recoverable_terminal_step",
8687
"update_run_state_after_resume",
8788
"rewind_session_items",
@@ -722,32 +723,71 @@ async def save_result_to_session(
722723
return saved_run_items_count
723724

724725

725-
def checkpointable_terminal_step(
726+
def terminal_checkpoint_owner(
726727
run_state: RunState[Any, Any] | None,
727-
) -> NextStepFinalOutput | None:
728-
"""Return the accepted terminal step whose deferred append a resumed state may own.
729-
730-
Only a plain string output round-trips through the RunState codec unchanged: richer values
731-
fall back to ``json.dumps(..., default=str)`` and would change type on the way back, so they
732-
keep the existing non-resumable behavior instead.
728+
session: Session | None,
729+
*,
730+
settle_terminal_output: bool,
731+
) -> RunState[Any, Any] | None:
732+
"""Return the state that may own a deferred terminal append.
733+
734+
A later resume settles this batch by replaying the append alone, so the checkpoint may only
735+
be armed when the append really is all that is left of the run:
736+
737+
- ``settle_terminal_output`` says every output guardrail completed. Only the successful
738+
finalize path can state that, so callers opt in instead of it being inferred from the step:
739+
a guardrail that raised must surface on the next resume rather than be settled away.
740+
- the accepted output is a plain string. Richer values fall back to
741+
``json.dumps(..., default=str)`` and would change type across the codec, so they keep the
742+
existing non-resumable behavior.
743+
- the Session owes no post-append maintenance. A compaction-aware Session also records a
744+
deferred compaction for the batch, which the checkpoint does not carry, so settling one
745+
would drop maintenance the first attempt would have performed.
733746
"""
734-
step = run_state._current_step if run_state is not None else None
747+
if not settle_terminal_output or run_state is None:
748+
return None
749+
step = run_state._current_step
735750
if not isinstance(step, NextStepFinalOutput) or type(step.output) is not str:
736751
return None
737-
return step
752+
if is_openai_responses_compaction_aware_session(session):
753+
return None
754+
return run_state
738755

739756

740757
def recoverable_terminal_step(run_state: RunState[Any, Any] | None) -> NextStepFinalOutput | None:
741758
"""Return the accepted terminal step a resumed run may settle without another model call.
742759
743-
A checkpointable step is only settleable while its append is still outstanding. The pending
744-
write is that record: once it settles, later persistence work for the same batch, such as
745-
compaction, can still fail, and those failures must not be reported as a completed run.
746-
Reconciling the pending write clears it, so callers capture this before settling it.
760+
The step is only settleable while the append it is waiting on is still outstanding. The
761+
pending write is that record, and it is only armed by ``terminal_checkpoint_owner``, so its
762+
presence is what proves the append was all that remained. Reconciling clears it, so callers
763+
capture this before settling it.
747764
"""
748765
if run_state is None or run_state._pending_session_write is None:
749766
return None
750-
return checkpointable_terminal_step(run_state)
767+
step = run_state._current_step
768+
if not isinstance(step, NextStepFinalOutput) or type(step.output) is not str:
769+
return None
770+
return step
771+
772+
773+
def resumed_write_owner(
774+
run_state: RunState[Any, Any] | None,
775+
session: Session | None,
776+
*,
777+
settle_terminal_output: bool = False,
778+
) -> RunState[Any, Any] | None:
779+
"""Return the state that owns a resumed write, so a failed append settles on the next resume.
780+
781+
A run-again or interruption step always owns its write. A terminal step only does under the
782+
stricter rule in ``terminal_checkpoint_owner``, because settling one ends the run.
783+
"""
784+
if run_state is None:
785+
return None
786+
if isinstance(run_state._current_step, NextStepRunAgain | NextStepInterruption):
787+
return run_state
788+
return terminal_checkpoint_owner(
789+
run_state, session, settle_terminal_output=settle_terminal_output
790+
)
751791

752792

753793
async def save_resumed_turn_items(
@@ -761,7 +801,11 @@ async def save_resumed_turn_items(
761801
wrapper: RunContextWrapper[Any] | None = None,
762802
run_state: RunState | None = None,
763803
) -> int:
764-
"""Persist resumed turn items and return the updated persisted count."""
804+
"""Persist resumed turn items and return the updated persisted count.
805+
806+
``run_state`` is the state that owns this write, as resolved by ``resumed_write_owner``,
807+
or ``None`` when the batch is not checkpointed.
808+
"""
765809
if session is None or not items:
766810
return persisted_count
767811
saved_count = await save_result_to_session(
@@ -773,15 +817,7 @@ async def save_resumed_turn_items(
773817
reasoning_item_id_policy=reasoning_item_id_policy,
774818
store=store,
775819
wrapper=wrapper,
776-
resumed_write_state=(
777-
run_state
778-
if run_state is not None
779-
and (
780-
isinstance(run_state._current_step, NextStepRunAgain | NextStepInterruption)
781-
or checkpointable_terminal_step(run_state) is not None
782-
)
783-
else None
784-
),
820+
resumed_write_state=run_state,
785821
)
786822
return persisted_count + saved_count
787823

0 commit comments

Comments
 (0)