Skip to content

Commit 1c1a28d

Browse files
fix: preserve completed tool outputs when a sibling fails (#5332)
* fix: preserve completed tool outputs when a sibling fails * fix: preserve resumed state and diagnostics after tool batch failures * fix: preserve completed tool history across failure lifecycles * fix: finish tool failure cleanup and preserve hosted results * fix: retain completed tools through finalization and input admission * fix: preserve tool cancellation across Python 3.10 tasks * fix: retain replay ownership in partial tool history * fix: preserve accepted outputs while metadata is pending * fix: preserve tool failure reporting and guarded recovery * fix: freeze recovery values and distinguish child cancellation * fix: cancel irrelevant guardrails after tool failure * fix: preserve live tool output types in failure snapshots * fix: preserve cyclic tool values during failure snapshots
1 parent 5eb47a5 commit 1c1a28d

16 files changed

Lines changed: 3118 additions & 313 deletions

‎src/agents/exceptions.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929

3030
from .util._pretty_print import pretty_print_run_error_details
3131

32+
_TOOL_LOCAL_CANCELLATION_ATTR = "_agents_tool_local_cancellation"
3233
_DRAIN_STREAM_EVENTS_ATTR = "_agents_drain_queued_stream_events"
3334
_DATA_REDACTED_ATTR = "_agents_data_redacted"
3435
_DATA_REDACTED_ERROR_MESSAGE = "Error details are redacted."
@@ -41,6 +42,16 @@ class _RedactedExceptionCancellationError(asyncio.CancelledError, Exception):
4142
"""Payload-free cancellation that remains catchable as an Exception."""
4243

4344

45+
def _mark_tool_local_cancellation(error: asyncio.CancelledError) -> None:
46+
setattr(error, _TOOL_LOCAL_CANCELLATION_ATTR, True)
47+
48+
49+
def _is_tool_local_cancellation(error: BaseException) -> bool:
50+
return isinstance(error, asyncio.CancelledError) and bool(
51+
getattr(error, _TOOL_LOCAL_CANCELLATION_ATTR, False)
52+
)
53+
54+
4455
def _mark_error_to_drain_stream_events(error: BaseException) -> None:
4556
setattr(error, _DRAIN_STREAM_EVENTS_ATTR, True)
4657

‎src/agents/items.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -448,7 +448,7 @@ class ToolCallOutputItem(RunItemBase[Any]):
448448
"""SDK-only custom data attached to this tool output.
449449
450450
This data is not part of ``raw_item`` and is not sent back to the model when the output item is
451-
replayed as input.
451+
replayed as input. On a failed run, unfinished custom-data extraction may leave this unset.
452452
"""
453453

454454
@property

‎src/agents/result.py‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,11 @@ def _populate_state_from_result(
153153
state._pending_input = copy.deepcopy(source_state._pending_input)
154154
state._pending_session_write = copy.deepcopy(source_state._pending_session_write)
155155
state._current_step = source_state._current_step
156+
if isinstance(source_state._current_step, NextStepInterruption):
157+
# Failure diagnostics may expose accepted outputs still awaiting their
158+
# normal resume write. Keep the checkpoint's publication boundary so
159+
# recovery does not mistake those outputs for already persisted work.
160+
state._session_items = list(source_state._session_items)
156161
# A streamed result exists before its terminal append does, so a checkpoint taken from a
157162
# failed stream has to keep the fail-closed marker or it would look resumable.
158163
state._terminal_unrecoverable = source_state._terminal_unrecoverable
@@ -704,6 +709,8 @@ class RunResultStreaming(RunResultBase):
704709
_triggered_input_guardrail_result: InputGuardrailResult | None = field(default=None, repr=False)
705710
_output_guardrails_task: asyncio.Task[Any] | None = field(default=None, repr=False)
706711
_stored_exception: BaseException | None = field(default=None, repr=False)
712+
_tool_error_selected: bool = field(default=False, init=False, repr=False)
713+
"""A selected tool error owns failure reporting while its input verdict settles."""
707714
_cancel_mode: Literal["none", "immediate", "after_turn"] = field(default="none", repr=False)
708715
_last_processed_response: ProcessedResponse | None = field(default=None, repr=False)
709716
"""The last processed model response. This is needed for resuming from interruptions."""
@@ -1176,7 +1183,7 @@ def _check_errors(self):
11761183
# Fetch all the completed guardrail results from the queue and raise if needed
11771184
while not self._input_guardrail_queue.empty():
11781185
guardrail_result = self._input_guardrail_queue.get_nowait()
1179-
if guardrail_result.output.tripwire_triggered:
1186+
if guardrail_result.output.tripwire_triggered and not self._tool_error_selected:
11801187
tripwire_exc = InputGuardrailTripwireTriggered(guardrail_result)
11811188
tripwire_exc.run_data = self._create_error_details()
11821189
self._stored_exception = tripwire_exc
@@ -1194,7 +1201,11 @@ def _check_errors(self):
11941201
run_impl_exc.run_data = self._create_error_details()
11951202
self._stored_exception = run_impl_exc
11961203

1197-
if self._input_guardrails_task and self._input_guardrails_task.done():
1204+
if (
1205+
not self._tool_error_selected
1206+
and self._input_guardrails_task
1207+
and self._input_guardrails_task.done()
1208+
):
11981209
if not self._input_guardrails_task.cancelled():
11991210
in_guard_exc = self._input_guardrails_task.exception()
12001211
if isinstance(in_guard_exc, Exception):

‎src/agents/run.py‎

Lines changed: 147 additions & 40 deletions
Large diffs are not rendered by default.

‎src/agents/run_internal/guardrails.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ async def run_input_guardrails_with_queue(
105105
isinstance(error, Exception)
106106
and asyncio.current_task() is streamed_result._input_guardrails_task
107107
and not streamed_result.is_complete
108+
and not streamed_result._tool_error_selected
108109
):
109110
if streamed_result.run_loop_task and not streamed_result.run_loop_task.done():
110111
streamed_result.run_loop_task.cancel()

‎src/agents/run_internal/run_loop.py‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
_detach_data_redacted_error_traceback,
4444
_detach_source_exception_traceback,
4545
_is_error_data_redacted,
46+
_is_tool_local_cancellation,
4647
_mark_error_data_redacted,
4748
_mark_error_to_drain_stream_events,
4849
)
@@ -188,6 +189,7 @@
188189
reconcile_nested_history_owned_session_item_refs,
189190
record_compaction_model_response,
190191
resume_pending_session_write,
192+
resumed_failure_items,
191193
resumed_response_store,
192194
resumed_turn_items,
193195
rewind_session_items,
@@ -965,6 +967,23 @@ async def _finalize_streamed_interruption(
965967
T = TypeVar("T")
966968

967969

970+
class _ToolTaskCancellation(Exception):
971+
"""Carry selected tool cancellation across Task on Python versions that replace it."""
972+
973+
def __init__(self, error: asyncio.CancelledError) -> None:
974+
super().__init__()
975+
self.error = error
976+
977+
978+
async def preserve_tool_task_cancellation(awaitable: Awaitable[T]) -> T:
979+
try:
980+
return await awaitable
981+
except asyncio.CancelledError as error:
982+
if _is_tool_local_cancellation(error):
983+
raise _ToolTaskCancellation(error) from None
984+
raise
985+
986+
968987
async def start_streaming(
969988
starting_input: str | list[TResponseInputItem],
970989
streamed_result: RunResultStreaming,
@@ -1494,6 +1513,12 @@ async def _save_max_turns_items(
14941513
streamed_result._model_input_items != streamed_result.new_items
14951514
)
14961515
if not _is_error_data_redacted(exc):
1516+
streamed_result.new_items = resumed_failure_items(
1517+
run_state, current_agent, run_config
1518+
)
1519+
streamed_result._replay_from_model_input_items = (
1520+
streamed_result._model_input_items != streamed_result.new_items
1521+
)
14971522
streamed_result.tool_input_guardrail_results = list(
14981523
run_state._tool_input_guardrail_results
14991524
)
@@ -1961,6 +1986,7 @@ def _record_max_turns_handler_output(
19611986
)
19621987
if current_turn_span is not None:
19631988
current_turn_span.start(mark_as_current=True)
1989+
partial_tool_results: list[SingleStepResult] = []
19641990
try:
19651991
if (
19661992
session is not None
@@ -1990,7 +2016,93 @@ def _record_max_turns_handler_output(
19902016
on_response_accepted=_commit_pending_server_response,
19912017
on_response_hooks_started=_mark_response_hooks_started,
19922018
run_state=run_state,
2019+
on_tool_execution_error=partial_tool_results.append,
19932020
)
2021+
except (Exception, asyncio.CancelledError) as tool_error:
2022+
if not partial_tool_results:
2023+
raise
2024+
input_task = streamed_result._input_guardrails_task
2025+
if partial_tool_results and input_task is not None and not input_task.done():
2026+
# The finalizer already waits for this verdict. Settle it before
2027+
# publishing, without letting a late verdict replace the tool error.
2028+
streamed_result._tool_error_selected = True
2029+
try:
2030+
await input_guardrail_tripwire_triggered_for_stream(
2031+
streamed_result, ignore_cancelled=True
2032+
)
2033+
except Exception:
2034+
pass
2035+
input_accepted = input_task is None or (
2036+
input_task.done()
2037+
and not input_task.cancelled()
2038+
and input_task.exception() is None
2039+
)
2040+
if (
2041+
partial_tool_results
2042+
and input_accepted
2043+
and not any(
2044+
result.output.tripwire_triggered
2045+
for result in streamed_result.input_guardrail_results
2046+
)
2047+
):
2048+
partial_result = partial_tool_results[0]
2049+
streamed_result._model_input_items.extend(partial_result.new_step_items)
2050+
streamed_result.new_items.extend(partial_result.new_step_items)
2051+
streamed_result.raw_responses.append(partial_result.model_response)
2052+
_accumulate_tool_guardrail_results(
2053+
streamed_result,
2054+
partial_result,
2055+
accepted_input_results=accepted_tool_input_guardrail_results,
2056+
accepted_output_results=accepted_tool_output_guardrail_results,
2057+
)
2058+
streamed_result._tool_use_tracker_snapshot = serialize_tool_use_tracker(
2059+
tool_use_tracker,
2060+
starting_agent=(
2061+
run_state._starting_agent
2062+
if run_state is not None and run_state._starting_agent is not None
2063+
else starting_agent
2064+
),
2065+
)
2066+
if run_state is not None:
2067+
_synchronize_accepted_run_state(
2068+
run_state,
2069+
generated_items=streamed_result._model_input_items,
2070+
session_items=streamed_result.new_items,
2071+
model_responses=streamed_result.raw_responses,
2072+
tool_input_guardrail_results=(
2073+
streamed_result.tool_input_guardrail_results
2074+
),
2075+
tool_output_guardrail_results=(
2076+
streamed_result.tool_output_guardrail_results
2077+
),
2078+
current_turn=current_turn,
2079+
)
2080+
run_state._current_step = NextStepRunAgain()
2081+
run_state.set_tool_use_tracker_snapshot(
2082+
streamed_result._tool_use_tracker_snapshot
2083+
)
2084+
stream_step_items_to_queue(
2085+
[
2086+
item
2087+
for item in partial_result.new_step_items
2088+
if item.type == "tool_call_output_item"
2089+
# Model-provided outputs were already emitted before execution.
2090+
and _stream_event_item_occurrence_key(item) is None
2091+
],
2092+
streamed_result._event_queue,
2093+
)
2094+
_mark_error_to_drain_stream_events(tool_error)
2095+
try:
2096+
await _save_stream_items_without_count(
2097+
partial_result.new_step_items,
2098+
partial_result.model_response.response_id,
2099+
current_agent.model_settings.resolve(
2100+
run_config.model_settings
2101+
).store,
2102+
)
2103+
except Exception:
2104+
logger.warning("Failed to save completed tools after a tool error")
2105+
raise
19942106
finally:
19952107
if current_turn_span is not None:
19962108
attach_usage_to_span(
@@ -2255,6 +2367,10 @@ def _record_max_turns_handler_output(
22552367
except BaseException as error:
22562368
if _is_error_data_redacted(error):
22572369
_detach_data_redacted_error_traceback(error)
2370+
if _is_tool_local_cancellation(error) and streamed_result._tool_error_selected:
2371+
# Task.exception() cannot report cancellation. Keep the selected
2372+
# tool failure observable instead of ending the stream successfully.
2373+
streamed_result._stored_exception = error
22582374
raise
22592375
else:
22602376
streamed_result.is_complete = True
@@ -2322,6 +2438,7 @@ async def run_single_turn_streamed(
23222438
on_response_accepted: Callable[[ModelResponse, ProcessedResponse | None], bool] | None = None,
23232439
on_response_hooks_started: Callable[[], None] | None = None,
23242440
run_state: RunState[Any] | None = None,
2441+
on_tool_execution_error: Callable[[SingleStepResult], None] | None = None,
23252442
) -> SingleStepResult:
23262443
"""Run a single streamed turn and emit events as results arrive."""
23272444
public_agent = bindings.public_agent
@@ -2629,6 +2746,10 @@ async def after_invocation_validation(
26292746
async def check_input_guardrails_before_side_effects() -> None:
26302747
await raise_if_input_guardrail_tripwire_known()
26312748

2749+
def on_tool_error_selected() -> None:
2750+
# The category has selected its failure, before draining native finalization.
2751+
streamed_result._tool_error_selected = True
2752+
26322753
single_step_result = await get_single_step_result_from_response(
26332754
bindings=bindings,
26342755
original_input=streamed_result.input,
@@ -2646,6 +2767,8 @@ async def check_input_guardrails_before_side_effects() -> None:
26462767
after_invocation_validation=after_invocation_validation,
26472768
before_side_effects=check_input_guardrails_before_side_effects,
26482769
run_state=run_state,
2770+
on_tool_execution_error=on_tool_execution_error,
2771+
on_tool_error_selected=on_tool_error_selected,
26492772
)
26502773

26512774
items_to_filter = session_items_for_turn(single_step_result)
@@ -2681,6 +2804,8 @@ async def run_single_turn(
26812804
on_response_accepted: Callable[[ModelResponse, ProcessedResponse | None], bool] | None = None,
26822805
on_response_hooks_started: Callable[[], None] | None = None,
26832806
run_state: RunState[Any] | None = None,
2807+
on_tool_execution_error: Callable[[SingleStepResult], None] | None = None,
2808+
on_tool_error_selected: Callable[[], None] | None = None,
26842809
) -> SingleStepResult:
26852810
"""Run a single non-streaming turn of the agent loop."""
26862811
public_agent = bindings.public_agent
@@ -2793,6 +2918,8 @@ async def after_invocation_validation(
27932918
server_manages_conversation=server_conversation_tracker is not None,
27942919
after_invocation_validation=after_invocation_validation,
27952920
run_state=run_state,
2921+
on_tool_execution_error=on_tool_execution_error,
2922+
on_tool_error_selected=on_tool_error_selected,
27962923
)
27972924

27982925

‎src/agents/run_internal/session_persistence.py‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616

1717
from .. import _debug
1818
from .._tool_identity import get_hosted_mcp_approval_request_identity
19+
from .._tool_invocation import tool_output_identity
20+
from ..agent import Agent
1921
from ..exceptions import UserError
2022
from ..items import (
2123
HandoffOutputItem,
@@ -45,9 +47,14 @@
4547
)
4648
from ..memory.session import _call_session_method, _get_session_wrapper
4749
from ..models.fake_id import FAKE_RESPONSES_ID
50+
from ..run_config import RunConfig
4851
from ..run_context import RunContextWrapper
4952
from ..run_state import RunState, _PendingSessionWrite
50-
from .blocked_output import _current_response_boundary, _identity_sequence_start
53+
from .blocked_output import (
54+
_current_response_boundary,
55+
_identity_sequence_start,
56+
_should_defer_interrupted_session_items,
57+
)
5158
from .items import (
5259
_TOOL_CALL_TO_OUTPUT_TYPE,
5360
NestedHistoryOwnedItem,
@@ -625,6 +632,31 @@ def session_items_for_turn(turn_result: SingleStepResult) -> list[RunItem]:
625632
return list(items)
626633

627634

635+
def resumed_failure_items(
636+
run_state: RunState, agent: Agent[Any], run_config: RunConfig
637+
) -> list[RunItem]:
638+
"""Expose accepted checkpoint outputs without marking them as already persisted."""
639+
public_items = list(run_state._session_items)
640+
if _should_defer_interrupted_session_items(agent, run_config):
641+
return public_items
642+
boundary = _current_response_boundary((), run_state._last_processed_response, run_state)
643+
if not boundary.proven or boundary.generated_start is None:
644+
return public_items
645+
published_outputs = {
646+
tool_output_identity(item.raw_item)
647+
for item in public_items
648+
if isinstance(item, ToolCallOutputItem)
649+
}
650+
return public_items + [
651+
item
652+
for item in run_state._generated_items[
653+
boundary.generated_start + len(boundary.processed_items) :
654+
]
655+
if isinstance(item, ToolCallOutputItem)
656+
and tool_output_identity(item.raw_item) not in published_outputs
657+
]
658+
659+
628660
def resumed_turn_items(
629661
turn_result: SingleStepResult, session_items: list[RunItem], run_state: RunState
630662
) -> tuple[list[RunItem], list[RunItem], list[RunItem]]:

0 commit comments

Comments
 (0)