Skip to content

Commit 1705dd6

Browse files
fix(core): deliver tool-not-found output on server-managed resume (#4947)
1 parent 83c737f commit 1705dd6

3 files changed

Lines changed: 259 additions & 8 deletions

File tree

‎src/agents/run_internal/agent_runner_helpers.py‎

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -205,19 +205,24 @@ def _extract_tool_call_id(raw: Any) -> str | None:
205205
return candidate if isinstance(candidate, str) else None
206206

207207

208+
def _latest_response_tool_call_ids(run_state: RunState[Any]) -> set[str]:
209+
"""Return the call or item identifiers carried by the most recent model response."""
210+
if not run_state._model_responses:
211+
return set()
212+
return {
213+
call_id
214+
for item in run_state._model_responses[-1].output
215+
if (call_id := _extract_tool_call_id(item)) is not None
216+
}
217+
218+
208219
def get_unsent_tool_call_ids_for_interrupted_state(run_state: RunState[Any] | None) -> set[str]:
209220
"""Return tool call IDs whose local outputs have not reached a server conversation."""
210221
if run_state is None:
211222
return set()
212223

213224
if isinstance(run_state._current_step, NextStepRunAgain):
214-
if not run_state._model_responses:
215-
return set()
216-
return {
217-
call_id
218-
for item in run_state._model_responses[-1].output
219-
if (call_id := _extract_tool_call_id(item)) is not None
220-
}
225+
return _latest_response_tool_call_ids(run_state)
221226

222227
if not isinstance(run_state._current_step, NextStepInterruption):
223228
return set()
@@ -226,7 +231,14 @@ def get_unsent_tool_call_ids_for_interrupted_state(run_state: RunState[Any] | No
226231
if processed_response is None:
227232
return set()
228233

229-
tool_call_ids: set[str] = set()
234+
# An interrupted turn stops before its next model request, so every tool output it built
235+
# locally is still unsent. Some of those outputs have no pending tool run left to enumerate
236+
# below, such as the error synthesized for a call to a tool that does not exist, and that
237+
# classification is not part of the serialized `RunState`. The latest model response is the
238+
# source of truth that survives both cases. Seeding from it stays safe because
239+
# `hydrate_from_state` records everything that response reported as server-owned before it
240+
# consults this set, so a server-owned item is never resent.
241+
tool_call_ids = _latest_response_tool_call_ids(run_state)
230242
tool_run_groups = (
231243
processed_response.handoffs,
232244
processed_response.functions,

‎tests/test_run_impl_resume_paths.py‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -918,6 +918,149 @@ async def fake_resolve_interrupted_turn(**kwargs: object) -> SingleStepResult:
918918
assert server_managed_values == [True]
919919

920920

921+
def _sent_tool_outputs(model: ScriptedModel, *, first_call_index: int) -> list[tuple[str, str]]:
922+
"""Collect the tool outputs the model received, from `first_call_index` onward."""
923+
outputs: list[tuple[str, str]] = []
924+
for call in model.calls[first_call_index:]:
925+
for item in cast(list[dict[str, Any]], call.input):
926+
if item.get("type") == "function_call_output":
927+
outputs.append((str(item.get("call_id")), str(item.get("output"))))
928+
return outputs
929+
930+
931+
async def _run_server_managed(
932+
agent: Agent[Any],
933+
agent_input: Any,
934+
*,
935+
run_config: RunConfig,
936+
use_conversation_id: bool,
937+
streaming: bool,
938+
) -> Any:
939+
"""Run the agent under one of the server-managed continuation modes."""
940+
kwargs: dict[str, Any] = (
941+
{"conversation_id": "conv-resume"}
942+
if use_conversation_id
943+
else {"auto_previous_response_id": True}
944+
)
945+
if streaming:
946+
streamed = Runner.run_streamed(agent, agent_input, run_config=run_config, **kwargs)
947+
async for _ in streamed.stream_events():
948+
pass
949+
return streamed
950+
return await Runner.run(agent, agent_input, run_config=run_config, **kwargs)
951+
952+
953+
@pytest.mark.parametrize("streaming", [False, True], ids=["non_streamed", "streamed"])
954+
@pytest.mark.parametrize("serialize_state", [False, True], ids=["live_state", "serialized_state"])
955+
@pytest.mark.parametrize(
956+
"use_conversation_id", [False, True], ids=["auto_previous_response_id", "conversation_id"]
957+
)
958+
@pytest.mark.asyncio
959+
async def test_resumed_server_managed_run_sends_tool_not_found_output(
960+
streaming: bool,
961+
serialize_state: bool,
962+
use_conversation_id: bool,
963+
) -> None:
964+
"""A resumed server-managed run must send the output built for a missing tool.
965+
966+
The interrupted turn answers the unknown tool locally while another call waits for
967+
approval. The server already owns both calls, so resuming has to deliver both outputs.
968+
"""
969+
970+
@function_tool(name_override="needs_ok", needs_approval=True)
971+
async def needs_ok(text: str) -> str:
972+
return text
973+
974+
model = ScriptedModel()
975+
agent = Agent(name="test", model=model, tools=[needs_ok])
976+
model.extend(
977+
[
978+
[
979+
get_function_tool_call(
980+
"needs_ok", json.dumps({"text": "one"}), call_id="call-approval"
981+
),
982+
get_function_tool_call("missing_tool", json.dumps({}), call_id="call-missing"),
983+
],
984+
[get_text_message("done")],
985+
]
986+
)
987+
run_config = RunConfig(tool_not_found_behavior="return_error_to_model")
988+
989+
async def run_once(agent_input: Any) -> Any:
990+
return await _run_server_managed(
991+
agent,
992+
agent_input,
993+
run_config=run_config,
994+
use_conversation_id=use_conversation_id,
995+
streaming=streaming,
996+
)
997+
998+
first = await run_once("Use needs_ok and missing_tool")
999+
state = first.to_state()
1000+
if serialize_state:
1001+
state = await RunState.from_json(agent, json.loads(json.dumps(state.to_json())))
1002+
interruptions = state.get_interruptions()
1003+
assert [item.raw_item.call_id for item in interruptions] == ["call-approval"]
1004+
state.approve(interruptions[0])
1005+
1006+
resumed = await run_once(state)
1007+
1008+
assert resumed.final_output == "done"
1009+
delivered = _sent_tool_outputs(model, first_call_index=1)
1010+
assert sorted(call_id for call_id, _ in delivered) == ["call-approval", "call-missing"]
1011+
assert "missing_tool" in dict(delivered)["call-missing"]
1012+
1013+
1014+
@pytest.mark.asyncio
1015+
async def test_resumed_server_managed_run_sends_each_tool_output_once() -> None:
1016+
"""Staged approvals must deliver every output exactly once to a server-managed conversation.
1017+
1018+
Approving one of two gated calls resumes and interrupts again without a model request, so
1019+
the same model response stays current across both resumes.
1020+
"""
1021+
1022+
@function_tool(name_override="needs_ok", needs_approval=True)
1023+
async def needs_ok(text: str) -> str:
1024+
return f"ok:{text}"
1025+
1026+
model = ScriptedModel()
1027+
agent = Agent(name="test", model=model, tools=[needs_ok])
1028+
model.extend(
1029+
[
1030+
[
1031+
get_function_tool_call("needs_ok", json.dumps({"text": "a"}), call_id="call-a"),
1032+
get_function_tool_call("needs_ok", json.dumps({"text": "b"}), call_id="call-b"),
1033+
get_function_tool_call("missing_tool", json.dumps({}), call_id="call-missing"),
1034+
],
1035+
[get_text_message("done")],
1036+
]
1037+
)
1038+
run_config = RunConfig(tool_not_found_behavior="return_error_to_model")
1039+
1040+
async def run_once(agent_input: Any) -> Any:
1041+
return await _run_server_managed(
1042+
agent,
1043+
agent_input,
1044+
run_config=run_config,
1045+
use_conversation_id=False,
1046+
streaming=False,
1047+
)
1048+
1049+
result = await run_once("Use needs_ok twice and missing_tool")
1050+
for expected_model_calls in (1, 2):
1051+
state = await RunState.from_json(agent, json.loads(json.dumps(result.to_state().to_json())))
1052+
interruptions = state.get_interruptions()
1053+
assert interruptions
1054+
state.approve(interruptions[0])
1055+
result = await run_once(state)
1056+
# Approving only the first gated call resumes without asking the model again.
1057+
assert len(model.calls) == expected_model_calls
1058+
1059+
assert result.final_output == "done"
1060+
delivered = _sent_tool_outputs(model, first_call_index=1)
1061+
assert sorted(call_id for call_id, _ in delivered) == ["call-a", "call-b", "call-missing"]
1062+
1063+
9211064
@pytest.mark.asyncio
9221065
async def test_resumed_approval_does_not_duplicate_session_items() -> None:
9231066
async def test_tool() -> str:

‎tests/test_server_conversation_tracker.py‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,7 @@ def test_hydrate_from_state_preserves_unsent_outputs_from_interrupted_turn() ->
149149
]
150150
interrupted_state = SimpleNamespace(
151151
_current_step=NextStepInterruption(interruptions=[]),
152+
_model_responses=[model_response],
152153
_last_processed_response=SimpleNamespace(
153154
handoffs=[],
154155
functions=[
@@ -211,6 +212,101 @@ def test_hydrate_from_state_preserves_unsent_outputs_from_interrupted_turn() ->
211212
] == ["call_DIAG", "call_CLEANUP1", "call_CLEANUP2"]
212213

213214

215+
def test_hydrate_from_state_preserves_unsent_output_without_a_pending_tool_run() -> None:
216+
"""An interrupted turn's local output is unsent even with no pending tool run to enumerate.
217+
218+
A call to a tool that does not exist is answered locally and leaves no pending tool run in
219+
the resumed `ProcessedResponse`, so the latest model response is what identifies its output
220+
as still unsent.
221+
"""
222+
agent = Agent(name="test")
223+
approval_call = ResponseFunctionToolCall(
224+
id="fc_101",
225+
type="function_call",
226+
call_id="call_APPROVAL",
227+
name="run_cleanup",
228+
arguments='{"target": "temp_files"}',
229+
status="completed",
230+
)
231+
missing_tool_call = ResponseFunctionToolCall(
232+
id="fc_102",
233+
type="function_call",
234+
call_id="call_MISSING",
235+
name="not_a_registered_tool",
236+
arguments="{}",
237+
status="completed",
238+
)
239+
model_response = ModelResponse(
240+
output=[approval_call, missing_tool_call],
241+
usage=Usage(),
242+
response_id="resp_101",
243+
)
244+
missing_tool_output = ToolCallOutputItem(
245+
agent=agent,
246+
raw_item={
247+
"type": "function_call_output",
248+
"call_id": "call_MISSING",
249+
"output": "Tool not_a_registered_tool not found.",
250+
},
251+
output="Tool not_a_registered_tool not found.",
252+
)
253+
generated_items: list[RunItem] = [
254+
ToolCallItem(agent=agent, raw_item=approval_call),
255+
ToolCallItem(agent=agent, raw_item=missing_tool_call),
256+
ToolApprovalItem(agent=agent, raw_item=approval_call, tool_name="run_cleanup"),
257+
missing_tool_output,
258+
]
259+
interrupted_state = SimpleNamespace(
260+
_current_step=NextStepInterruption(interruptions=[]),
261+
_model_responses=[model_response],
262+
_last_processed_response=SimpleNamespace(
263+
handoffs=[],
264+
functions=[SimpleNamespace(tool_call=approval_call)],
265+
computer_actions=[],
266+
custom_tool_calls=[],
267+
local_shell_calls=[],
268+
shell_calls=[],
269+
apply_patch_calls=[],
270+
),
271+
)
272+
273+
tracker = OpenAIServerConversationTracker(previous_response_id="resp_101")
274+
tracker.hydrate_from_state(
275+
original_input="Clean up and call the missing tool.",
276+
generated_items=generated_items,
277+
model_responses=[model_response],
278+
unsent_tool_call_ids=get_unsent_tool_call_ids_for_interrupted_state(
279+
cast(Any, interrupted_state)
280+
),
281+
)
282+
283+
assert "call_MISSING" not in tracker.server_tool_call_ids
284+
285+
prepared = tracker.prepare_input(
286+
"Clean up and call the missing tool.",
287+
[
288+
ToolCallItem(agent=agent, raw_item=approval_call),
289+
ToolCallItem(agent=agent, raw_item=missing_tool_call),
290+
missing_tool_output,
291+
ToolCallOutputItem(
292+
agent=agent,
293+
raw_item={
294+
"type": "function_call_output",
295+
"call_id": "call_APPROVAL",
296+
"output": "Cleanup finished.",
297+
},
298+
output="Cleanup finished.",
299+
),
300+
],
301+
)
302+
303+
assert [
304+
item.get("call_id")
305+
for item in prepared
306+
if isinstance(item, dict) and item.get("type") == "function_call_output"
307+
] == ["call_MISSING", "call_APPROVAL"]
308+
309+
214310
def test_hydrate_from_state_does_not_track_string_initial_input_by_object_identity() -> None:
215311
tracker = OpenAIServerConversationTracker(
216312
conversation_id="conv-init-string", previous_response_id=None

0 commit comments

Comments
 (0)