Skip to content

Commit f3a15f6

Browse files
authored
fix(core): preserve guarded final outputs in agent tools (#5115)
1 parent 40956e0 commit f3a15f6

3 files changed

Lines changed: 190 additions & 1 deletion

File tree

‎src/agents/agent.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -621,7 +621,9 @@ def as_tool(
621621
tool_description: The description of the tool, which should indicate what it does and
622622
when to use it.
623623
custom_output_extractor: A function that extracts the output from the agent. If not
624-
provided, the last message from the agent will be used. Nested run results expose
624+
provided, the final output is used. When output guardrails ran, empty final
625+
outputs are preserved; otherwise, empty final outputs fall back to the latest
626+
nonempty message or tool output from the nested run. Nested run results expose
625627
`agent_tool_invocation` metadata when this agent is invoked via `as_tool()`.
626628
is_enabled: Whether the tool is enabled. Can be a bool or a callable that takes the run
627629
context and agent and returns whether the tool is enabled. Disabled tools are hidden
@@ -1053,6 +1055,10 @@ async def enqueue_stream_events() -> None:
10531055

10541056
from .items import ItemHelpers, MessageOutputItem, ToolCallOutputItem
10551057

1058+
# Output guardrails validated the final output, not intermediate run items.
1059+
if run_result.output_guardrail_results:
1060+
return run_result.final_output
1061+
10561062
for item in reversed(run_result.new_items):
10571063
if isinstance(item, MessageOutputItem):
10581064
text_output = ItemHelpers.text_message_output(item)

‎tests/test_agent_as_tool.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -645,6 +645,7 @@ async def test_agent_as_tool_fallback_uses_current_run_items_only(
645645
class DummyResult:
646646
def __init__(self) -> None:
647647
self.final_output = ""
648+
self.output_guardrail_results: list[Any] = []
648649
self.new_items = [
649650
ToolCallOutputItem(
650651
agent=agent,
@@ -738,6 +739,7 @@ async def test_agent_as_tool_fallback_returns_most_recent_current_run_output(
738739
class DummyResult:
739740
def __init__(self) -> None:
740741
self.final_output = ""
742+
self.output_guardrail_results: list[Any] = []
741743
self.new_items = [
742744
MessageOutputItem(agent=agent, raw_item=older_message),
743745
ToolCallOutputItem(
Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
from __future__ import annotations
2+
3+
from typing import Any
4+
5+
import pytest
6+
from openai.types.responses import ResponseOutputMessage, ResponseOutputRefusal
7+
8+
from agents import (
9+
Agent,
10+
GuardrailFunctionOutput,
11+
ModelRefusalError,
12+
RunConfig,
13+
RunContextWrapper,
14+
Runner,
15+
RunState,
16+
ToolCallOutputItem,
17+
)
18+
from agents.decorators import output_guardrail, tool
19+
from agents.testing import ScriptedModel
20+
from tests.test_responses import get_function_tool_call, get_text_message
21+
22+
23+
@pytest.mark.asyncio
24+
@pytest.mark.parametrize("streamed", [False, True])
25+
@pytest.mark.parametrize(
26+
("guardrail_owner", "fallback_source", "completion", "custom"),
27+
[
28+
("agent", "tool", "empty", False),
29+
("config", "message", "empty", False),
30+
("handoff", "tool", "empty", False),
31+
("agent", "tool", "none", False),
32+
("agent", "tool", "nonempty", False),
33+
("none", "tool", "empty", False),
34+
("none", "message", "empty", False),
35+
("agent", "tool", "refusal", False),
36+
("agent", "tool", "empty", True),
37+
],
38+
)
39+
async def test_default_agent_tool_exports_guarded_final_output(
40+
streamed: bool, guardrail_owner: str, fallback_source: str, completion: str, custom: bool
41+
) -> None:
42+
checked: list[tuple[str, Any]] = []
43+
44+
@output_guardrail
45+
def check_output(
46+
context: RunContextWrapper[Any], agent: Agent[Any], output: Any
47+
) -> GuardrailFunctionOutput:
48+
checked.append((agent.name, output))
49+
return GuardrailFunctionOutput(output_info=None, tripwire_triggered=output == "internal")
50+
51+
@tool
52+
def lookup() -> str:
53+
return "internal" if fallback_source == "tool" else ""
54+
55+
first_turn = [get_function_tool_call("lookup", "{}")]
56+
if fallback_source == "message":
57+
first_turn.insert(0, get_text_message("internal"))
58+
final_message = get_text_message("done" if completion == "nonempty" else "")
59+
if completion == "none":
60+
final_message = get_text_message('{"response":null}')
61+
if completion == "refusal":
62+
final_message = ResponseOutputMessage(
63+
id="refusal",
64+
type="message",
65+
role="assistant",
66+
status="completed",
67+
content=[ResponseOutputRefusal(type="refusal", refusal="Cannot answer")],
68+
)
69+
final_agent = Agent(
70+
name="worker",
71+
model=ScriptedModel([first_turn, [final_message]]),
72+
tools=[lookup],
73+
output_type=type(None) if completion == "none" else None,
74+
output_guardrails=[check_output] if guardrail_owner in ("agent", "handoff") else [],
75+
)
76+
nested = final_agent
77+
if guardrail_owner == "handoff":
78+
nested = Agent(
79+
name="delegator",
80+
handoffs=[final_agent],
81+
model=ScriptedModel(
82+
[[get_function_tool_call("transfer_to_worker", "{}", call_id="handoff")]]
83+
),
84+
)
85+
86+
async def extract(result: Any) -> str:
87+
return "custom output"
88+
89+
nested_tool = nested.as_tool(
90+
"delegate",
91+
"Delegate to a worker",
92+
on_stream=(lambda event: None) if streamed else None,
93+
run_config=RunConfig(output_guardrails=[check_output])
94+
if guardrail_owner == "config"
95+
else None,
96+
custom_output_extractor=extract if custom else None,
97+
failure_error_function=None,
98+
)
99+
parent_model = ScriptedModel(
100+
[
101+
[get_function_tool_call("delegate", '{"input":"work"}')],
102+
[get_text_message("parent done")],
103+
]
104+
)
105+
parent = Agent(name="parent", model=parent_model, tools=[nested_tool])
106+
if completion == "refusal":
107+
with pytest.raises(ModelRefusalError):
108+
await Runner.run(parent, "go")
109+
assert checked == []
110+
assert len(parent_model.calls) == 1
111+
return
112+
result = await Runner.run(parent, "go")
113+
expected_final = None if completion == "none" else "done" if completion == "nonempty" else ""
114+
expected_export = (
115+
"custom output" if custom else "internal" if guardrail_owner == "none" else expected_final
116+
)
117+
assert checked == ([] if guardrail_owner == "none" else [("worker", expected_final)])
118+
outputs = [item.output for item in result.new_items if isinstance(item, ToolCallOutputItem)]
119+
assert outputs == [expected_export]
120+
parent_input = parent_model.calls[1].input
121+
assert isinstance(parent_input, list)
122+
forwarded = [item for item in parent_input if item.get("type") == "function_call_output"]
123+
assert [item["output"] for item in forwarded] == [str(expected_export)]
124+
125+
126+
@pytest.mark.asyncio
127+
@pytest.mark.parametrize("streamed", [False, True])
128+
async def test_guarded_empty_agent_tool_output_after_approval_resume(streamed: bool) -> None:
129+
checked: list[Any] = []
130+
131+
@output_guardrail
132+
def check_output(
133+
context: RunContextWrapper[Any], agent: Agent[Any], output: Any
134+
) -> GuardrailFunctionOutput:
135+
checked.append(output)
136+
return GuardrailFunctionOutput(output_info=None, tripwire_triggered=output == "internal")
137+
138+
@tool(needs_approval=True)
139+
def lookup() -> str:
140+
return "internal"
141+
142+
nested = Agent(
143+
name="worker",
144+
tools=[lookup],
145+
output_guardrails=[check_output],
146+
model=ScriptedModel(
147+
[
148+
[get_function_tool_call("lookup", "{}")],
149+
[get_text_message("")],
150+
]
151+
),
152+
)
153+
nested_tool = nested.as_tool(
154+
"delegate",
155+
"Delegate",
156+
on_stream=(lambda event: None) if streamed else None,
157+
failure_error_function=None,
158+
)
159+
parent_model = ScriptedModel(
160+
[
161+
[get_function_tool_call("delegate", '{"input":"work"}')],
162+
[get_text_message("done")],
163+
]
164+
)
165+
parent = Agent(name="parent", model=parent_model, tools=[nested_tool])
166+
paused = await Runner.run(parent, "go")
167+
assert len(paused.interruptions) == 1
168+
assert checked == []
169+
assert not [item for item in paused.new_items if isinstance(item, ToolCallOutputItem)]
170+
state = await RunState.from_json(parent, paused.to_state().to_json())
171+
state.approve(state.get_interruptions()[0])
172+
resumed = await Runner.run(parent, state)
173+
assert checked == [""]
174+
assert [item.output for item in resumed.new_items if isinstance(item, ToolCallOutputItem)] == [
175+
""
176+
]
177+
parent_input = parent_model.calls[1].input
178+
assert isinstance(parent_input, list)
179+
assert [
180+
item["output"] for item in parent_input if item.get("type") == "function_call_output"
181+
] == [""]

0 commit comments

Comments
 (0)