Skip to content

Commit 518b1f2

Browse files
authored
fix(realtime): enforce Realtime tool output guardrails (#5127)
1 parent 45e6f6c commit 518b1f2

3 files changed

Lines changed: 353 additions & 1 deletion

File tree

‎src/agents/realtime/session.py‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
from ..exceptions import (
2525
ModelBehaviorError,
2626
ToolInputGuardrailTripwireTriggered,
27+
ToolOutputGuardrailTripwireTriggered,
2728
UserError,
2829
_clear_data_redacted_error_traceback,
2930
_detach_data_redacted_error_traceback,
@@ -42,7 +43,7 @@
4243
from ..run_context import RunContextWrapper, TContext
4344
from ..tool import DEFAULT_APPROVAL_REJECTION_MESSAGE, FunctionTool, Tool, invoke_function_tool
4445
from ..tool_context import ToolContext
45-
from ..tool_guardrails import ToolInputGuardrailData
46+
from ..tool_guardrails import ToolInputGuardrailData, ToolOutputGuardrailData
4647
from ..util._approvals import evaluate_function_tool_approval
4748
from ..util._asyncio_tasks import gather_with_cancel
4849
from ._tool_filtering import filter_enabled_tools
@@ -817,6 +818,30 @@ async def _run_tool_input_guardrails(
817818
return gr_out.behavior["message"]
818819
return None
819820

821+
async def _run_tool_output_guardrails(
822+
self,
823+
*,
824+
tool: FunctionTool,
825+
tool_context: ToolContext[Any],
826+
agent: RealtimeAgent,
827+
output: Any,
828+
) -> Any:
829+
"""Check the result before caching or publishing a function tool output."""
830+
guardrails = tool.tool_output_guardrails
831+
if not guardrails:
832+
return output
833+
for guardrail in guardrails:
834+
gr_out = await guardrail.run(
835+
ToolOutputGuardrailData(
836+
context=tool_context, agent=cast(Agent[Any], agent), output=output
837+
)
838+
)
839+
if gr_out.behavior["type"] == "raise_exception":
840+
raise ToolOutputGuardrailTripwireTriggered(guardrail=guardrail, output=gr_out)
841+
if gr_out.behavior["type"] == "reject_content":
842+
return gr_out.behavior["message"]
843+
return output
844+
820845
def _build_realtime_tool_output(
821846
self,
822847
*,
@@ -1243,6 +1268,15 @@ async def _handle_tool_call(
12431268
if self._closing or self._closed:
12441269
return
12451270

1271+
result = await self._run_tool_output_guardrails(
1272+
tool=func_tool,
1273+
tool_context=tool_context,
1274+
agent=agent,
1275+
output=result,
1276+
)
1277+
if self._closing or self._closed:
1278+
return
1279+
12461280
await self._send_tool_output_completion(
12471281
_PendingToolOutput(
12481282
tool_call=event,

‎tests/realtime/session_test_support.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ def _set_default_timeout_fields(tool: Mock) -> Mock:
8282
tool.timeout_seconds = None
8383
tool.timeout_behavior = "error_as_result"
8484
tool.timeout_error_function = None
85+
tool.tool_output_guardrails = []
8586
return tool
8687

8788

Lines changed: 317 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,317 @@
1+
"""Function tool output policies at the Realtime session dispatch boundary."""
2+
3+
import asyncio
4+
import json
5+
from typing import Any, Literal
6+
7+
import pytest
8+
9+
from agents.decorators import tool, tool_output_guardrail
10+
from agents.exceptions import ModelBehaviorError, ToolOutputGuardrailTripwireTriggered
11+
from agents.realtime import RealtimeAgent, RealtimeSession
12+
from agents.realtime.events import RealtimeSessionEvent, RealtimeToolEnd
13+
from agents.realtime.model_events import RealtimeModelToolCallEvent
14+
from agents.realtime.model_inputs import RealtimeModelSendEvent, RealtimeModelSendToolOutput
15+
from agents.realtime.testing import ScriptedRealtimeModel
16+
from agents.tool_context import ToolContext
17+
from agents.tool_guardrails import ToolGuardrailFunctionOutput, ToolOutputGuardrailData
18+
19+
from ..mcp.helpers import FakeMCPServer
20+
21+
22+
def tool_call() -> RealtimeModelToolCallEvent:
23+
return RealtimeModelToolCallEvent(name="lookup", call_id="call_lookup", arguments="{}")
24+
25+
26+
def outputs(model: ScriptedRealtimeModel) -> list[str]:
27+
return [
28+
event.output
29+
for event in model.sent_events
30+
if isinstance(event, RealtimeModelSendToolOutput)
31+
]
32+
33+
34+
async def collect_events(session: RealtimeSession, events: list[RealtimeSessionEvent]) -> None:
35+
async for event in session:
36+
events.append(event)
37+
38+
39+
async def next_event(session: RealtimeSession, event_type: str) -> RealtimeSessionEvent:
40+
async def wait() -> RealtimeSessionEvent:
41+
async for event in session:
42+
if event.type == event_type:
43+
return event
44+
raise AssertionError(f"Session closed before {event_type}")
45+
46+
return await asyncio.wait_for(wait(), timeout=2)
47+
48+
49+
@pytest.mark.asyncio
50+
@pytest.mark.parametrize("reject", [False, True])
51+
async def test_tool_output_policies_check_original_result_in_order(reject: bool) -> None:
52+
original = {"value": "synthetic tool result"}
53+
seen: list[tuple[str, ToolOutputGuardrailData]] = []
54+
invocation_contexts: list[ToolContext[Any]] = []
55+
56+
@tool_output_guardrail
57+
def first(data: ToolOutputGuardrailData) -> ToolGuardrailFunctionOutput:
58+
seen.append(("first", data))
59+
return ToolGuardrailFunctionOutput.allow()
60+
61+
@tool_output_guardrail
62+
async def second(data: ToolOutputGuardrailData) -> ToolGuardrailFunctionOutput:
63+
seen.append(("second", data))
64+
if reject:
65+
return ToolGuardrailFunctionOutput.reject_content("replacement output")
66+
return ToolGuardrailFunctionOutput.allow()
67+
68+
@tool_output_guardrail
69+
def third(data: ToolOutputGuardrailData) -> ToolGuardrailFunctionOutput:
70+
seen.append(("third", data))
71+
return ToolGuardrailFunctionOutput.allow()
72+
73+
@tool(tool_output_guardrails=[first, second, third])
74+
async def lookup(context: ToolContext[Any]) -> dict[str, str]:
75+
invocation_contexts.append(context)
76+
return original
77+
78+
agent = RealtimeAgent(name="source", tools=[lookup])
79+
model = ScriptedRealtimeModel(strict=False)
80+
async with RealtimeSession(
81+
model, agent, {"user": "synthetic"}, run_config={"async_tool_calls": False}
82+
) as session:
83+
await model.emit(tool_call())
84+
end = await next_event(session, "tool_end")
85+
assert isinstance(end, RealtimeToolEnd)
86+
assert end.output == ("replacement output" if reject else original)
87+
assert end.agent is agent
88+
assert outputs(model) == ["replacement output" if reject else json.dumps(original)]
89+
assert [name for name, _ in seen] == (
90+
["first", "second"] if reject else ["first", "second", "third"]
91+
)
92+
for _, data in seen:
93+
assert data.output is original
94+
assert data.agent is agent
95+
assert data.context is invocation_contexts[0]
96+
assert data.context.tool_call_id == "call_lookup"
97+
assert data.context.tool_arguments == "{}"
98+
await model.emit(tool_call())
99+
assert len(invocation_contexts) == 1
100+
assert len(outputs(model)) == 1
101+
102+
103+
@pytest.mark.asyncio
104+
@pytest.mark.parametrize("async_tools", [False, True])
105+
@pytest.mark.parametrize("behavior", ["tripwire", "error"])
106+
async def test_output_policy_failure_never_publishes_or_reexecutes(
107+
async_tools: bool, behavior: str
108+
) -> None:
109+
calls = 0
110+
checks = 0
111+
events: list[RealtimeSessionEvent] = []
112+
113+
@tool_output_guardrail
114+
async def policy(data: ToolOutputGuardrailData) -> ToolGuardrailFunctionOutput:
115+
nonlocal checks
116+
checks += 1
117+
assert data.output == "unchecked result"
118+
if behavior == "error":
119+
raise ValueError("policy unavailable")
120+
return ToolGuardrailFunctionOutput.raise_exception()
121+
122+
@tool(tool_output_guardrails=[policy])
123+
async def lookup() -> str:
124+
nonlocal calls
125+
calls += 1
126+
return "unchecked result"
127+
128+
model = ScriptedRealtimeModel(strict=False)
129+
session = RealtimeSession(
130+
model,
131+
RealtimeAgent(name="source", tools=[lookup]),
132+
None,
133+
run_config={"async_tool_calls": async_tools},
134+
)
135+
expected_error = ValueError if behavior == "error" else ToolOutputGuardrailTripwireTriggered
136+
async with session:
137+
if async_tools:
138+
consumer = asyncio.create_task(collect_events(session, events))
139+
await model.emit(tool_call())
140+
with pytest.raises(expected_error):
141+
await asyncio.wait_for(consumer, timeout=2)
142+
else:
143+
with pytest.raises(expected_error):
144+
await model.emit(tool_call())
145+
with pytest.raises(ModelBehaviorError, match="already executed"):
146+
await model.emit(tool_call())
147+
if not async_tools:
148+
await collect_events(session, events)
149+
assert calls == checks == 1
150+
assert outputs(model) == []
151+
assert not any(isinstance(event, RealtimeToolEnd) for event in events)
152+
assert session._pending_tool_outputs == {}
153+
154+
155+
@pytest.mark.asyncio
156+
async def test_mcp_output_policy_is_enforced_after_conversion() -> None:
157+
seen: list[Any] = []
158+
159+
@tool_output_guardrail
160+
def policy(data: ToolOutputGuardrailData) -> ToolGuardrailFunctionOutput:
161+
seen.append(data.output)
162+
return ToolGuardrailFunctionOutput.reject_content("MCP replacement")
163+
164+
server = FakeMCPServer(tool_output_guardrails=[policy])
165+
server.add_tool("lookup", {"type": "object", "properties": {}})
166+
agent = RealtimeAgent(name="source", mcp_servers=[server])
167+
model = ScriptedRealtimeModel(strict=False)
168+
async with RealtimeSession(
169+
model, agent, None, run_config={"async_tool_calls": False}
170+
) as session:
171+
await model.emit(tool_call())
172+
end = await next_event(session, "tool_end")
173+
assert isinstance(end, RealtimeToolEnd)
174+
assert end.output == "MCP replacement"
175+
assert outputs(model) == ["MCP replacement"]
176+
assert server.tool_calls == ["lookup"]
177+
assert seen == [{"type": "text", "text": server.tool_results[0]}]
178+
179+
180+
@pytest.mark.asyncio
181+
async def test_approved_async_tool_output_policy_keeps_dispatch_agent() -> None:
182+
started = asyncio.Event()
183+
release = asyncio.Event()
184+
checks: list[ToolOutputGuardrailData] = []
185+
186+
@tool_output_guardrail
187+
async def policy(data: ToolOutputGuardrailData) -> ToolGuardrailFunctionOutput:
188+
checks.append(data)
189+
started.set()
190+
await release.wait()
191+
return ToolGuardrailFunctionOutput.reject_content("approved replacement")
192+
193+
@tool(needs_approval=True, tool_output_guardrails=[policy])
194+
async def lookup() -> str:
195+
return "unchecked result"
196+
197+
agent = RealtimeAgent(name="source", tools=[lookup])
198+
model = ScriptedRealtimeModel(strict=False)
199+
async with RealtimeSession(model, agent, None) as session:
200+
await model.emit(tool_call())
201+
await next_event(session, "tool_approval_required")
202+
assert checks == []
203+
await session.approve_tool_call("call_lookup")
204+
await asyncio.wait_for(started.wait(), timeout=2)
205+
await session.update_agent(RealtimeAgent(name="replacement"))
206+
assert outputs(model) == []
207+
release.set()
208+
end = await next_event(session, "tool_end")
209+
assert isinstance(end, RealtimeToolEnd)
210+
assert end.agent is agent
211+
assert end.output == "approved replacement"
212+
assert checks[0].agent is agent
213+
assert checks[0].context.agent is agent
214+
assert outputs(model) == ["approved replacement"]
215+
216+
217+
class FailOnceModel(ScriptedRealtimeModel):
218+
def __init__(self) -> None:
219+
super().__init__(strict=False)
220+
self.attempts: list[str] = []
221+
222+
async def send_event(self, event: RealtimeModelSendEvent) -> None:
223+
if isinstance(event, RealtimeModelSendToolOutput):
224+
self.attempts.append(event.output)
225+
if len(self.attempts) == 1:
226+
raise RuntimeError("transport unavailable")
227+
await super().send_event(event)
228+
229+
230+
@pytest.mark.asyncio
231+
async def test_transport_retry_reuses_only_checked_output() -> None:
232+
calls = 0
233+
checks = 0
234+
235+
@tool_output_guardrail
236+
def policy(data: ToolOutputGuardrailData) -> ToolGuardrailFunctionOutput:
237+
nonlocal checks
238+
checks += 1
239+
return ToolGuardrailFunctionOutput.reject_content("retry replacement")
240+
241+
@tool(tool_output_guardrails=[policy])
242+
async def lookup() -> str:
243+
nonlocal calls
244+
calls += 1
245+
return "unchecked result"
246+
247+
model = FailOnceModel()
248+
async with RealtimeSession(
249+
model, RealtimeAgent(name="source", tools=[lookup]), None
250+
) as session:
251+
await model.emit(tool_call())
252+
await next_event(session, "error")
253+
assert outputs(model) == []
254+
pending = session._pending_tool_outputs["call_lookup"]
255+
assert pending.output == "retry replacement"
256+
assert pending.tool_end_event is not None
257+
assert pending.tool_end_event.output == "retry replacement"
258+
await model.emit(tool_call())
259+
end = await next_event(session, "tool_end")
260+
assert isinstance(end, RealtimeToolEnd)
261+
assert end.output == "retry replacement"
262+
assert outputs(model) == ["retry replacement"]
263+
assert model.attempts == ["retry replacement", "retry replacement"]
264+
assert calls == checks == 1
265+
266+
267+
@pytest.mark.asyncio
268+
@pytest.mark.parametrize("shutdown", ["cancel", "close", "close_suppress_cancel"])
269+
async def test_output_policy_shutdown_prevents_late_output(
270+
shutdown: Literal["cancel", "close", "close_suppress_cancel"],
271+
) -> None:
272+
started = asyncio.Event()
273+
cancelled = asyncio.Event()
274+
calls = 0
275+
events: list[RealtimeSessionEvent] = []
276+
277+
@tool_output_guardrail
278+
async def policy(data: ToolOutputGuardrailData) -> ToolGuardrailFunctionOutput:
279+
started.set()
280+
try:
281+
await asyncio.Event().wait()
282+
except asyncio.CancelledError:
283+
cancelled.set()
284+
if shutdown != "close_suppress_cancel":
285+
raise
286+
return ToolGuardrailFunctionOutput.allow()
287+
288+
@tool(tool_output_guardrails=[policy])
289+
async def lookup() -> str:
290+
nonlocal calls
291+
calls += 1
292+
return "unchecked result"
293+
294+
model = ScriptedRealtimeModel(strict=False)
295+
async with RealtimeSession(
296+
model,
297+
RealtimeAgent(name="source", tools=[lookup]),
298+
None,
299+
run_config={"async_tool_calls": shutdown != "cancel"},
300+
) as session:
301+
if shutdown == "cancel":
302+
dispatch = asyncio.create_task(session.on_event(tool_call()))
303+
await asyncio.wait_for(started.wait(), timeout=2)
304+
dispatch.cancel()
305+
with pytest.raises(asyncio.CancelledError):
306+
await dispatch
307+
with pytest.raises(ModelBehaviorError, match="already executed"):
308+
await session.on_event(tool_call())
309+
else:
310+
await model.emit(tool_call())
311+
await asyncio.wait_for(started.wait(), timeout=2)
312+
await session.close()
313+
assert cancelled.is_set()
314+
await collect_events(session, events)
315+
assert calls == 1
316+
assert outputs(model) == []
317+
assert not any(isinstance(event, RealtimeToolEnd) for event in events)

0 commit comments

Comments
 (0)