Skip to content

Commit 40956e0

Browse files
authored
fix(core): redact default tool failure details (#5112)
1 parent 7fce7f6 commit 40956e0

14 files changed

Lines changed: 258 additions & 82 deletions

‎src/agents/run_internal/tool_execution.py‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,9 +82,11 @@
8282
_consume_function_tool_default_failure,
8383
_invoke_function_tool_with_metadata,
8484
_is_programmatic_tool_call,
85+
default_tool_error_function,
8586
get_function_tool_origin,
8687
maybe_invoke_function_tool_failure_error_function,
8788
resolve_computer,
89+
resolve_function_tool_failure_error_function,
8890
)
8991
from ..tool_context import ToolContext
9092
from ..tool_guardrails import (
@@ -2094,9 +2096,14 @@ async def _invoke_tool_and_run_post_invoke(
20942096
tool_context
20952097
) and not _uses_programmatic_output_schema(func_tool, tool_call)
20962098

2099+
include_error_detail = (
2100+
self.config.trace_include_sensitive_data
2101+
and resolve_function_tool_failure_error_function(func_tool, tool_context)
2102+
is not default_tool_error_function
2103+
)
20972104
trace_error = get_trace_tool_error(
2098-
trace_include_sensitive_data=self.config.trace_include_sensitive_data,
2099-
error_message=str(e),
2105+
trace_include_sensitive_data=include_error_detail,
2106+
error_message=str(e) if include_error_detail else "",
21002107
)
21012108
_error_tracing.attach_error_to_current_span(
21022109
SpanError(

‎src/agents/tool.py‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1889,16 +1889,20 @@ def _on_handled_error(
18891889
input_json: str,
18901890
context: ToolContext[Any],
18911891
) -> None:
1892-
json_decode_error = _extract_tool_argument_json_error(error)
1892+
trace_include_sensitive_data = (
1893+
context.run_config is None or context.run_config.trace_include_sensitive_data
1894+
) and resolve_function_tool_failure_error_function(
1895+
function_tool, context
1896+
) is not default_tool_error_function
1897+
json_decode_error = (
1898+
_extract_tool_argument_json_error(error) if trace_include_sensitive_data else None
1899+
)
18931900
if json_decode_error is not None and span_message_for_json_decode_error is not None:
18941901
resolved_span_message = span_message_for_json_decode_error
18951902
span_error_detail = str(json_decode_error)
18961903
else:
18971904
resolved_span_message = span_message
1898-
span_error_detail = str(error)
1899-
trace_include_sensitive_data = (
1900-
context.run_config is None or context.run_config.trace_include_sensitive_data
1901-
)
1905+
span_error_detail = str(error) if trace_include_sensitive_data else ""
19021906
trace_error = get_trace_tool_error(
19031907
trace_include_sensitive_data=trace_include_sensitive_data,
19041908
error_message=span_error_detail,
@@ -1964,15 +1968,11 @@ def _log_function_tool_invocation(*, tool_name: str, input_json: str) -> None:
19641968

19651969

19661970
def default_tool_error_function(ctx: RunContextWrapper[Any], error: Exception) -> str:
1967-
"""The default tool error function, which just returns a generic error message."""
1968-
json_decode_error = _extract_tool_argument_json_error(error)
1969-
if json_decode_error is not None:
1970-
return (
1971-
"An error occurred while parsing tool arguments. "
1972-
"Please try again with valid JSON. "
1973-
f"Error: {json_decode_error}"
1974-
)
1975-
return f"An error occurred while running the tool. Please try again. Error: {str(error)}"
1971+
"""Return a fixed error response without exposing the exception to the model.
1972+
1973+
Provide a custom ``failure_error_function`` to return application-approved error details.
1974+
"""
1975+
return "An error occurred while running the tool. Please try again."
19761976

19771977

19781978
_FUNCTION_TOOL_TIMEOUT_BEHAVIORS: tuple[ToolTimeoutBehavior, ...] = (

‎tests/mcp/test_mcp_util.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1110,7 +1110,7 @@ async def test_mcp_tool_inner_cancellation_becomes_tool_error():
11101110

11111111
result = await function_tool.on_invoke_tool(tool_context, "{}")
11121112
assert isinstance(result, str)
1113-
assert "tool execution was cancelled" in result
1113+
assert result == "An error occurred while running the tool. Please try again."
11141114

11151115

11161116
@pytest.mark.asyncio
@@ -1399,7 +1399,7 @@ async def test_mcp_default_tool_error_hides_url_credentials():
13991399
result = await function_tool.on_invoke_tool(tool_context, "{}")
14001400

14011401
assert isinstance(result, str)
1402-
assert _SANITIZED_SERVER_NAME in result
1402+
assert result == "An error occurred while running the tool. Please try again."
14031403
for secret in _SERVER_URL_SECRETS:
14041404
assert secret not in result
14051405

@@ -1449,7 +1449,7 @@ async def call_tool(
14491449
# Verify that the result is an error message
14501450
assert isinstance(result, str)
14511451
assert "error" in result.lower() or "occurred" in result.lower()
1452-
assert "Timed out" in result
1452+
assert result == "An error occurred while running the tool. Please try again."
14531453

14541454

14551455
@pytest.mark.asyncio
@@ -1473,7 +1473,7 @@ async def test_mcp_tool_cancellation_returns_error_message():
14731473
result = await function_tool.on_invoke_tool(tool_context, "{}")
14741474

14751475
assert isinstance(result, str)
1476-
assert "cancelled" in result.lower()
1476+
assert result == "An error occurred while running the tool. Please try again."
14771477

14781478

14791479
@pytest.mark.asyncio

‎tests/test_agent_as_tool.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1347,7 +1347,7 @@ def builder(_options):
13471347
)
13481348
result = await tool.on_invoke_tool(tool_context, '{"input": "hi"}')
13491349

1350-
assert "Agent tool called with invalid input" in result
1350+
assert result == "An error occurred while running the tool. Please try again."
13511351

13521352

13531353
@pytest.mark.asyncio
@@ -3781,7 +3781,7 @@ async def extract(result: RunResult | RunResultStreaming) -> str:
37813781
assert len(errors) == 1
37823782
assert isinstance(errors[0], UserError)
37833783
else:
3784-
assert "on_stream_max_pending_events=2" in output
3784+
assert output == "An error occurred while running the tool. Please try again."
37853785
assert "synthetic delta" not in output
37863786

37873787
assert handler_cancelled.is_set()

‎tests/test_agent_runner.py‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1969,9 +1969,7 @@ async def _cancel_tool() -> str:
19691969
{"call_id": "call_ok", "output": "ok", "type": "function_call_output"},
19701970
{
19711971
"call_id": "call_cancel",
1972-
"output": (
1973-
"An error occurred while running the tool. Please try again. Error: tool-cancelled"
1974-
),
1972+
"output": ("An error occurred while running the tool. Please try again."),
19751973
"type": "function_call_output",
19761974
},
19771975
]
@@ -2008,9 +2006,7 @@ async def _cancel_tool() -> str:
20082006
assert tool_outputs == [
20092007
{
20102008
"call_id": "call_cancel",
2011-
"output": (
2012-
"An error occurred while running the tool. Please try again. Error: tool-cancelled"
2013-
),
2009+
"output": ("An error occurred while running the tool. Please try again."),
20142010
"type": "function_call_output",
20152011
},
20162012
]

‎tests/test_agent_runner_streamed.py‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -900,9 +900,7 @@ async def _cancel_tool() -> str:
900900
{"call_id": "call_ok", "output": "ok", "type": "function_call_output"},
901901
{
902902
"call_id": "call_cancel",
903-
"output": (
904-
"An error occurred while running the tool. Please try again. Error: tool-cancelled"
905-
),
903+
"output": ("An error occurred while running the tool. Please try again."),
906904
"type": "function_call_output",
907905
},
908906
]
@@ -940,9 +938,7 @@ async def _cancel_tool() -> str:
940938
assert tool_outputs == [
941939
{
942940
"call_id": "call_cancel",
943-
"output": (
944-
"An error occurred while running the tool. Please try again. Error: tool-cancelled"
945-
),
941+
"output": ("An error occurred while running the tool. Please try again."),
946942
"type": "function_call_output",
947943
},
948944
]
Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
from __future__ import annotations
2+
3+
import asyncio
4+
import json
5+
import logging
6+
from typing import Any, Literal
7+
8+
import pytest
9+
10+
import agents._debug as _debug
11+
from agents import Agent, RunConfig, RunContextWrapper, Runner, UserError
12+
from agents.decorators import tool
13+
from agents.testing import ScriptedModel
14+
from agents.tool import default_tool_error_function
15+
16+
from .test_responses import get_function_tool_call, get_text_message
17+
from .testing_processor import SPAN_PROCESSOR_TESTING
18+
19+
SENTINEL = "synthetic-private-tool-detail"
20+
GENERIC_ERROR = "An error occurred while running the tool. Please try again."
21+
22+
23+
@pytest.mark.asyncio
24+
@pytest.mark.parametrize("streamed", [False, True])
25+
@pytest.mark.parametrize("sensitive_tracing", [False, True])
26+
@pytest.mark.parametrize("failure", ["sync", "async", "cancel"])
27+
async def test_default_failure_keeps_exception_out_of_model_history_and_traces(
28+
streamed: bool,
29+
sensitive_tracing: bool,
30+
failure: Literal["sync", "async", "cancel"],
31+
caplog: pytest.LogCaptureFixture,
32+
) -> None:
33+
def raise_error() -> str:
34+
try:
35+
raise RuntimeError(f"cause-{SENTINEL}")
36+
except RuntimeError as cause:
37+
error = ValueError(SENTINEL)
38+
if hasattr(error, "add_note"):
39+
error.add_note(f"note-{SENTINEL}")
40+
raise error from cause
41+
42+
async def raise_async_error() -> str:
43+
if failure == "cancel":
44+
raise asyncio.CancelledError(SENTINEL)
45+
return raise_error()
46+
47+
failing_tool = tool(raise_error if failure == "sync" else raise_async_error)
48+
model = ScriptedModel(
49+
[
50+
[get_function_tool_call(failing_tool.name, "{}", call_id="failure-call")],
51+
[get_text_message("recovered")],
52+
]
53+
)
54+
agent = Agent(name="test", model=model, tools=[failing_tool])
55+
config = RunConfig(trace_include_sensitive_data=sensitive_tracing)
56+
with caplog.at_level(logging.DEBUG, logger="openai.agents"):
57+
if streamed:
58+
result = Runner.run_streamed(agent, "start", run_config=config)
59+
async for _ in result.stream_events():
60+
pass
61+
else:
62+
result = await Runner.run(agent, "start", run_config=config)
63+
64+
assert result.final_output == "recovered"
65+
next_input = model.calls[-1].input
66+
assert isinstance(next_input, list)
67+
outputs = [item for item in next_input if item.get("type") == "function_call_output"]
68+
assert len(outputs) == 1
69+
assert outputs[0]["output"] == GENERIC_ERROR
70+
assert SENTINEL not in json.dumps(next_input)
71+
assert SENTINEL not in json.dumps(result.to_input_list())
72+
spans = [span.export() for span in SPAN_PROCESSOR_TESTING.get_ordered_spans()]
73+
assert spans
74+
assert SENTINEL not in json.dumps(spans)
75+
for record in caplog.records:
76+
assert SENTINEL not in repr(record.__dict__)
77+
assert record.exc_info is None
78+
assert record.exc_text is None
79+
assert SENTINEL not in logging.Formatter().format(record)
80+
81+
82+
@pytest.mark.asyncio
83+
@pytest.mark.parametrize("policy", ["custom", "propagate", "explicit-default"])
84+
async def test_explicit_failure_policies_remain_available(
85+
policy: Literal["custom", "propagate", "explicit-default"],
86+
) -> None:
87+
received: list[Exception] = []
88+
89+
def custom_error(context: RunContextWrapper[Any], error: Exception) -> str:
90+
received.append(error)
91+
return "Application-approved feedback"
92+
93+
@tool(
94+
failure_error_function=(
95+
custom_error
96+
if policy == "custom"
97+
else default_tool_error_function
98+
if policy == "explicit-default"
99+
else None
100+
)
101+
)
102+
async def failing_tool() -> str:
103+
raise ValueError(SENTINEL)
104+
105+
model = ScriptedModel(
106+
[
107+
[get_function_tool_call("failing_tool", "{}")],
108+
[get_text_message("recovered")],
109+
]
110+
)
111+
agent = Agent(name="test", model=model, tools=[failing_tool])
112+
if policy == "propagate":
113+
with pytest.raises(UserError, match=SENTINEL) as exc_info:
114+
await Runner.run(agent, "start")
115+
assert isinstance(exc_info.value.__cause__, ValueError)
116+
return
117+
118+
result = await Runner.run(agent, "start")
119+
assert result.final_output == "recovered"
120+
next_input = model.calls[-1].input
121+
assert isinstance(next_input, list)
122+
output = next(item for item in next_input if item.get("type") == "function_call_output")
123+
if policy == "custom":
124+
assert output["output"] == "Application-approved feedback"
125+
assert len(received) == 1
126+
assert str(received[0]) == SENTINEL
127+
else:
128+
assert output["output"] == GENERIC_ERROR
129+
spans = [span.export() for span in SPAN_PROCESSOR_TESTING.get_ordered_spans()]
130+
assert SENTINEL not in json.dumps(spans)
131+
132+
133+
@pytest.mark.asyncio
134+
async def test_default_failure_does_not_format_the_exception() -> None:
135+
class UnprintableError(Exception):
136+
def __str__(self) -> str:
137+
raise AssertionError("Exception text must not be inspected")
138+
139+
@tool
140+
async def failing_tool() -> str:
141+
raise UnprintableError()
142+
143+
model = ScriptedModel(
144+
[[get_function_tool_call("failing_tool", "{}")], [get_text_message("recovered")]]
145+
)
146+
result = await Runner.run(Agent(name="test", model=model, tools=[failing_tool]), "start")
147+
assert result.final_output == "recovered"
148+
149+
150+
@pytest.mark.asyncio
151+
async def test_explicit_local_diagnostics_keep_exception_detail(
152+
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
153+
) -> None:
154+
monkeypatch.setattr(_debug, "DONT_LOG_TOOL_DATA", False)
155+
156+
@tool
157+
async def failing_tool() -> str:
158+
raise ValueError(SENTINEL)
159+
160+
model = ScriptedModel(
161+
[[get_function_tool_call("failing_tool", "{}")], [get_text_message("recovered")]]
162+
)
163+
with caplog.at_level(logging.DEBUG, logger="openai.agents"):
164+
result = await Runner.run(Agent(name="test", model=model, tools=[failing_tool]), "start")
165+
assert result.final_output == "recovered"
166+
assert SENTINEL in caplog.text
167+
assert any(record.exc_info is not None for record in caplog.records)
168+
assert SENTINEL not in json.dumps(model.calls[-1].input)
169+
assert SENTINEL not in json.dumps(
170+
[span.export() for span in SPAN_PROCESSOR_TESTING.get_ordered_spans()]
171+
)

0 commit comments

Comments
 (0)