Skip to content

Commit 6ba967b

Browse files
fix: recover from invalid sandbox tool arguments (#5321)
1 parent c757a43 commit 6ba967b

5 files changed

Lines changed: 194 additions & 13 deletions

File tree

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
from pydantic import BaseModel, ValidationError
2+
3+
4+
def format_invalid_arguments(error: ValidationError, model: type[BaseModel]) -> str:
5+
"""Identify invalid declared fields without exposing input values or exception text."""
6+
locations = {
7+
item["loc"][0]
8+
for item in error.errors(include_input=False, include_context=False, include_url=False)
9+
if item["loc"]
10+
}
11+
fields = ", ".join(name for name in model.model_fields if name in locations)
12+
target = f" for {fields}" if fields else ""
13+
return f"Invalid tool arguments{target}. Check the tool schema and try again."

‎src/agents/sandbox/capabilities/tools/shell_tool.py‎

Lines changed: 30 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,21 @@
77
from dataclasses import dataclass, field
88
from typing import Any, ClassVar
99

10-
from pydantic import BaseModel, Field
10+
from pydantic import BaseModel, Field, ValidationError
1111

1212
from ....run_context import RunContextWrapper
1313
from ....tool import FunctionTool
14-
from ...errors import ExecTimeoutError, ExecTransportError, PtySessionNotFoundError
14+
from ...errors import (
15+
ExecTimeoutError,
16+
ExecTransportError,
17+
InvalidManifestPathError,
18+
PtySessionNotFoundError,
19+
)
1520
from ...session.base_sandbox_session import BaseSandboxSession
1621
from ...types import User
1722
from ...util.token_truncation import formatted_truncate_text_with_token_count
1823
from ...workspace_paths import SandboxWorkspaceScope, coerce_posix_path, sandbox_path_str
24+
from ._input_errors import format_invalid_arguments
1925

2026
_DEFAULT_EXEC_YIELD_TIME_MS = 10_000
2127
_DEFAULT_WRITE_STDIN_YIELD_TIME_MS = 250
@@ -196,17 +202,30 @@ def __init__(
196202
)
197203

198204
async def _invoke(self, _: object, raw_input: str) -> str:
199-
return await self.run(self.args_model.model_validate_json(raw_input))
205+
try:
206+
args = self.args_model.model_validate_json(raw_input)
207+
except ValidationError as exc:
208+
return format_invalid_arguments(exc, self.args_model)
209+
try:
210+
command = self._prepare_command(args)
211+
except InvalidManifestPathError:
212+
return "Invalid workdir. Use a path within the workspace or an explicitly granted path."
213+
return await self._run(args, command)
200214

201215
async def run(self, args: ExecCommandArgs) -> str:
202-
start = time.perf_counter()
203-
timeout_s = args.yield_time_ms / 1000
204-
wrapped_command = _resolve_workdir_command(
216+
return await self._run(args, self._prepare_command(args))
217+
218+
def _prepare_command(self, args: ExecCommandArgs) -> str:
219+
return _resolve_workdir_command(
205220
session=self.session,
206221
workspace_scope=self.workspace_scope,
207222
command=args.cmd,
208223
workdir=args.workdir,
209224
)
225+
226+
async def _run(self, args: ExecCommandArgs, wrapped_command: str) -> str:
227+
start = time.perf_counter()
228+
timeout_s = args.yield_time_ms / 1000
210229
shell = _resolve_shell(args.shell, args.login)
211230
fallback_notice: str | None = None
212231

@@ -297,7 +316,11 @@ def __init__(
297316
)
298317

299318
async def _invoke(self, _: object, raw_input: str) -> str:
300-
return await self.run(self.args_model.model_validate_json(raw_input))
319+
try:
320+
args = self.args_model.model_validate_json(raw_input)
321+
except ValidationError as exc:
322+
return format_invalid_arguments(exc, self.args_model)
323+
return await self.run(args)
301324

302325
async def run(self, args: WriteStdinArgs) -> str:
303326
if not self.session.supports_pty():

‎src/agents/sandbox/capabilities/tools/view_image.py‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,15 @@
66
from pathlib import Path
77
from typing import Any, ClassVar
88

9-
from pydantic import BaseModel, Field
9+
from pydantic import BaseModel, Field, ValidationError
1010

1111
from ....run_context import RunContextWrapper
1212
from ....tool import FunctionTool, ToolOutputImage
1313
from ...errors import InvalidManifestPathError, WorkspaceReadNotFoundError
1414
from ...session.base_sandbox_session import BaseSandboxSession
1515
from ...types import User
1616
from ...workspace_paths import SandboxWorkspaceScope, coerce_posix_path, sandbox_path_str
17+
from ._input_errors import format_invalid_arguments
1718

1819
_MAX_IMAGE_BYTES = 10 * 1024 * 1024
1920
_MAX_IMAGE_SIZE_LABEL = "10MB"
@@ -110,9 +111,20 @@ def __init__(
110111
)
111112

112113
async def _invoke(self, _: object, raw_input: str) -> ToolOutputImage | str:
113-
return await self.run(self.args_model.model_validate_json(raw_input))
114+
try:
115+
args = self.args_model.model_validate_json(raw_input)
116+
except ValidationError as exc:
117+
return format_invalid_arguments(exc, self.args_model)
118+
try:
119+
resolved_path, display_path = self._prepare_path(args)
120+
except InvalidManifestPathError:
121+
return "Invalid path. Use a path within the workspace or an explicitly granted path."
122+
return await self._run(resolved_path, display_path)
114123

115124
async def run(self, args: ViewImageArgs) -> ToolOutputImage | str:
125+
return await self._run(*self._prepare_path(args))
126+
127+
def _prepare_path(self, args: ViewImageArgs) -> tuple[Path, str]:
116128
input_path = args.path
117129
scoped_path = self.workspace_scope.anchor(coerce_posix_path(input_path))
118130
path_policy = self.session._workspace_path_policy()
@@ -125,7 +137,9 @@ async def run(self, args: ViewImageArgs) -> ToolOutputImage | str:
125137
).as_posix()
126138
except InvalidManifestPathError:
127139
display_path = sandbox_path_str(resolved_path)
140+
return resolved_path, display_path
128141

142+
async def _run(self, resolved_path: Path, display_path: str) -> ToolOutputImage | str:
129143
try:
130144
file_obj = await self.session.read(resolved_path, user=self.user)
131145
except (FileNotFoundError, WorkspaceReadNotFoundError):
Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
from __future__ import annotations
2+
3+
import asyncio
4+
from typing import Any
5+
6+
import pytest
7+
from pydantic import ValidationError
8+
9+
from agents import Agent, RunConfig, Runner, UserError
10+
from agents.sandbox import Manifest
11+
from agents.sandbox.capabilities.tools import (
12+
ExecCommandArgs,
13+
ExecCommandTool,
14+
ViewImageTool,
15+
WriteStdinTool,
16+
)
17+
from agents.sandbox.errors import InvalidManifestPathError
18+
from agents.testing import ScriptedModel, assistant_message, function_call, scripted_sandbox_session
19+
from tests.testing_processor import SPAN_PROCESSOR_TESTING
20+
21+
22+
@pytest.mark.asyncio
23+
@pytest.mark.parametrize("streamed", [False, True])
24+
@pytest.mark.parametrize(
25+
"tool_type, arguments, expected",
26+
[
27+
(
28+
ExecCommandTool,
29+
{"cmd": "pwd", "workdir": "/outside/private-sentinel"},
30+
"Invalid workdir.",
31+
),
32+
(ExecCommandTool, {"cmd": ""}, "Invalid tool arguments for cmd."),
33+
(
34+
ExecCommandTool,
35+
{"cmd": "private-sentinel", "max_output_tokens": 0},
36+
"Invalid tool arguments for max_output_tokens.",
37+
),
38+
(ExecCommandTool, '{"cmd": "private-sentinel"', "Invalid tool arguments."),
39+
(ViewImageTool, {"path": "/outside/private-sentinel"}, "Invalid path."),
40+
(
41+
ViewImageTool,
42+
{"path": {"private-sentinel": "value"}},
43+
"Invalid tool arguments for path.",
44+
),
45+
(WriteStdinTool, {"chars": "private-sentinel"}, "Invalid tool arguments for session_id."),
46+
],
47+
)
48+
async def test_invalid_model_input_returns_safe_feedback_without_backend_calls(
49+
tool_type: type[ExecCommandTool] | type[ViewImageTool] | type[WriteStdinTool],
50+
arguments: Any,
51+
expected: str,
52+
streamed: bool,
53+
caplog: pytest.LogCaptureFixture,
54+
) -> None:
55+
session = scripted_sandbox_session(manifest=Manifest(root="/workspace"))
56+
tool = tool_type(session=session)
57+
model = ScriptedModel(
58+
[
59+
[function_call(tool.name, arguments, call_id="bad-input")],
60+
[assistant_message("recovered")],
61+
]
62+
)
63+
agent = Agent(name="test", model=model, tools=[tool])
64+
config = RunConfig(trace_include_sensitive_data=False)
65+
if streamed:
66+
result = Runner.run_streamed(agent, "go", run_config=config)
67+
async for _ in result.stream_events():
68+
pass
69+
else:
70+
result = await Runner.run(agent, "go", run_config=config)
71+
72+
assert result.final_output == "recovered"
73+
outputs = [item.output for item in result.new_items if item.type == "tool_call_output_item"]
74+
assert len(outputs) == 1
75+
assert str(outputs[0]).startswith(expected)
76+
assert "private-sentinel" not in str(outputs[0])
77+
assert model.calls[1].input[-1]["output"] == outputs[0]
78+
assert session.calls == ()
79+
model.assert_complete()
80+
assert "private-sentinel" not in caplog.text
81+
assert "private-sentinel" not in repr(
82+
[span.export() for span in SPAN_PROCESSOR_TESTING.get_ordered_spans()]
83+
)
84+
85+
86+
@pytest.mark.asyncio
87+
@pytest.mark.parametrize("failure_kind", ["runtime", "path", "validation", "cancel"])
88+
async def test_exec_backend_failures_are_not_converted_to_argument_feedback(
89+
failure_kind: str,
90+
) -> None:
91+
failure: BaseException
92+
if failure_kind == "path":
93+
failure = InvalidManifestPathError(rel="backend-private", reason="escape_root")
94+
elif failure_kind == "validation":
95+
with pytest.raises(ValidationError) as captured:
96+
ExecCommandArgs(cmd="")
97+
failure = captured.value
98+
elif failure_kind == "cancel":
99+
failure = asyncio.CancelledError()
100+
else:
101+
failure = RuntimeError("backend failure")
102+
103+
def fail(_call: Any) -> None:
104+
raise failure
105+
106+
session = scripted_sandbox_session([{"method": "exec", "responder": fail}])
107+
tool = ExecCommandTool(session=session)
108+
model = ScriptedModel(
109+
[
110+
[function_call(tool.name, {"cmd": "pwd"}, call_id="exec")],
111+
[assistant_message("done")],
112+
]
113+
)
114+
agent = Agent(name="test", model=model, tools=[tool])
115+
# Tool-local cancellation is converted by the existing runner, not by input handling.
116+
if failure_kind == "cancel":
117+
result = await Runner.run(agent, "go")
118+
outputs = [item.output for item in result.new_items if item.type == "tool_call_output_item"]
119+
assert outputs == ["An error occurred while running the tool. Please try again."]
120+
assert result.final_output == "done"
121+
else:
122+
with pytest.raises(UserError) as raised:
123+
await Runner.run(agent, "go")
124+
assert raised.value.__cause__ is failure
125+
assert len(session.calls) == 1
126+
session.assert_complete()
127+
128+
129+
@pytest.mark.asyncio
130+
async def test_direct_exec_run_still_raises_for_invalid_workdir() -> None:
131+
session = scripted_sandbox_session(manifest=Manifest(root="/workspace"))
132+
with pytest.raises(InvalidManifestPathError):
133+
await ExecCommandTool(session=session).run(ExecCommandArgs(cmd="pwd", workdir="/outside"))
134+
assert session.calls == ()

‎tests/sandbox/capabilities/test_view_image_tool.py‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -74,10 +74,7 @@ async def test_view_image_still_rejects_ungranted_absolute_path(self) -> None:
7474
tool = ViewImageTool(session=session)
7575

7676
with pytest.raises(InvalidManifestPathError):
77-
await tool.on_invoke_tool(
78-
cast(ToolContext[object], None),
79-
'{"path":"/shared/dot.png"}',
80-
)
77+
await tool.run(ViewImageArgs(path="/shared/dot.png"))
8178

8279
assert session.calls == ()
8380

0 commit comments

Comments
 (0)