Skip to content

Commit 216febd

Browse files
committed
Confine the agent with sandbox-exec, below the client (#54)
OpenCode cannot be confined by configuration on 1.18.25. A permission.external_directory rule of {"*": "deny"} loads, orders last (merge is flat(), lookup is findLast) and is still bypassed, because anomalyco/opencode#41067 submits out-of-worktree paths as ../... which no pattern matches. A confined run of ds4 x opencode was 0/3, wrote no file, and all three trials still reached ~/git/gmail-archive. So enforce it below the client. sandbox-exec is built into macOS, needs no daemon or image, and the deny is inherited by every descendant -- verified through bash -> sh -> cat. It also does not care what shape the path took, which is exactly why it defeats #41067 where a config rule cannot. The profile hides every other copy of the target repos plus the directories holding previous trials' solutions and transcripts. ~/bench-solutions has 186 correct patches and this repo's tracked results.jsonl names their absolute paths, so that closes the answer-leak from this issue in the same stroke. It is allow-default on purpose: the agent still needs its venv, caches, the model server and the trial checkout. This hides answers, nothing else. Where sandbox-exec is absent the harness warns and runs unconfined rather than silently not running. Wiring it up caught something in our own suite. The integration fixture's stub agent "solves" a task by copying files from the un-excised reference repo -- the very shortcut this exists to stop -- so those plumbing tests now opt out explicitly via sandbox=False, with a comment saying why. Our own fixture was cheating the same way OpenCode was. Claude-Session: https://claude.ai/code/session_01XqDtT1vzshYY3evmNhavyW
1 parent 6b4e2b5 commit 216febd

3 files changed

Lines changed: 103 additions & 8 deletions

File tree

‎benchmarks/agent/results.jsonl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -853,3 +853,4 @@
853853
{"schema_version": 2, "task": "parser-date", "backend": "ds4", "client": "opencode", "trial": 1, "started": "2026-08-30T19:36:36", "model": "deepseek-v4-flash", "context_tokens": 100000, "effort": null, "env": {"claude": "2.1.251 (Claude Code)", "opencode": "1.18.25", "codex": "codex-cli 0.151.0", "macos": "26.6.2", "machine": "Apple M5 Max", "target_commit": "56e55cc", "ds4_head": "399acbb", "ds4_dirty": false, "ds4_server_mtime": "2026-08-27T20:06:43", "servers": {"ds4": {"sampling": {}, "sampling_source": "engine defaults (not reported by ds4)", "accepts_sampling": ["tools", "tool_choice", "max_tokens", "temperature", "top_p", "top_k", "min_p", "stop", "seed", "stream", "reasoning_effort"], "context_length": 100000}}, "client": "opencode"}, "excluded": false, "exclusion_reason": null, "keep_docstring": true, "removed_lines": 18, "removed_symbols": ["_date"], "control_fails_as_expected": true, "gates_before": {"ruff": 0, "mypy": 18}, "wall_seconds": 116.0, "client_log": "/Users/evanhoffman/bench-logs/parser-date-ds4-opencode-1.stdout.jsonl", "client_log_partial": false, "num_turns": 5, "input_tokens": 16542, "output_tokens": 1835, "reasoning_tokens": 0, "passed": false, "pytest": "49 failed, 6 passed in 0.86s", "touched_tests": false, "gates_after": {"ruff": 0, "mypy": 18}, "gates_delta": {"ruff": 0, "mypy": 0}, "restored_verbatim": false, "target_repo": "~/git/local-llm-testing/gmail-archive", "source_repo_intact": true, "solution_empty": true, "finished": "2026-08-30T19:38:42", "schema_valid": true, "schema_errors": []}
854854
{"schema_version": 2, "task": "mbox-strip-envelope", "backend": "ds4", "client": "opencode", "trial": 1, "started": "2026-08-30T19:55:12", "model": "deepseek-v4-flash", "context_tokens": 100000, "effort": null, "env": {"claude": "2.1.251 (Claude Code)", "opencode": "1.18.25", "codex": "codex-cli 0.151.0", "macos": "26.6.2", "machine": "Apple M5 Max", "target_commit": "56e55cc", "ds4_head": "399acbb", "ds4_dirty": false, "ds4_server_mtime": "2026-08-27T20:06:43", "servers": {"ds4": {"sampling": {}, "sampling_source": "engine defaults (not reported by ds4)", "accepts_sampling": ["tools", "tool_choice", "max_tokens", "temperature", "top_p", "top_k", "min_p", "stop", "seed", "stream", "reasoning_effort"], "context_length": 100000}}, "client": "opencode"}, "excluded": false, "exclusion_reason": null, "keep_docstring": true, "removed_lines": 5, "removed_symbols": ["strip_envelope"], "control_fails_as_expected": true, "gates_before": {"ruff": 0, "mypy": 18}, "wall_seconds": 1264.9, "client_log": "/Users/evanhoffman/bench-logs/mbox-strip-envelope-ds4-opencode-1.stdout.jsonl", "client_log_partial": false, "workspace_escapes": ["/Users/evanhoffman/.claude/skills", "/Users/evanhoffman/.agents/skills", "/Users/evanhoffman/git/gmail-archive", "/Users/evanhoffman/.local/share", "/Users/evanhoffman/git/gmail-archive."], "num_turns": 5, "input_tokens": 21427, "output_tokens": 2568, "reasoning_tokens": 0, "passed": false, "pytest": "3 failed, 13 passed in 0.13s", "touched_tests": false, "gates_after": {"ruff": 0, "mypy": 18}, "gates_delta": {"ruff": 0, "mypy": 0}, "restored_verbatim": false, "target_repo": "~/git/local-llm-testing/gmail-archive", "source_repo_intact": true, "solution_empty": true, "finished": "2026-08-30T20:16:23", "schema_valid": true, "schema_errors": []}
855855
{"schema_version": 2, "task": "parser-mbox-quoting", "backend": "ds4", "client": "opencode", "trial": 1, "started": "2026-08-30T20:16:23", "model": "deepseek-v4-flash", "context_tokens": 100000, "effort": null, "env": {"claude": "2.1.251 (Claude Code)", "opencode": "1.18.25", "codex": "codex-cli 0.151.0", "macos": "26.6.2", "machine": "Apple M5 Max", "target_commit": "56e55cc", "ds4_head": "399acbb", "ds4_dirty": false, "ds4_server_mtime": "2026-08-27T20:06:43", "servers": {"ds4": {"sampling": {}, "sampling_source": "engine defaults (not reported by ds4)", "accepts_sampling": ["tools", "tool_choice", "max_tokens", "temperature", "top_p", "top_k", "min_p", "stop", "seed", "stream", "reasoning_effort"], "context_length": 100000}}, "client": "opencode"}, "excluded": false, "exclusion_reason": null, "keep_docstring": true, "removed_lines": 10, "removed_symbols": ["unquote_mbox"], "control_fails_as_expected": true, "gates_before": {"ruff": 0, "mypy": 18}, "wall_seconds": 403.7, "client_log": "/Users/evanhoffman/bench-logs/parser-mbox-quoting-ds4-opencode-1.stdout.jsonl", "client_log_partial": false, "workspace_escapes": ["/Users/evanhoffman/git/local-llm-testing", "/Users/evanhoffman/git/gmail-archive", "/Users/evanhoffman/.claude/skills", "/Users/evanhoffman/.agents/skills", "/Users/evanhoffman/.local/share"], "num_turns": 21, "input_tokens": 16846, "output_tokens": 9653, "reasoning_tokens": 0, "passed": false, "pytest": "34 failed, 21 passed in 0.86s", "touched_tests": false, "gates_after": {"ruff": 0, "mypy": 18}, "gates_delta": {"ruff": 0, "mypy": 0}, "restored_verbatim": false, "target_repo": "~/git/local-llm-testing/gmail-archive", "source_repo_intact": true, "solution_empty": true, "finished": "2026-08-30T20:23:17", "schema_valid": true, "schema_errors": []}
856+
{"schema_version": 2, "task": "storage-blob-put", "backend": "ds4", "client": "opencode", "trial": 1, "started": "2026-08-30T20:23:17", "model": "deepseek-v4-flash", "context_tokens": 100000, "effort": null, "env": {"claude": "2.1.251 (Claude Code)", "opencode": "1.18.25", "codex": "codex-cli 0.151.0", "macos": "26.6.2", "machine": "Apple M5 Max", "target_commit": "56e55cc", "ds4_head": "399acbb", "ds4_dirty": false, "ds4_server_mtime": "2026-08-27T20:06:43", "servers": {"ds4": {"sampling": {}, "sampling_source": "engine defaults (not reported by ds4)", "accepts_sampling": ["tools", "tool_choice", "max_tokens", "temperature", "top_p", "top_k", "min_p", "stop", "seed", "stream", "reasoning_effort"], "context_length": 100000}}, "client": "opencode"}, "excluded": true, "exclusion_reason": "Answer exposure (#54): agent worked in /Users/evanhoffman/git/local-llm, which can disclose a previous trial's solution patch. Not a verdict about the model.", "keep_docstring": true, "removed_lines": 26, "removed_symbols": ["BlobStore.put"], "control_fails_as_expected": true, "gates_before": {"ruff": 1, "mypy": 18}, "wall_seconds": 624.1, "client_log": "/Users/evanhoffman/bench-logs/storage-blob-put-ds4-opencode-1.stdout.jsonl", "client_log_partial": false, "workspace_escapes": ["/Users/evanhoffman/git/gmail-archive", "/Users/evanhoffman/.claude/skills", "/Users/evanhoffman/.agents/skills", "/Users/evanhoffman/.local/share", "/Users/evanhoffman/git/local-llm"], "num_turns": 31, "input_tokens": 16832, "output_tokens": 17001, "reasoning_tokens": 0, "passed": false, "pytest": "14 failed, 3 passed in 0.12s", "touched_tests": false, "gates_after": {"ruff": 1, "mypy": 18}, "gates_delta": {"ruff": 0, "mypy": 0}, "restored_verbatim": false, "target_repo": "~/git/local-llm-testing/gmail-archive", "source_repo_intact": true, "solution_empty": true, "finished": "2026-08-30T20:33:47", "schema_valid": true, "schema_errors": []}

‎benchmarks/agent/run.py‎

Lines changed: 75 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -695,6 +695,64 @@ def paths_outside(stdout, worktree):
695695
return sorted(seen, key=lambda k: -seen[k])
696696

697697

698+
def sandbox_profile(worktree, repo):
699+
"""A macOS sandbox profile that hides every other copy of the answer.
700+
701+
#54: OpenCode is not confined to its workspace. `opencode run` is headless,
702+
`external_directory` defaults to `ask`, and with nobody to ask it read the
703+
operator's real un-excised repository -- saw green tests, concluded nothing
704+
needed fixing, and wrote nothing. One trial went further and deleted 33
705+
lines from a working function in a checkout it was never pointed at.
706+
707+
Configuration cannot fix this on 1.18.25. A `{"*": "deny"}` rule loads and
708+
orders last, and is still bypassed, because anomalyco/opencode#41067
709+
submits out-of-worktree paths as `../...` which cannot match. The client's
710+
own permission layer is the wrong place to enforce this.
711+
712+
So enforce it below the client. `sandbox-exec` denies the read in the
713+
kernel, is inherited by every descendant (verified bash -> sh -> cat), and
714+
does not care what shape the path took.
715+
716+
Denied: every other checkout of the target repos, plus the directories that
717+
hold previous trials' solutions and transcripts -- `~/bench-solutions` has
718+
186 correct patches and this repo's tracked results.jsonl names their paths.
719+
720+
`allow default` on purpose: the agent still needs its venv, caches, the
721+
model server and the trial checkout. This hides the answers, nothing else.
722+
"""
723+
home = pathlib.Path.home()
724+
keep = str(pathlib.Path(worktree).resolve())
725+
denied = []
726+
for path in (
727+
home / "git/gmail-archive",
728+
home / "git/monitor",
729+
home / "git/local-llm-testing",
730+
home / "bench-solutions",
731+
home / "bench-logs",
732+
pathlib.Path(repo).expanduser().resolve(),
733+
):
734+
path = str(path)
735+
if path == keep or keep.startswith(path + "/"):
736+
continue # never hide the tree the trial is supposed to edit
737+
if path not in denied:
738+
denied.append(path)
739+
rules = "\n".join(f'(deny file-read* (subpath "{d}"))' for d in denied)
740+
return f"(version 1)\n(allow default)\n{rules}\n", denied
741+
742+
743+
def sandboxed(argv, worktree, repo, tmpdir):
744+
"""Wrap an agent invocation in the sandbox. Returns argv unchanged if the
745+
platform has no sandbox-exec, so this degrades to today's behaviour rather
746+
than silently not running."""
747+
if not pathlib.Path("/usr/bin/sandbox-exec").exists():
748+
logger.warning("no sandbox-exec on this platform; agent runs unconfined")
749+
return argv, []
750+
profile, denied = sandbox_profile(worktree, repo)
751+
path = pathlib.Path(tmpdir) / "confine.sb"
752+
path.write_text(profile)
753+
return ["/usr/bin/sandbox-exec", "-f", str(path), *argv], denied
754+
755+
698756
def save_transcript(client_log, name, stdout, stderr, result, partial=False):
699757
"""Keep the client's own event stream for this trial, if asked to.
700758
@@ -772,6 +830,7 @@ def one_trial(
772830
client_log=None,
773831
solutions=None,
774832
gates=True,
833+
sandbox=True,
775834
):
776835
target = task_target(cfg, task)
777836
repo = pathlib.Path(target["repo"]).expanduser()
@@ -851,8 +910,23 @@ def one_trial(
851910
# 3. Hand it to the agent.
852911
build_argv, parse = CLIENTS[client]
853912
t0 = time.monotonic()
913+
# #54: confine the agent below the client. Its own permission layer
914+
# cannot do this (anomalyco/opencode#41067 submits out-of-worktree
915+
# paths as `../...`, which no pattern matches), so the kernel does.
916+
# sandbox=False only for the integration fixtures, whose stub agent
917+
# "solves" a task by copying from the un-excised reference -- the very
918+
# shortcut this confinement exists to stop.
919+
argv, denied = (
920+
sandboxed(
921+
build_argv(task, backend), worktree, target["repo"], worktree.parent
922+
)
923+
if sandbox
924+
else (build_argv(task, backend), [])
925+
)
926+
if denied:
927+
result["sandbox_denied"] = denied
854928
proc = run(
855-
build_argv(task, backend),
929+
argv,
856930
cwd=worktree,
857931
env=agent_env(backend),
858932
timeout=timeout,

‎benchmarks/agent/test_trial_integration.py‎

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
cleanly without them. Marked `integration`: it runs the oracle twice and both
1616
gates, which is seconds, not milliseconds.
1717
"""
18+
1819
from __future__ import annotations
1920

2021
import pathlib
@@ -36,18 +37,24 @@
3637
def _available() -> bool:
3738
if not REPO.is_dir() or shutil.which("uv") is None:
3839
return False
39-
got = subprocess.run(["git", "cat-file", "-e", f"{CFG['base_commit']}^{{commit}}"],
40-
cwd=REPO, capture_output=True, check=False)
40+
got = subprocess.run(
41+
["git", "cat-file", "-e", f"{CFG['base_commit']}^{{commit}}"],
42+
cwd=REPO,
43+
capture_output=True,
44+
check=False,
45+
)
4146
return got.returncode == 0
4247

4348

44-
needs_repo = pytest.mark.skipif(not _available(),
45-
reason="gmail-archive at the pinned commit, and uv")
49+
needs_repo = pytest.mark.skipif(
50+
not _available(), reason="gmail-archive at the pinned commit, and uv"
51+
)
4652

4753

4854
@pytest.fixture
4955
def scripted_agent(monkeypatch):
5056
"""A client that solves the task by copying the original file back."""
57+
5158
def argv(task, backend):
5259
files = [t["file"] for t in run.targets(task)]
5360
script = (
@@ -67,9 +74,21 @@ def _task(name: str) -> dict:
6774

6875
def _run(task_name, tmp_path, client, **kw):
6976
return run.one_trial(
70-
CFG, _task(task_name), "stub", {"model": "stub", "context_tokens": 1},
71-
trial=1, workdir=tmp_path, timeout=600, dry_run=False,
72-
client=client, solutions=tmp_path / "solutions", **kw,
77+
CFG,
78+
_task(task_name),
79+
"stub",
80+
{"model": "stub", "context_tokens": 1},
81+
trial=1,
82+
workdir=tmp_path,
83+
timeout=600,
84+
dry_run=False,
85+
client=client,
86+
solutions=tmp_path / "solutions",
87+
# The stub agent copies from the un-excised reference repo, which the
88+
# #54 sandbox denies. That denial is correct; it just means these
89+
# plumbing tests have to opt out of it.
90+
sandbox=False,
91+
**kw,
7392
)
7493

7594

@@ -161,6 +180,7 @@ def test_the_agent_is_handed_a_working_environment(monkeypatch, tmp_path):
161180
change moves the control after the agent, or drops `uv run`, this goes red
162181
and the confound becomes real without anyone noticing.
163182
"""
183+
164184
def argv(task, backend):
165185
files = [t["file"] for t in run.targets(task)]
166186
script = (

0 commit comments

Comments
 (0)