Skip to content
This repository was archived by the owner on Oct 2, 2026. It is now read-only.

Commit e30a237

Browse files
authored
Merge pull request #8 from lbgos/fix/oversized-command
Score oversized model commands without aborting runs
2 parents 5ecd2ea + 040b690 commit e30a237

4 files changed

Lines changed: 110 additions & 3 deletions

File tree

‎rangebench/env.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
from __future__ import annotations
44

5+
import errno
56
import hashlib
67
import json
78
import re
@@ -14,6 +15,9 @@
1415
ROOT = Path(__file__).resolve().parent.parent
1516
TASKS_DIR = ROOT / "tasks"
1617
ATTACKER_IMAGE = "rb-attacker:latest"
18+
# Linux limits each argv string to 128 KiB including its terminator. Leave
19+
# room for platform differences and report model-generated excess as a command failure.
20+
MAX_COMMAND_BYTES = 120_000
1721
COMPOSE_PROJECT_LABEL = "com.docker.compose.project"
1822
# Docker's inspect API has added and removed non-image Config fields across versions.
1923
# Keep the fields that belong to the image configuration.
@@ -381,6 +385,13 @@ def exec(
381385
the attempt if Docker still has not returned.
382386
"""
383387

388+
try:
389+
command_bytes = len(cmd.encode("utf-8"))
390+
except UnicodeEncodeError:
391+
return 1, "[command contains invalid UTF-8]"
392+
if command_bytes > MAX_COMMAND_BYTES:
393+
return 1, f"[command exceeds {MAX_COMMAND_BYTES} UTF-8 bytes]"
394+
384395
def attacker_available() -> bool:
385396
try:
386397
probe = subprocess.run(
@@ -393,6 +404,8 @@ def attacker_available() -> bool:
393404
)
394405
except subprocess.TimeoutExpired as exc:
395406
raise EnvError("Docker exec probe timed out") from exc
407+
except OSError as exc:
408+
raise EnvError(f"Docker exec probe could not start (errno {exc.errno})") from exc
396409
return probe.returncode == 0
397410

398411
try:
@@ -449,6 +462,10 @@ def attacker_available() -> bool:
449462
if not attacker_available():
450463
raise EnvError("Docker exec unavailable after attacker command timeout") from exc
451464
raise EnvError("Docker exec did not finish after attacker command timeout") from exc
465+
except OSError as exc:
466+
if exc.errno == errno.E2BIG:
467+
return 1, "[command could not start: argument list too long]"
468+
raise EnvError(f"Docker exec could not start (errno {exc.errno})") from exc
452469

453470
def read_flag(self, stage: Stage) -> str:
454471
compose = [

‎rangebench/runner.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -539,7 +539,9 @@ def run_attempt(
539539

540540
def emit(kind: str, **kv: Any) -> None:
541541
rec: dict[str, Any] = {"t": round(time.time() - t0, 1), "kind": kind, **kv}
542-
log.write(json.dumps(rec, ensure_ascii=False) + "\n")
542+
# Model output may contain lone Unicode surrogates. Escape them so a
543+
# malformed command can receive an ordinary observation below.
544+
log.write(json.dumps(rec, ensure_ascii=True) + "\n")
543545
log.flush()
544546

545547
try:

‎tests/test_accounting.py‎

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import argparse
2+
import errno
23
import json
34
import tempfile
45
import subprocess
@@ -15,7 +16,15 @@
1516
cmd_preflight,
1617
cmd_run,
1718
)
18-
from rangebench.env import EnvError, Stage, Task, TaskEnv, _run, image_content_fingerprint
19+
from rangebench.env import (
20+
MAX_COMMAND_BYTES,
21+
EnvError,
22+
Stage,
23+
Task,
24+
TaskEnv,
25+
_run,
26+
image_content_fingerprint,
27+
)
1928
from rangebench.runner import AttemptResult, _maybe_compact
2029

2130

@@ -394,6 +403,40 @@ def binary_output(_cmd: list[str], **kwargs):
394403
with patch("rangebench.env.subprocess.run", side_effect=binary_output):
395404
self.assertEqual(env.exec("cat binary-file"), (0, "out\ufffd\n[stderr]\nerr\ufffd"))
396405

406+
def test_oversized_utf8_command_is_scored_without_spawning_docker(self) -> None:
407+
task = Task("sample", Path("/tmp"), "web", 1, "Find the flag")
408+
env = TaskEnv(task, "rb-test")
409+
command = "é" * (MAX_COMMAND_BYTES // 2 + 1)
410+
with patch("rangebench.env.subprocess.run") as run:
411+
rc, output = env.exec(command)
412+
self.assertEqual(rc, 1)
413+
self.assertIn("exceeds", output)
414+
self.assertNotIn(command[:20], output)
415+
run.assert_not_called()
416+
417+
def test_invalid_utf8_command_is_scored_without_spawning_docker(self) -> None:
418+
task = Task("sample", Path("/tmp"), "web", 1, "Find the flag")
419+
env = TaskEnv(task, "rb-test")
420+
with patch("rangebench.env.subprocess.run") as run:
421+
self.assertEqual(env.exec("echo \udcff"), (1, "[command contains invalid UTF-8]"))
422+
run.assert_not_called()
423+
424+
def test_e2big_from_subprocess_is_scored_without_command_content(self) -> None:
425+
task = Task("sample", Path("/tmp"), "web", 1, "Find the flag")
426+
env = TaskEnv(task, "rb-test")
427+
with patch("rangebench.env.subprocess.run", side_effect=OSError(errno.E2BIG, "too long")):
428+
self.assertEqual(
429+
env.exec("private-model-command"),
430+
(1, "[command could not start: argument list too long]"),
431+
)
432+
433+
def test_other_subprocess_start_error_invalidates_attempt(self) -> None:
434+
task = Task("sample", Path("/tmp"), "web", 1, "Find the flag")
435+
env = TaskEnv(task, "rb-test")
436+
with patch("rangebench.env.subprocess.run", side_effect=OSError(errno.ENOENT, "missing")):
437+
with self.assertRaisesRegex(EnvError, "Docker exec could not start"):
438+
env.exec("echo ok")
439+
397440
def test_attacker_uses_recorded_image_id(self) -> None:
398441
task = Task("sample", Path("/tmp"), "web", 1, "Find the flag")
399442
env = TaskEnv(task, "rb-test", "sha256:recorded")

‎tests/test_runner.py‎

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
from unittest.mock import patch
77

88
from rangebench.agent import Usage
9-
from rangebench.env import ATTACKER_IMAGE, EnvError, Stage, Task
9+
from rangebench.env import ATTACKER_IMAGE, EnvError, Stage, Task, TaskEnv
1010
from rangebench.runner import run_attempt, run_oracle
1111

1212

@@ -41,6 +41,51 @@ def chat(self, *_args: object, **_kwargs: object) -> None:
4141

4242

4343
class RunnerTests(unittest.TestCase):
44+
def test_unencodable_model_command_gets_observation_without_aborting(self) -> None:
45+
class GuardedEnv(TaskEnv):
46+
def up(self) -> None:
47+
self.service_image_ids = {"target": "sha256:" + "a" * 64}
48+
self.service_image_fingerprints = {"target": "sha256:" + "b" * 64}
49+
50+
def read_flag(self, stage: Stage) -> str:
51+
return "flag{test}"
52+
53+
def down(self) -> tuple[bool, str]:
54+
return True, ""
55+
56+
class SurrogateClient:
57+
def __init__(self) -> None:
58+
self.calls = 0
59+
60+
def chat(
61+
self, _messages: list[dict], max_tokens: int, temperature: float = 0.2
62+
) -> tuple[str, Usage, None]:
63+
self.calls += 1
64+
content = "COMMAND:\necho \udcff" if self.calls == 1 else "ANSWER: flag{test}"
65+
return content, Usage(prompt_tokens=20, completion_tokens=10), None
66+
67+
with (
68+
tempfile.TemporaryDirectory() as tmp,
69+
patch("rangebench.runner.TaskEnv", GuardedEnv),
70+
patch("rangebench.env.subprocess.run") as run,
71+
):
72+
task = Task(
73+
"sample", Path(tmp), "web", 1, "Find the flag",
74+
stages=[Stage("one", "/flag", "target")], turns=2,
75+
)
76+
result = run_attempt(SurrogateClient(), task, 1, "rb-test", Path(tmp), verbose=False)
77+
raw = (Path(tmp) / "sample-t1.jsonl").read_text()
78+
records = [json.loads(line) for line in raw.splitlines()]
79+
80+
self.assertEqual(result.end_reason, "all stages captured")
81+
self.assertEqual(result.commands, 1)
82+
self.assertIn("\\udcff", raw)
83+
self.assertEqual(
84+
next(record for record in records if record["kind"] == "exec")["out"],
85+
"[command contains invalid UTF-8]",
86+
)
87+
run.assert_not_called()
88+
4489
def test_keep_writes_event_before_closing_log(self) -> None:
4590
with tempfile.TemporaryDirectory() as tmp, patch("rangebench.runner.TaskEnv", FakeEnv):
4691
task = Task("sample", Path(tmp), "web", 1, "Find the flag", stages=[Stage("one", "/flag", "target")])

0 commit comments

Comments
 (0)