|
1 | 1 | import argparse |
| 2 | +import errno |
2 | 3 | import json |
3 | 4 | import tempfile |
4 | 5 | import subprocess |
|
15 | 16 | cmd_preflight, |
16 | 17 | cmd_run, |
17 | 18 | ) |
18 | | -from rangebench.env import EnvError, Stage, Task, TaskEnv, _run, image_content_fingerprint |
| 19 | +from rangebench.env import ( |
| 20 | + MAX_COMMAND_BYTES, |
| 21 | + EnvError, |
| 22 | + Stage, |
| 23 | + Task, |
| 24 | + TaskEnv, |
| 25 | + _run, |
| 26 | + image_content_fingerprint, |
| 27 | +) |
19 | 28 | from rangebench.runner import AttemptResult, _maybe_compact |
20 | 29 |
|
21 | 30 |
|
@@ -394,6 +403,40 @@ def binary_output(_cmd: list[str], **kwargs): |
394 | 403 | with patch("rangebench.env.subprocess.run", side_effect=binary_output): |
395 | 404 | self.assertEqual(env.exec("cat binary-file"), (0, "out\ufffd\n[stderr]\nerr\ufffd")) |
396 | 405 |
|
| 406 | + def test_oversized_utf8_command_is_scored_without_spawning_docker(self) -> None: |
| 407 | + task = Task("sample", Path("/tmp"), "web", 1, "Find the flag") |
| 408 | + env = TaskEnv(task, "rb-test") |
| 409 | + command = "é" * (MAX_COMMAND_BYTES // 2 + 1) |
| 410 | + with patch("rangebench.env.subprocess.run") as run: |
| 411 | + rc, output = env.exec(command) |
| 412 | + self.assertEqual(rc, 1) |
| 413 | + self.assertIn("exceeds", output) |
| 414 | + self.assertNotIn(command[:20], output) |
| 415 | + run.assert_not_called() |
| 416 | + |
| 417 | + def test_invalid_utf8_command_is_scored_without_spawning_docker(self) -> None: |
| 418 | + task = Task("sample", Path("/tmp"), "web", 1, "Find the flag") |
| 419 | + env = TaskEnv(task, "rb-test") |
| 420 | + with patch("rangebench.env.subprocess.run") as run: |
| 421 | + self.assertEqual(env.exec("echo \udcff"), (1, "[command contains invalid UTF-8]")) |
| 422 | + run.assert_not_called() |
| 423 | + |
| 424 | + def test_e2big_from_subprocess_is_scored_without_command_content(self) -> None: |
| 425 | + task = Task("sample", Path("/tmp"), "web", 1, "Find the flag") |
| 426 | + env = TaskEnv(task, "rb-test") |
| 427 | + with patch("rangebench.env.subprocess.run", side_effect=OSError(errno.E2BIG, "too long")): |
| 428 | + self.assertEqual( |
| 429 | + env.exec("private-model-command"), |
| 430 | + (1, "[command could not start: argument list too long]"), |
| 431 | + ) |
| 432 | + |
| 433 | + def test_other_subprocess_start_error_invalidates_attempt(self) -> None: |
| 434 | + task = Task("sample", Path("/tmp"), "web", 1, "Find the flag") |
| 435 | + env = TaskEnv(task, "rb-test") |
| 436 | + with patch("rangebench.env.subprocess.run", side_effect=OSError(errno.ENOENT, "missing")): |
| 437 | + with self.assertRaisesRegex(EnvError, "Docker exec could not start"): |
| 438 | + env.exec("echo ok") |
| 439 | + |
397 | 440 | def test_attacker_uses_recorded_image_id(self) -> None: |
398 | 441 | task = Task("sample", Path("/tmp"), "web", 1, "Find the flag") |
399 | 442 | env = TaskEnv(task, "rb-test", "sha256:recorded") |
|
0 commit comments