|
| 1 | +"""Run the actual baseline shell against a tiny repo and a fake model runner.""" |
| 2 | + |
| 3 | +from __future__ import annotations |
| 4 | + |
| 5 | +import os |
| 6 | +from pathlib import Path |
| 7 | +import subprocess |
| 8 | + |
| 9 | +import pytest |
| 10 | + |
| 11 | +LIBRARY = Path(__file__).resolve().parents[2] / "benchmarks/lib/bench_regression.sh" |
| 12 | +DATASETS = { |
| 13 | + "longmemeval": "longmemeval_s.json", |
| 14 | + "locomo": "locomo10.json", |
| 15 | +} |
| 16 | + |
| 17 | + |
| 18 | +def _git(repo: Path, *args: str) -> str: |
| 19 | + return subprocess.check_output(["git", "-C", str(repo), *args], text=True).strip() |
| 20 | + |
| 21 | + |
| 22 | +def _repository(tmp_path: Path) -> Path: |
| 23 | + repo = tmp_path / "repo" |
| 24 | + repo.mkdir() |
| 25 | + _git(repo, "init", "-q") |
| 26 | + _git( repo, "config", "user.email", "[email protected]") |
| 27 | + _git(repo, "config", "user.name", "Fixture") |
| 28 | + (repo / ".gitignore").write_text("*.json\n.claude/\nresults/\n") |
| 29 | + for name in DATASETS: |
| 30 | + directory = repo / "benchmarks" / name |
| 31 | + directory.mkdir(parents=True) |
| 32 | + (directory / "run_benchmark.py").write_text("# Fake runner reads no code.\n") |
| 33 | + _git(repo, "add", ".") |
| 34 | + _git(repo, "commit", "-qm", "baseline") |
| 35 | + _git(repo, "branch", "baseline") |
| 36 | + (repo / "candidate.txt").write_text("candidate\n") |
| 37 | + _git(repo, "add", "candidate.txt") |
| 38 | + _git(repo, "commit", "-qm", "candidate") |
| 39 | + return repo |
| 40 | + |
| 41 | + |
| 42 | +def _runner(tmp_path: Path) -> Path: |
| 43 | + binary = tmp_path / "bin" |
| 44 | + binary.mkdir() |
| 45 | + uv = binary / "uv" |
| 46 | + uv.write_text("""#!/bin/sh |
| 47 | +set -eu |
| 48 | +for argument do |
| 49 | + case "$argument" in |
| 50 | + */run_benchmark.py) script="$argument" ;; |
| 51 | + esac |
| 52 | +done |
| 53 | +directory="${script%/*}" |
| 54 | +name="${directory##*/}" |
| 55 | +case "$name" in |
| 56 | + longmemeval) filename=longmemeval_s.json ;; |
| 57 | + locomo) filename=locomo10.json ;; |
| 58 | +esac |
| 59 | +cmp "$directory/$filename" "$REPO_ROOT/benchmarks/$name/$filename" |
| 60 | +printf '%s\\n' "$name" >> "$RUNNER_LOG" |
| 61 | +if [ "$FAIL_RUNNER" = 1 ]; then exit 17; fi |
| 62 | +""") |
| 63 | + uv.chmod(0o755) |
| 64 | + return binary |
| 65 | + |
| 66 | + |
| 67 | +def _run(tmp_path: Path, selected: str, *, missing=False, fail=False): |
| 68 | + repo = _repository(tmp_path) |
| 69 | + for name, filename in DATASETS.items(): |
| 70 | + if name in selected.split(",") and not missing: |
| 71 | + (repo / "benchmarks" / name / filename).write_bytes( |
| 72 | + b'["exact", "\xc3\xa9"]\n' |
| 73 | + ) |
| 74 | + binary = _runner(tmp_path) |
| 75 | + environment = dict( |
| 76 | + os.environ, |
| 77 | + PATH=str(binary) + os.pathsep + os.environ["PATH"], |
| 78 | + REPO_ROOT=str(repo), |
| 79 | + RESULTS_DIR=str(repo / "results"), |
| 80 | + BASELINE_REF="baseline", |
| 81 | + ONLY=selected, |
| 82 | + QUICK="0", |
| 83 | + LIMIT="", |
| 84 | + BENCH_DB_URL="unused-fixture", |
| 85 | + RUNNER_LOG=str(tmp_path / "calls"), |
| 86 | + FAIL_RUNNER="1" if fail else "0", |
| 87 | + LIBRARY=str(LIBRARY), |
| 88 | + ) |
| 89 | + result = subprocess.run( |
| 90 | + [ |
| 91 | + "bash", |
| 92 | + "-c", |
| 93 | + """set -euo pipefail |
| 94 | +source "$LIBRARY" |
| 95 | +PASSTHROUGH=() |
| 96 | +want_bench() { case ",$ONLY," in *",$1,"*) return 0;; *) return 1;; esac; } |
| 97 | +run_baseline_benchmarks |
| 98 | +""", |
| 99 | + ], |
| 100 | + env=environment, |
| 101 | + capture_output=True, |
| 102 | + text=True, |
| 103 | + ) |
| 104 | + return result, repo, tmp_path / "calls" |
| 105 | + |
| 106 | + |
| 107 | +@pytest.mark.parametrize("selected", ["longmemeval", "locomo", "longmemeval,locomo"]) |
| 108 | +def test_baseline_uses_exact_selected_ignored_inputs(tmp_path, selected): |
| 109 | + result, repo, calls = _run(tmp_path, selected) |
| 110 | + assert result.returncode == 0, result.stderr |
| 111 | + assert calls.read_text().splitlines() == selected.split(",") |
| 112 | + assert _git(repo, "worktree", "list", "--porcelain").count("worktree ") == 1 |
| 113 | + |
| 114 | + |
| 115 | +def test_missing_dataset_fails_before_runner_and_cleans_worktree(tmp_path): |
| 116 | + result, repo, calls = _run(tmp_path, "longmemeval", missing=True) |
| 117 | + assert result.returncode != 0 |
| 118 | + assert "dataset missing or unreadable" in result.stderr |
| 119 | + assert not calls.exists() |
| 120 | + assert _git(repo, "worktree", "list", "--porcelain").count("worktree ") == 1 |
| 121 | + |
| 122 | + |
| 123 | +def test_first_runner_failure_is_preserved_and_cleans_worktree(tmp_path): |
| 124 | + result, repo, calls = _run(tmp_path, "longmemeval,locomo", fail=True) |
| 125 | + assert result.returncode == 17 |
| 126 | + assert calls.read_text().splitlines() == ["longmemeval"] |
| 127 | + assert _git(repo, "worktree", "list", "--porcelain").count("worktree ") == 1 |
0 commit comments