Skip to content

Commit 5000f75

Browse files
committed
Merge commit 'f8e57919cd9e8813a284a3eb81f172d97243f2f1' into HEAD
2 parents 8cdef40 + f8e5791 commit 5000f75

3 files changed

Lines changed: 164 additions & 0 deletions

File tree

‎benchmarks/lib/bench_regression.sh‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,37 @@ REGRESSION_TOLERANCE="${REGRESSION_TOLERANCE:-0.005}"
4141
BASELINE_REF="${BASELINE_REF:-origin/main}"
4242
BASELINE_RESULTS_DIR=""
4343

44+
# Baseline worktrees omit ignored datasets. Reuse the exact HEAD inputs,
45+
# never a second download which could change the corpus between comparisons.
46+
preflight_regression_datasets() {
47+
local name relative
48+
for name in longmemeval locomo; do
49+
if ! want_bench "$name"; then continue; fi
50+
case "$name" in
51+
longmemeval) relative="benchmarks/longmemeval/longmemeval_s.json" ;;
52+
locomo) relative="benchmarks/locomo/locomo10.json" ;;
53+
esac
54+
if [ ! -r "$REPO_ROOT/$relative" ]; then
55+
echo "error: regression dataset missing or unreadable: $REPO_ROOT/$relative" >&2
56+
return 1
57+
fi
58+
done
59+
}
60+
61+
copy_regression_datasets() {
62+
local destination="$1" name relative
63+
preflight_regression_datasets || return 1
64+
for name in longmemeval locomo; do
65+
if ! want_bench "$name"; then continue; fi
66+
case "$name" in
67+
longmemeval) relative="benchmarks/longmemeval/longmemeval_s.json" ;;
68+
locomo) relative="benchmarks/locomo/locomo10.json" ;;
69+
esac
70+
cp "$REPO_ROOT/$relative" "$destination/$relative" || return 1
71+
cmp -s "$REPO_ROOT/$relative" "$destination/$relative" || return 1
72+
done
73+
}
74+
4475
# Run the same benchmark set (respecting --only/--quick/--limit) against
4576
# BASELINE_REF's code, in an isolated git worktree, writing results under
4677
# $RESULTS_DIR/baseline/. Reuses the already-running container: harnesses
@@ -83,6 +114,10 @@ run_baseline_benchmarks() {
83114
# running it under HEAD's resolved env would not actually measure the
84115
# baseline's code.
85116
(
117+
# Preserve errexit while cleaning up even when provisioning or a runner
118+
# fails. An OR-list around this subshell would disable errexit within it.
119+
trap 'git -C "$REPO_ROOT" worktree remove --force "$wt_dir" >/dev/null 2>&1 || true' EXIT
120+
copy_regression_datasets "$wt_dir"
86121
cd "$wt_dir"
87122
if want_bench longmemeval; then
88123
echo "==> [baseline] longmemeval-s"

‎benchmarks/reproduce.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -472,6 +472,8 @@ main() {
472472
if [ "$RUN_BENCHMARKS" = "1" ] && want_bench longmemeval; then fetch_longmemeval; fi
473473
if [ "$RUN_ABLATION" = "1" ] && [ "$ABLATE_ON" = "longmemeval-s" ]; then fetch_longmemeval; fi
474474

475+
if [ "$NO_REGRESSION" = "1" ]; then preflight_regression_datasets; fi
476+
475477
acquire_lock
476478
trap teardown EXIT
477479
start_db
Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
"""Run the actual baseline shell against a tiny repo and a fake model runner."""
2+
3+
from __future__ import annotations
4+
5+
import os
6+
from pathlib import Path
7+
import subprocess
8+
9+
import pytest
10+
11+
LIBRARY = Path(__file__).resolve().parents[2] / "benchmarks/lib/bench_regression.sh"
12+
DATASETS = {
13+
"longmemeval": "longmemeval_s.json",
14+
"locomo": "locomo10.json",
15+
}
16+
17+
18+
def _git(repo: Path, *args: str) -> str:
19+
return subprocess.check_output(["git", "-C", str(repo), *args], text=True).strip()
20+
21+
22+
def _repository(tmp_path: Path) -> Path:
23+
repo = tmp_path / "repo"
24+
repo.mkdir()
25+
_git(repo, "init", "-q")
26+
_git(repo, "config", "user.email", "[email protected]")
27+
_git(repo, "config", "user.name", "Fixture")
28+
(repo / ".gitignore").write_text("*.json\n.claude/\nresults/\n")
29+
for name in DATASETS:
30+
directory = repo / "benchmarks" / name
31+
directory.mkdir(parents=True)
32+
(directory / "run_benchmark.py").write_text("# Fake runner reads no code.\n")
33+
_git(repo, "add", ".")
34+
_git(repo, "commit", "-qm", "baseline")
35+
_git(repo, "branch", "baseline")
36+
(repo / "candidate.txt").write_text("candidate\n")
37+
_git(repo, "add", "candidate.txt")
38+
_git(repo, "commit", "-qm", "candidate")
39+
return repo
40+
41+
42+
def _runner(tmp_path: Path) -> Path:
43+
binary = tmp_path / "bin"
44+
binary.mkdir()
45+
uv = binary / "uv"
46+
uv.write_text("""#!/bin/sh
47+
set -eu
48+
for argument do
49+
case "$argument" in
50+
*/run_benchmark.py) script="$argument" ;;
51+
esac
52+
done
53+
directory="${script%/*}"
54+
name="${directory##*/}"
55+
case "$name" in
56+
longmemeval) filename=longmemeval_s.json ;;
57+
locomo) filename=locomo10.json ;;
58+
esac
59+
cmp "$directory/$filename" "$REPO_ROOT/benchmarks/$name/$filename"
60+
printf '%s\\n' "$name" >> "$RUNNER_LOG"
61+
if [ "$FAIL_RUNNER" = 1 ]; then exit 17; fi
62+
""")
63+
uv.chmod(0o755)
64+
return binary
65+
66+
67+
def _run(tmp_path: Path, selected: str, *, missing=False, fail=False):
68+
repo = _repository(tmp_path)
69+
for name, filename in DATASETS.items():
70+
if name in selected.split(",") and not missing:
71+
(repo / "benchmarks" / name / filename).write_bytes(
72+
b'["exact", "\xc3\xa9"]\n'
73+
)
74+
binary = _runner(tmp_path)
75+
environment = dict(
76+
os.environ,
77+
PATH=str(binary) + os.pathsep + os.environ["PATH"],
78+
REPO_ROOT=str(repo),
79+
RESULTS_DIR=str(repo / "results"),
80+
BASELINE_REF="baseline",
81+
ONLY=selected,
82+
QUICK="0",
83+
LIMIT="",
84+
BENCH_DB_URL="unused-fixture",
85+
RUNNER_LOG=str(tmp_path / "calls"),
86+
FAIL_RUNNER="1" if fail else "0",
87+
LIBRARY=str(LIBRARY),
88+
)
89+
result = subprocess.run(
90+
[
91+
"bash",
92+
"-c",
93+
"""set -euo pipefail
94+
source "$LIBRARY"
95+
PASSTHROUGH=()
96+
want_bench() { case ",$ONLY," in *",$1,"*) return 0;; *) return 1;; esac; }
97+
run_baseline_benchmarks
98+
""",
99+
],
100+
env=environment,
101+
capture_output=True,
102+
text=True,
103+
)
104+
return result, repo, tmp_path / "calls"
105+
106+
107+
@pytest.mark.parametrize("selected", ["longmemeval", "locomo", "longmemeval,locomo"])
108+
def test_baseline_uses_exact_selected_ignored_inputs(tmp_path, selected):
109+
result, repo, calls = _run(tmp_path, selected)
110+
assert result.returncode == 0, result.stderr
111+
assert calls.read_text().splitlines() == selected.split(",")
112+
assert _git(repo, "worktree", "list", "--porcelain").count("worktree ") == 1
113+
114+
115+
def test_missing_dataset_fails_before_runner_and_cleans_worktree(tmp_path):
116+
result, repo, calls = _run(tmp_path, "longmemeval", missing=True)
117+
assert result.returncode != 0
118+
assert "dataset missing or unreadable" in result.stderr
119+
assert not calls.exists()
120+
assert _git(repo, "worktree", "list", "--porcelain").count("worktree ") == 1
121+
122+
123+
def test_first_runner_failure_is_preserved_and_cleans_worktree(tmp_path):
124+
result, repo, calls = _run(tmp_path, "longmemeval,locomo", fail=True)
125+
assert result.returncode == 17
126+
assert calls.read_text().splitlines() == ["longmemeval"]
127+
assert _git(repo, "worktree", "list", "--porcelain").count("worktree ") == 1

0 commit comments

Comments
 (0)