Skip to content

Commit cddad46

Browse files
anandgupta42claude
andcommitted
test: [#1406] combine topic-switch leak scans; safer workdir reset and baseline default
- topic-switch records combine first- and second-turn leak scans - prepare_workdir deletes only a destination it created (marker + git origin) - run_corrections evaluates a fresh baseline unless one is given explicitly Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 6b034b5 commit cddad46

5 files changed

Lines changed: 174 additions & 8 deletions

File tree

‎experiments/rsi-workspace/demo/prepare_workdir.py‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@
33
44
Copy project/ into a fresh dir, init a repo with a fixed fake remote, commit,
55
and run `dbt seed`. Idempotent: an existing dest previously created by this
6-
script (marker file) is rebuilt from scratch; any other non-empty dest is refused.
6+
script (marker file and matching git repository/origin) is rebuilt from scratch;
7+
any other non-empty dest is refused.
78
Env: DBT_BIN (default documented in README.md). Prints the task prompt.
89
"""
910
import json
@@ -17,6 +18,7 @@
1718
HERE = os.path.dirname(os.path.abspath(__file__))
1819
PROJECT = os.path.join(HERE, "project")
1920
MARKER = ".prepared"
21+
REMOTE = "[email protected]:acme/acme-shop.git"
2022
DBT_BIN = os.environ.get("DBT_BIN") or shutil.which("dbt") or "dbt"
2123
GIT_ENV = {
2224
"GIT_AUTHOR_NAME": "Acme Dev", "GIT_AUTHOR_EMAIL": "[email protected]",
@@ -57,6 +59,18 @@ def prepared_task(dest):
5759
return task_path.is_file() and json.loads(task_path.read_text()).get("id") == task_id
5860

5961

62+
def prepared_workdir(dest):
63+
if not prepared_task(dest):
64+
return False
65+
root = subprocess.run(["git", "rev-parse", "--show-toplevel"], cwd=dest,
66+
capture_output=True, text=True)
67+
if root.returncode or Path(root.stdout.strip()).resolve() != Path(dest).resolve():
68+
return False
69+
origin = subprocess.run(["git", "config", "--local", "--get", "remote.origin.url"], cwd=dest,
70+
capture_output=True, text=True)
71+
return origin.returncode == 0 and origin.stdout.strip() == REMOTE
72+
73+
6074
def main():
6175
if len(sys.argv) != 3:
6276
sys.exit(__doc__)
@@ -72,16 +86,17 @@ def main():
7286
sys.exit(f"unknown task {task_id}")
7387
task = json.load(open(tpath))
7488
if os.path.exists(dest):
75-
if os.path.isdir(dest) and (prepared_task(dest) or not os.listdir(dest)):
89+
if os.path.isdir(dest) and (not os.listdir(dest) or prepared_workdir(dest)):
7690
shutil.rmtree(dest)
7791
else:
78-
sys.exit(f"refusing to overwrite {dest}: not created by prepare_workdir.py")
92+
sys.exit(f"refusing to overwrite {dest}: non-empty destinations require a valid {MARKER} "
93+
f"marker and a git repository rooted here with origin {REMOTE}")
7994
shutil.copytree(PROJECT, dest, ignore=shutil.ignore_patterns("target", "logs", "*.duckdb", "*.duckdb.wal", ".user.yml"))
8095
open(os.path.join(dest, MARKER), "w").write(task_id + "\n")
8196
with open(os.path.join(dest, ".gitignore"), "a") as f:
8297
f.write(MARKER + "\n")
8398
sh(["git", "init", "-q", "-b", "main"], dest)
84-
sh(["git", "remote", "add", "origin", "[email protected]:acme/acme-shop.git"], dest)
99+
sh(["git", "remote", "add", "origin", REMOTE], dest)
85100
sh(["git", "add", "-A"], dest)
86101
sh(["git", "commit", "-q", "-m", "chore: initial acme-shop dbt project"], dest, GIT_ENV)
87102
sh([DBT_BIN, "seed", "--profiles-dir", dest, "--project-dir", dest], dest,

‎experiments/rsi-workspace/harness/run_corrections.sh‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,23 @@
22
# Corrections-only RSI experiment. usage: run_corrections.sh <run_id>
33
# loop (teammate corrections, no verifier in training) -> final arms -> rescore -> report.
44
# env: K (iterations, 2) PARALLEL (4) RUNS (final-arm runs, 3) SPLITS (heldout,control) BACKEND (saas; explicit opt-in required) WORKSPACE_ID (required for saas)
5-
# BASELINE_FROM (default runs/saas-v2/eval/none.jsonl; empty evaluates anew)
5+
# BASELINE_FROM (default empty: evaluate a fresh baseline; explicitly set to an existing file to reuse)
66
# TRAIN_LIMIT (tasks, default all 4) ALTIMATE_CMD DBT_BIN AGENT_MODEL REFLECTOR_MODEL REVIEWER_MODEL
77
set -euo pipefail
88
HERE="$(cd "$(dirname "$0")" && pwd)"
99
cd "$HERE"
1010
source "$HERE/shell_common.sh"
1111
RUN_ID="${1:?usage: run_corrections.sh <run_id>}"
12+
BASELINE_FROM="${BASELINE_FROM:-}"
13+
if [ -n "$BASELINE_FROM" ] && [ ! -f "$BASELINE_FROM" ]; then
14+
echo "BASELINE_FROM file does not exist or is not a regular file: $BASELINE_FROM" >&2
15+
exit 2
16+
fi
1217
WORKSPACE_ID="${WORKSPACE_ID:-}"
1318
backend_args
1419
python3 -c "import common; common.require_learn(); common.require_dbt()"
1520
fresh_run "$RUN_ID"
1621
K="${K:-2}"; PARALLEL="${PARALLEL:-4}"; RUNS="${RUNS:-3}"; SPLITS="${SPLITS:-heldout,control}"
17-
BASELINE_FROM="${BASELINE_FROM-$HERE/runs/saas-v2/eval/none.jsonl}"
1822
RD="$HERE/runs/$RUN_ID"
1923
echo "run dir: $RD"
2024
LIMIT_ARGS=(); [ -n "${TRAIN_LIMIT:-}" ] && LIMIT_ARGS=(--train-limit "$TRAIN_LIMIT")

‎experiments/rsi-workspace/harness/selftest.py‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,54 @@ def test_delete_and_output_containment(self):
5252
self.assertFalse(child.exists())
5353
self.assertTrue(outside.exists())
5454

55+
def test_prepare_workdir_requires_marker_and_repository_provenance(self):
56+
script = str(Path(C.DEMO) / "prepare_workdir.py")
57+
env = dict(os.environ, DBT_BIN=shutil.which("true"), GIT_CONFIG_NOSYSTEM="1",
58+
GIT_CONFIG_GLOBAL=os.devnull)
59+
60+
def git(dest, *args):
61+
subprocess.run(["git", "-C", str(dest), *args], env=env,
62+
capture_output=True, text=True, check=True)
63+
64+
def prepare(dest):
65+
return subprocess.run([sys.executable, script, "train-refunds", str(dest)], env=env,
66+
capture_output=True, text=True)
67+
68+
with tempfile.TemporaryDirectory() as d:
69+
root = Path(d)
70+
for case in ("marker-only", "missing-origin", "wrong-origin", "missing-marker", "nested"):
71+
with self.subTest(case=case):
72+
dest = root / case
73+
dest.mkdir()
74+
if case != "marker-only":
75+
git(dest, "init", "-q")
76+
if case in ("wrong-origin", "missing-marker", "nested"):
77+
remote = "[email protected]:other/project.git" if case == "wrong-origin" else C.REMOTE
78+
git(dest, "remote", "add", "origin", remote)
79+
if case == "nested":
80+
dest = dest / "child"
81+
dest.mkdir()
82+
if case != "missing-marker":
83+
(dest / ".prepared").write_text("train-refunds\n")
84+
sentinel = dest / "keep.txt"
85+
sentinel.write_text("preserve me\n")
86+
result = prepare(dest)
87+
self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)
88+
self.assertIn("refusing to overwrite", result.stderr)
89+
self.assertIn("git repository rooted here with origin", result.stderr)
90+
self.assertEqual(sentinel.read_text(), "preserve me\n")
91+
92+
dest = root / "prepared"
93+
dest.mkdir() # Empty destinations still work.
94+
first = prepare(dest)
95+
self.assertEqual(first.returncode, 0, first.stdout + first.stderr)
96+
sentinel = dest / "discard.txt"
97+
sentinel.write_text("previous run\n")
98+
rebuilt = prepare(dest)
99+
self.assertEqual(rebuilt.returncode, 0, rebuilt.stdout + rebuilt.stderr)
100+
self.assertFalse(sentinel.exists())
101+
self.assertEqual((dest / ".prepared").read_text(), "train-refunds\n")
102+
55103
def test_explicit_fake_and_saas_opt_in(self):
56104
with tempfile.TemporaryDirectory() as d, patch.dict(os.environ, {}, clear=True):
57105
C.setup_users(d)
@@ -329,6 +377,67 @@ def test_shell_ids(self):
329377
capture_output=True, text=True)
330378
self.assertNotEqual(proc.returncode, 0)
331379

380+
def test_corrections_baseline_requires_explicit_reuse(self):
381+
# Exercise the real shell branches without running agents or remote calls.
382+
here = Path(__file__).resolve().parent
383+
with tempfile.TemporaryDirectory() as d:
384+
root = Path(d)
385+
harness = root / "harness"
386+
harness.mkdir()
387+
for name in ("run_corrections.sh", "shell_common.sh"):
388+
shutil.copy(here / name, harness)
389+
historical = harness / "runs/saas-v2/eval/none.jsonl"
390+
historical.parent.mkdir(parents=True)
391+
historical.write_text('{"historical": true}\n')
392+
source = root / "explicit baseline.jsonl"
393+
source.write_text('{"reused": true}\n')
394+
calls = root / "calls.jsonl"
395+
shim = root / "bin"
396+
shim.mkdir()
397+
python = shim / "python3"
398+
python.write_text(f"#!{sys.executable}\n" + '''import json, os, sys
399+
from pathlib import Path
400+
args = sys.argv[1:]
401+
with open(os.environ["CALLS_LOG"], "a") as log:
402+
log.write(json.dumps(args) + "\\n")
403+
if args[0] == "eval.py":
404+
Path(args[args.index("--out") + 1]).write_text('{"fresh": true}\\n')
405+
elif args[0] == "loop_corrections.py":
406+
final = Path(args[args.index("--run-dir") + 1]) / "playbooks/final.md"
407+
final.parent.mkdir(parents=True)
408+
final.write_text("test playbook")
409+
''')
410+
python.chmod(0o755)
411+
for label, baseline in (("unset", None), ("empty", ""), ("explicit", str(source)),
412+
("missing", str(root / "missing.jsonl")), ("directory", str(root))):
413+
with self.subTest(baseline=label):
414+
calls.write_text("")
415+
env = dict(os.environ, PATH=str(shim) + os.pathsep + os.environ["PATH"],
416+
BACKEND="fake", CALLS_LOG=str(calls))
417+
env.pop("BASELINE_FROM", None)
418+
if baseline is not None:
419+
env["BASELINE_FROM"] = baseline
420+
proc = subprocess.run(["bash", str(harness / "run_corrections.sh"), label],
421+
env=env, capture_output=True, text=True)
422+
run = harness / "runs" / label
423+
invocations = C.read_jsonl(str(calls))
424+
if label in ("missing", "directory"):
425+
self.assertNotEqual(proc.returncode, 0)
426+
self.assertIn("BASELINE_FROM file does not exist or is not a regular file", proc.stderr)
427+
self.assertIn(baseline, proc.stderr)
428+
self.assertFalse(run.exists())
429+
self.assertEqual(invocations, [])
430+
continue
431+
self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
432+
baseline_calls = [args for args in invocations if args[0] == "eval.py"
433+
and args[args.index("--arm") + 1] == "none"]
434+
fresh = label in ("unset", "empty")
435+
self.assertEqual(len(baseline_calls), 1 if fresh else 0)
436+
self.assertEqual((run / "eval/none.jsonl").read_text(),
437+
'{"fresh": true}\n' if fresh else source.read_text())
438+
self.assertEqual(historical.read_text(), '{"historical": true}\n')
439+
self.assertEqual(source.read_text(), '{"reused": true}\n')
440+
332441
def test_budget_and_matrix_failures_propagate(self):
333442
# Copy runners and stub Python/sleep: this can never launch a model or wait 30s.
334443
here = Path(__file__).resolve().parent

‎experiments/rsi-workspace/harness/v1bench/selftest.py‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -218,10 +218,13 @@ def test_topic_incomplete_and_resumption(self):
218218
spec = {"run_dir": d, "session": {"id": "test-session"}, "t1": {"id": "one", "prompt": "explain"},
219219
"t2": task, "arm": "none", "run_idx": 0, "arm_obj": lib.parse_arm("none"),
220220
"model": "test/model", "prompt2": "build"}
221-
r1 = {"workdir": d, "session_id": "s1", "completed": False, "timed_out": True}
221+
r1 = {"workdir": d, "session_id": "s1", "completed": False, "timed_out": True,
222+
"leak": True, "leak_hits": [{"pattern": "verifier", "tool": "read", "input": "verifier.py"}]}
222223
with patch.object(C, "run_task", return_value=r1), patch.object(topic, "agent_turn") as turn2:
223224
rec = topic.run_session(spec)
224225
self.assertFalse(rec["pass"])
226+
self.assertTrue(rec["leak"])
227+
self.assertEqual(rec["leak_hits"], {"turn1": r1["leak_hits"], "turn2": []})
225228
turn2.assert_not_called()
226229
r1["completed"] = True
227230
r1["timed_out"] = False
@@ -250,6 +253,38 @@ def test_topic_incomplete_and_resumption(self):
250253
neutral = subprocess.run(topic.NOOP_VERIFY, capture_output=True, text=True, check=True)
251254
self.assertTrue(json.loads(neutral.stdout)["checks"])
252255

256+
def test_topic_leaks_preserve_both_turns(self):
257+
with tempfile.TemporaryDirectory() as d:
258+
os.mkdir(os.path.join(d, "logs"))
259+
task = next(t for t in C.load_tasks().values() if t["split"] == "heldout")
260+
spec = {"run_dir": d, "session": {"id": "test-session"}, "t1": {"id": "one", "prompt": "explain"},
261+
"t2": task, "arm": "none", "run_idx": 0, "arm_obj": lib.parse_arm("none"),
262+
"model": "test/model", "prompt2": "build"}
263+
first_hits = C.leak_scan([{"tool": "read", "input": "gold_playbook.md"}], d)
264+
second_inputs = [{"tool": "read", "input": "verifier.py"}]
265+
second_hits = C.leak_scan(second_inputs, d)
266+
self.assertTrue(first_hits)
267+
self.assertTrue(second_hits)
268+
for turn1_leak, turn2_leak in ((False, False), (True, False), (False, True), (True, True)):
269+
r1 = {"workdir": d, "session_id": "s1", "completed": True,
270+
"leak": turn1_leak, "leak_hits": first_hits if turn1_leak else []}
271+
ev = {"session_id": "s1", "tokens": {}, "cost": 0, "tool_calls": 1, "steps": 1,
272+
"tools": {}, "tool_inputs": second_inputs if turn2_leak else []}
273+
with self.subTest(turn1=turn1_leak, turn2=turn2_leak), \
274+
patch.object(C, "run_task", return_value=r1), \
275+
patch.object(topic, "agent_turn", return_value=(0, False, 123.25, 4.5)), \
276+
patch.object(C, "parse_events", return_value=ev), patch.object(C, "user_env", return_value={}), \
277+
patch.object(C, "agent_completed", return_value=True), \
278+
patch.object(C, "run_verify", return_value={"pass": True, "score": 1.0,
279+
"checks": [{"name": "C1", "ok": True}]}), \
280+
patch.object(C, "leak_scan", wraps=C.leak_scan) as scan:
281+
rec = topic.run_session(spec)
282+
self.assertTrue(rec["completed"])
283+
self.assertEqual(rec["leak"], turn1_leak or turn2_leak)
284+
self.assertEqual(rec["leak_hits"], {"turn1": r1["leak_hits"],
285+
"turn2": second_hits if turn2_leak else []})
286+
scan.assert_called_once_with(ev["tool_inputs"], d)
287+
253288
def test_topic_launch_time_excludes_lock_queue(self):
254289
clock = {"now": 10.0}
255290
class DelayedLock:

‎experiments/rsi-workspace/harness/v1bench/topic_switch/run_topic_switch.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@ def run_session(spec):
9090
"playbook": spec["arm_obj"].token, "model": spec["model"], "env_extra": dict(spec["arm_obj"].env)})
9191
rec = {"session": ses["id"], "request1": t1["id"], "request2": t2["id"], "arm": arm, "run_idx": spec["run_idx"],
9292
"workdir": r1.get("workdir"), "session_id": r1.get("session_id"), "model": spec["model"],
93+
"leak": bool(r1.get("leak")), "leak_hits": {"turn1": r1.get("leak_hits", []), "turn2": []},
9394
"turn1": {k: r1.get(k) for k in ("tokens", "cost", "tool_calls", "steps", "duration", "timed_out",
9495
"skill_loaded", "playbook_in_context", "events", "error", "completed", "agent_rc")}}
9596
if not r1.get("completed") or not r1.get("session_id") or r1.get("error"):
@@ -134,6 +135,8 @@ def run_session(spec):
134135
recall_turn2=(sum(1 for n in need if n in late) / len(need)) if need else None,
135136
in_context_from_turn1=[n for n in need if n in early])
136137
rec["shown"], rec["retrieval"] = shown, m
138+
turn2_hits = C.leak_scan(ev["tool_inputs"], r1["workdir"])
139+
rec["leak_hits"]["turn2"] = turn2_hits
137140
rec.update({
138141
"arm_spec": arm_obj.label, "limits": arm_obj.env,
139142
"completed": completed,
@@ -143,7 +146,7 @@ def run_session(spec):
143146
"turn2": {"tokens": ev["tokens"], "cost": round(ev["cost"], 5), "tool_calls": ev["tool_calls"], "steps": ev["steps"],
144147
"duration": round(duration, 1), "timed_out": timed_out, "rc": rc, "tools": ev["tools"],
145148
"events": os.path.relpath(ev_path, run_dir)},
146-
"leak": bool(C.leak_scan(ev["tool_inputs"], r1["workdir"])),
149+
"leak": rec["leak"] or bool(turn2_hits),
147150
"trace_paths": trace_paths,
148151
})
149152
if not completed:

0 commit comments

Comments
 (0)