Skip to content

Commit 3b5b733

Browse files
anandgupta42claude
andcommitted
test: [#1406] fix two regressions from the previous harness fix
- ablation validates --from-loop before resetting learning state - publish_replace requires a workspace ID only for the SaaS backend Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 5eb1a7f commit 3b5b733

3 files changed

Lines changed: 31 additions & 14 deletions

File tree

‎experiments/rsi-workspace/harness/ablation.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,9 @@ def main():
2929
C.require_learn()
3030
C.require_dbt()
3131
run_dir = C.run_dir_for(a.run_dir, a.run_id)
32+
src = a.from_loop or run_dir # sessions live in <run>/home-a, so reuse only works inside the same run dir
33+
if os.path.abspath(src) != run_dir:
34+
raise SystemExit("--from-loop must be the run dir itself (its sessions are stored in that run's home-a)")
3235
# A retry starts a new learning pass; reflections below still accumulate together.
3336
C.safe_rmtree(os.path.join(run_dir, "work", "maint-ablation"), run_dir)
3437
C.safe_rmtree(os.path.join(run_dir, "learn-history-ablation"), run_dir)
@@ -38,9 +41,6 @@ def main():
3841
C.warm_users(run_dir)
3942
model, reflector = C.resolve_models(run_dir, a.model, a.reflector_model)
4043

41-
src = a.from_loop or run_dir # sessions live in <run>/home-a, so reuse only works inside the same run dir
42-
if os.path.abspath(src) != run_dir:
43-
raise SystemExit("--from-loop must be the run dir itself (its sessions are stored in that run's home-a)")
4444
reuse = {r["task"]: r for r in C.read_jsonl(os.path.join(src, "loop.jsonl"))
4545
if r.get("phase") == "train" and r.get("iter") == 1 and r.get("session_id")
4646
and not r.get("playbook_sha") and r.get("model") == model and r.get("completed")}

‎experiments/rsi-workspace/harness/publish_replace.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
"""Publish/update a run's approved playbook and verify workspace delivery as B.
22
33
Usage: publish_replace.py runs/<run_id> [--backend fake|saas] [--workspace-id ID]
4-
The workspace ID must be supplied by --workspace-id or WORKSPACE_ID.
4+
SaaS needs a workspace ID from --workspace-id or WORKSPACE_ID; fake discovers its seeded binding.
55
SaaS also requires ALLOW_REAL_SAAS=1 and SAAS_CREDS_DIR. No --replace CLI flag is used.
66
"""
77
import argparse
@@ -18,8 +18,8 @@ def main():
1818
ap.add_argument("--backend", choices=["fake", "saas"], default="saas")
1919
ap.add_argument("--workspace-id", type=int, default=os.environ.get("WORKSPACE_ID"))
2020
a = ap.parse_args()
21-
if a.workspace_id is None:
22-
ap.error("provide --workspace-id or set WORKSPACE_ID")
21+
if a.backend == "saas" and a.workspace_id is None:
22+
ap.error("--backend saas requires --workspace-id or WORKSPACE_ID")
2323
run_dir = os.path.abspath(a.run_dir)
2424
maint = os.path.join(run_dir, "work", "maint")
2525
final = open(os.path.join(run_dir, "playbooks", "final.md")).read()

‎experiments/rsi-workspace/harness/selftest.py‎

Lines changed: 25 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -116,14 +116,15 @@ def test_explicit_fake_and_saas_opt_in(self):
116116
self.assertEqual(C.Backend(d, "saas", 4242).workspace_id, 4242)
117117

118118
def test_publish_replace_requires_workspace_id_and_honors_precedence(self):
119-
cases = [(None, None, None), ("4242", None, 4242), (None, "5252", 5252),
120-
("4242", "5252", 5252), ("invalid", "5252", 5252)]
121-
for env_id, cli_id, expected in cases:
122-
with self.subTest(env_id=env_id, cli_id=cli_id), tempfile.TemporaryDirectory() as d, ExitStack() as stack:
119+
missing = object()
120+
cases = [("saas", None, None, missing), ("fake", None, None, None), ("saas", "4242", None, 4242),
121+
("saas", None, "5252", 5252), ("saas", "4242", "5252", 5252), ("fake", "invalid", "5252", 5252)]
122+
for backend_mode, env_id, cli_id, expected in cases:
123+
with self.subTest(backend=backend_mode, env_id=env_id, cli_id=cli_id), tempfile.TemporaryDirectory() as d, ExitStack() as stack:
123124
final = Path(d) / "playbooks/final.md"
124125
final.parent.mkdir()
125126
final.write_text("approved lessons")
126-
argv = ["publish_replace.py", d, "--backend", "fake"]
127+
argv = ["publish_replace.py", d, "--backend", backend_mode]
127128
if cli_id is not None:
128129
argv += ["--workspace-id", cli_id]
129130
stack.enter_context(patch.object(sys, "argv", argv))
@@ -132,17 +133,17 @@ def test_publish_replace_requires_workspace_id_and_honors_precedence(self):
132133
backend = stack.enter_context(patch.object(C, "Backend"))
133134
publish = stack.enter_context(patch.object(PR, "publish"))
134135
error = stack.enter_context(patch.object(sys, "stderr", new_callable=io.StringIO))
135-
if expected is None:
136+
if expected is missing:
136137
with self.assertRaises(SystemExit) as caught:
137138
PR.main()
138139
self.assertNotEqual(caught.exception.code, 0)
139-
self.assertIn("provide --workspace-id or set WORKSPACE_ID", error.getvalue())
140+
self.assertIn("--backend saas requires --workspace-id or WORKSPACE_ID", error.getvalue())
140141
approved.assert_not_called()
141142
backend.assert_not_called()
142143
publish.assert_not_called()
143144
else:
144145
PR.main()
145-
backend.assert_called_once_with(d, "fake", expected)
146+
backend.assert_called_once_with(d, backend_mode, expected)
146147
publish.assert_called_once()
147148

148149
def events(self):
@@ -259,6 +260,22 @@ def test_ablation_rejects_failed_reflection(self):
259260
A.main()
260261
self.assertFalse((Path(d) / "playbook-nofeedback.md").exists())
261262

263+
def test_ablation_rejects_cross_run_reuse_before_resetting_learning(self):
264+
with tempfile.TemporaryDirectory() as d, tempfile.TemporaryDirectory() as other, ExitStack() as stack:
265+
candidate = Path(d) / "work/maint-ablation/candidate.json"
266+
candidate.parent.mkdir(parents=True)
267+
candidate.write_text("[]")
268+
archived = Path(d) / "learn-history-ablation/history/kept.json"
269+
archived.parent.mkdir(parents=True)
270+
archived.write_text("{}")
271+
stack.enter_context(patch.object(sys, "argv", ["ablation.py", "--run-dir", d, "--from-loop", other]))
272+
for name in ("require_learn", "require_dbt"):
273+
stack.enter_context(patch.object(C, name))
274+
with self.assertRaisesRegex(SystemExit, "--from-loop must be the run dir itself"):
275+
A.main()
276+
self.assertTrue(candidate.exists())
277+
self.assertTrue(archived.exists())
278+
262279
def test_ablation_retry_resets_learning_and_preserves_pass_accumulation(self):
263280
with tempfile.TemporaryDirectory() as d, ExitStack() as stack:
264281
maint = Path(d) / "work/maint-ablation"

0 commit comments

Comments
 (0)