Skip to content

Commit af3ad44

Browse files
anandgupta42claude
andcommitted
test: [#1406] reset ablation state only after preflight; scope WORKSPACE_ID to SaaS
- ablation keeps prior learning state when task, user or model setup fails - publish_replace reads WORKSPACE_ID only for the SaaS backend Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 3b5b733 commit af3ad44

3 files changed

Lines changed: 32 additions & 8 deletions

File tree

‎experiments/rsi-workspace/harness/ablation.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,6 @@ def main():
3232
src = a.from_loop or run_dir # sessions live in <run>/home-a, so reuse only works inside the same run dir
3333
if os.path.abspath(src) != run_dir:
3434
raise SystemExit("--from-loop must be the run dir itself (its sessions are stored in that run's home-a)")
35-
# A retry starts a new learning pass; reflections below still accumulate together.
36-
C.safe_rmtree(os.path.join(run_dir, "work", "maint-ablation"), run_dir)
37-
C.safe_rmtree(os.path.join(run_dir, "learn-history-ablation"), run_dir)
3835
log_p = os.path.join(run_dir, "ablation.jsonl")
3936
train = C.select_tasks(["train"], a.train_limit)
4037
C.setup_users(run_dir) # local arm; never needs real workspace credentials
@@ -51,6 +48,9 @@ def main():
5148
specs = [{"run_dir": run_dir, "task": t, "arm": "ablation-train", "user": "a", "run_idx": 0, "playbook": None,
5249
"workspace": False, "model": model, "tag": "abl-"} for t in train]
5350
recs = C.run_many(specs, a.parallel, on_done=lambda r: C.append_jsonl(log_p, dict(r, phase="train")))
51+
# Reset only after inputs, models and trajectories are ready; reflections below still accumulate together.
52+
C.safe_rmtree(os.path.join(run_dir, "work", "maint-ablation"), run_dir)
53+
C.safe_rmtree(os.path.join(run_dir, "learn-history-ablation"), run_dir)
5454
maint = L.maintainer(run_dir, "maint-ablation")
5555
_, cand_p = L.learn_paths(maint)
5656
for r in recs:

‎experiments/rsi-workspace/harness/publish_replace.py‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
"""Publish/update a run's approved playbook and verify workspace delivery as B.
22
33
Usage: publish_replace.py runs/<run_id> [--backend fake|saas] [--workspace-id ID]
4-
SaaS needs a workspace ID from --workspace-id or WORKSPACE_ID; fake discovers its seeded binding.
4+
SaaS needs a workspace ID from --workspace-id or WORKSPACE_ID; fake ignores WORKSPACE_ID and discovers its seeded binding.
55
SaaS also requires ALLOW_REAL_SAAS=1 and SAAS_CREDS_DIR. No --replace CLI flag is used.
66
"""
77
import argparse
@@ -16,10 +16,15 @@ def main():
1616
ap = argparse.ArgumentParser(description=__doc__)
1717
ap.add_argument("run_dir")
1818
ap.add_argument("--backend", choices=["fake", "saas"], default="saas")
19-
ap.add_argument("--workspace-id", type=int, default=os.environ.get("WORKSPACE_ID"))
19+
ap.add_argument("--workspace-id", type=int)
2020
a = ap.parse_args()
2121
if a.backend == "saas" and a.workspace_id is None:
22-
ap.error("--backend saas requires --workspace-id or WORKSPACE_ID")
22+
env_id = os.environ.get("WORKSPACE_ID")
23+
if not env_id:
24+
ap.error("--backend saas requires --workspace-id or WORKSPACE_ID")
25+
if not env_id.isdigit():
26+
ap.error("WORKSPACE_ID must be a numeric workspace ID")
27+
a.workspace_id = int(env_id)
2328
run_dir = os.path.abspath(a.run_dir)
2429
maint = os.path.join(run_dir, "work", "maint")
2530
final = open(os.path.join(run_dir, "playbooks", "final.md")).read()

‎experiments/rsi-workspace/harness/selftest.py‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,8 @@ def test_explicit_fake_and_saas_opt_in(self):
118118
def test_publish_replace_requires_workspace_id_and_honors_precedence(self):
119119
missing = object()
120120
cases = [("saas", None, None, missing), ("fake", None, None, None), ("saas", "4242", None, 4242),
121-
("saas", None, "5252", 5252), ("saas", "4242", "5252", 5252), ("fake", "invalid", "5252", 5252)]
121+
("saas", None, "5252", 5252), ("saas", "4242", "5252", 5252), ("fake", "invalid", "5252", 5252),
122+
("fake", "9999", None, None), ("fake", "invalid", None, None), ("saas", "invalid", None, missing)]
122123
for backend_mode, env_id, cli_id, expected in cases:
123124
with self.subTest(backend=backend_mode, env_id=env_id, cli_id=cli_id), tempfile.TemporaryDirectory() as d, ExitStack() as stack:
124125
final = Path(d) / "playbooks/final.md"
@@ -137,7 +138,7 @@ def test_publish_replace_requires_workspace_id_and_honors_precedence(self):
137138
with self.assertRaises(SystemExit) as caught:
138139
PR.main()
139140
self.assertNotEqual(caught.exception.code, 0)
140-
self.assertIn("--backend saas requires --workspace-id or WORKSPACE_ID", error.getvalue())
141+
self.assertIn("WORKSPACE_ID", error.getvalue())
141142
approved.assert_not_called()
142143
backend.assert_not_called()
143144
publish.assert_not_called()
@@ -276,6 +277,24 @@ def test_ablation_rejects_cross_run_reuse_before_resetting_learning(self):
276277
self.assertTrue(candidate.exists())
277278
self.assertTrue(archived.exists())
278279

280+
def test_ablation_keeps_prior_learning_when_preflight_fails(self):
281+
with tempfile.TemporaryDirectory() as d, ExitStack() as stack:
282+
candidate = Path(d) / "work/maint-ablation/candidate.json"
283+
candidate.parent.mkdir(parents=True)
284+
candidate.write_text("[]")
285+
archived = Path(d) / "learn-history-ablation/history/kept.json"
286+
archived.parent.mkdir(parents=True)
287+
archived.write_text("{}")
288+
stack.enter_context(patch.object(sys, "argv", ["ablation.py", "--run-dir", d]))
289+
for name in ("require_learn", "require_dbt", "setup_users", "warm_users"):
290+
stack.enter_context(patch.object(C, name))
291+
stack.enter_context(patch.object(C, "select_tasks", return_value=[{"id": "train-test"}]))
292+
stack.enter_context(patch.object(C, "resolve_models", side_effect=SystemExit("model unavailable")))
293+
with self.assertRaisesRegex(SystemExit, "model unavailable"):
294+
A.main()
295+
self.assertTrue(candidate.exists())
296+
self.assertTrue(archived.exists())
297+
279298
def test_ablation_retry_resets_learning_and_preserves_pass_accumulation(self):
280299
with tempfile.TemporaryDirectory() as d, ExitStack() as stack:
281300
maint = Path(d) / "work/maint-ablation"

0 commit comments

Comments
 (0)