|
8 | 8 | import glob |
9 | 9 | import json |
10 | 10 | import os |
| 11 | +import subprocess |
11 | 12 | import tempfile |
12 | 13 | import unittest |
13 | 14 | from dataclasses import replace |
14 | 15 | from unittest import mock |
15 | 16 |
|
16 | | -from skillopt_sleep.backend import MockBackend |
| 17 | +from skillopt_sleep.backend import ClaudeCliBackend, MockBackend |
17 | 18 | from skillopt_sleep.config import load_config |
18 | 19 | from skillopt_sleep.consolidate import consolidate |
19 | 20 | from skillopt_sleep.cycle import run_sleep_cycle |
@@ -112,6 +113,68 @@ def _arithmetic_tasks(): |
112 | 113 | return assign_splits(tasks, holdout_fraction=0.34, seed=42) |
113 | 114 |
|
114 | 115 |
|
| 116 | +def _tool_tasks(): |
| 117 | + """Tasks whose rule judge needs a real tool call (the tool replay route).""" |
| 118 | + tasks = [ |
| 119 | + TaskRecord( |
| 120 | + id=f"look{i}", project="/p", |
| 121 | + intent=f"Find note {i} and summarize it.", |
| 122 | + reference_kind="rule", |
| 123 | + judge={"kind": "rule", "checks": [{"op": "tool_called", "arg": "search"}]}, |
| 124 | + ) |
| 125 | + for i in range(1, 7) |
| 126 | + ] |
| 127 | + return assign_splits(tasks, val_fraction=0.34, seed=42) |
| 128 | + |
| 129 | + |
| 130 | +_REAL_RUN = subprocess.run |
| 131 | +_SEARCH_RULE = "Always run ./search before answering." |
| 132 | + |
| 133 | + |
| 134 | +class SyntheticClaudeProcess: |
| 135 | + """Stands in for the external ``claude`` process; no model is called. |
| 136 | +
|
| 137 | + Records every process the backend spawns as ``(phase, route)``, where the |
| 138 | + route is ``tools`` for ``attempt_with_tools`` and ``text`` for ``_call``. |
| 139 | + Replies with a fixed answer, or raises ``TimeoutExpired`` like a process |
| 140 | + that ran past the backend's timeout. A reflect call (text route during the |
| 141 | + train phase) proposes ``_SEARCH_RULE``; a tool replay whose prompt carries |
| 142 | + that rule logs a call to the ``search`` shim, so its rule judge passes. |
| 143 | + """ |
| 144 | + |
| 145 | + def __init__(self, backend: ClaudeCliBackend, *, timeout: bool = False) -> None: |
| 146 | + self.backend = backend |
| 147 | + self.timeout = timeout |
| 148 | + self.calls: list[tuple[str, str]] = [] |
| 149 | + |
| 150 | + def __call__(self, cmd, **kwargs): |
| 151 | + if not cmd or cmd[0] != self.backend.claude_path: |
| 152 | + return _REAL_RUN(cmd, **kwargs) |
| 153 | + route = "tools" if "--allowedTools" in cmd else "text" |
| 154 | + phase = getattr(self.backend, "evidence_phase", "") |
| 155 | + self.calls.append((phase, route)) |
| 156 | + if self.timeout: |
| 157 | + raise subprocess.TimeoutExpired(cmd, kwargs.get("timeout")) |
| 158 | + prompt = kwargs.get("input") or "" |
| 159 | + if route == "text" and phase == "train": |
| 160 | + reply = json.dumps([{"op": "add", "content": _SEARCH_RULE}]) |
| 161 | + else: |
| 162 | + reply = "Here is the summary." |
| 163 | + if route == "tools" and _SEARCH_RULE in prompt: |
| 164 | + with open(os.path.join(kwargs["cwd"], "_tool_calls.log"), "a") as log: |
| 165 | + log.write("search\n") |
| 166 | + return subprocess.CompletedProcess(cmd, 0, stdout=reply, stderr="") |
| 167 | + |
| 168 | + def phases(self) -> list[str]: |
| 169 | + return [phase for phase, _route in self.calls] |
| 170 | + |
| 171 | + |
| 172 | +def _synthetic_claude(timeout: bool = False): |
| 173 | + backend = ClaudeCliBackend(claude_path="synthetic-claude-for-tests") |
| 174 | + process = SyntheticClaudeProcess(backend, timeout=timeout) |
| 175 | + return backend, process |
| 176 | + |
| 177 | + |
115 | 178 | class TokenBudgetCycleTest(unittest.TestCase): |
116 | 179 | def setUp(self) -> None: |
117 | 180 | self._proj = tempfile.TemporaryDirectory() |
@@ -176,7 +239,10 @@ def test_night_stops_scheduling_work_once_the_budget_is_spent(self): |
176 | 239 | events = read_events(os.path.join(outcome.staging_dir, "evidence.jsonl")) |
177 | 240 | end = [e for e in events if (e["stage"], e["event"]) == ("cycle", "end")] |
178 | 241 | self.assertEqual(end[-1]["max_tokens_per_night"], limit) |
179 | | - self.assertEqual(end[-1]["budget_skipped"], ["skill edits", "memory edits"]) |
| 242 | + self.assertEqual( |
| 243 | + end[-1]["budget_skipped"], |
| 244 | + ["skill edits", "memory edits", "final validation replay"], |
| 245 | + ) |
180 | 246 |
|
181 | 247 | def test_one_token_of_headroom_lets_the_next_step_run(self): |
182 | 248 | tasks = _tasks() |
@@ -463,5 +529,132 @@ def test_groups_not_started_are_skipped_once_the_budget_is_spent(self): |
463 | 529 | self.assertIsNone(second.result) |
464 | 530 |
|
465 | 531 |
|
| 532 | +class TokenBudgetProviderCallTest(unittest.TestCase): |
| 533 | + """Count the provider processes a budget-stopped night really starts. |
| 534 | +
|
| 535 | + These drive ``ClaudeCliBackend`` through its own replay routes, with |
| 536 | + ``subprocess.run`` answered by ``SyntheticClaudeProcess``. Replies the |
| 537 | + response cache does not serve (tool replay, a timed out call) are visible |
| 538 | + as new processes, which a caching mock cannot show. |
| 539 | + """ |
| 540 | + |
| 541 | + def setUp(self) -> None: |
| 542 | + self._proj = tempfile.TemporaryDirectory() |
| 543 | + self._home = tempfile.TemporaryDirectory() |
| 544 | + self.addCleanup(self._proj.cleanup) |
| 545 | + self.addCleanup(self._home.cleanup) |
| 546 | + patcher = mock.patch.dict(os.environ, {"SKILLOPT_SLEEP_WORKERS": "1"}) |
| 547 | + patcher.start() |
| 548 | + self.addCleanup(patcher.stop) |
| 549 | + |
| 550 | + def _night(self, tasks, *, timeout: bool): |
| 551 | + cfg = load_config( |
| 552 | + invoked_project=self._proj.name, |
| 553 | + projects="invoked", |
| 554 | + backend="claude", |
| 555 | + claude_home=os.path.join(self._home.name, ".claude"), |
| 556 | + state_dir=os.path.join(self._home.name, "state"), |
| 557 | + auto_adopt=False, |
| 558 | + ) |
| 559 | + # Any first call spends this, so only the initial replay may run. |
| 560 | + cfg.data["max_tokens_per_night"] = 1 |
| 561 | + backend, process = _synthetic_claude(timeout) |
| 562 | + with mock.patch("skillopt_sleep.backend.subprocess.run", side_effect=process): |
| 563 | + outcome = run_sleep_cycle(cfg, seed_tasks=tasks, backend=backend) |
| 564 | + return process, outcome |
| 565 | + |
| 566 | + def _assert_only_the_initial_replay_ran(self, tasks, process, outcome) -> None: |
| 567 | + n_val = sum(1 for t in tasks if t.split == "val") |
| 568 | + n_train = sum(1 for t in tasks if t.split == "train") |
| 569 | + self.assertGreater(n_val, 0) |
| 570 | + phases = process.phases() |
| 571 | + self.assertEqual(phases.count("final_val"), 0) |
| 572 | + self.assertEqual( |
| 573 | + phases, ["baseline_val"] * n_val + ["train"] * n_train |
| 574 | + ) |
| 575 | + report = outcome.report |
| 576 | + self.assertFalse(report.accepted) |
| 577 | + notes = _budget_notes(report) |
| 578 | + self.assertEqual(len(notes), 1) |
| 579 | + self.assertIn( |
| 580 | + "skipped: skill edits, memory edits, final validation replay", notes[0] |
| 581 | + ) |
| 582 | + events = read_events(os.path.join(outcome.staging_dir, "evidence.jsonl")) |
| 583 | + self.assertFalse([ |
| 584 | + e for e in events |
| 585 | + if (e["stage"], e["event"]) == ("replay", "result") |
| 586 | + and e.get("phase") == "final_val" |
| 587 | + ]) |
| 588 | + skips = [e["step"] for e in events if (e["stage"], e["event"]) == ("budget", "skip")] |
| 589 | + self.assertIn("final validation replay", skips) |
| 590 | + decision = [e for e in events if (e["stage"], e["event"]) == ("gate", "decision")] |
| 591 | + self.assertEqual(len(decision), 1) |
| 592 | + self.assertEqual( |
| 593 | + decision[0]["candidate_score"], decision[0]["baseline_score"] |
| 594 | + ) |
| 595 | + |
| 596 | + def test_tool_route_answers_make_no_final_validation_call(self): |
| 597 | + tasks = _tool_tasks() |
| 598 | + process, outcome = self._night(tasks, timeout=False) |
| 599 | + self.assertEqual({route for _phase, route in process.calls}, {"tools"}) |
| 600 | + self._assert_only_the_initial_replay_ran(tasks, process, outcome) |
| 601 | + |
| 602 | + def test_tool_route_timeouts_make_no_final_validation_call(self): |
| 603 | + tasks = _tool_tasks() |
| 604 | + process, outcome = self._night(tasks, timeout=True) |
| 605 | + self.assertEqual({route for _phase, route in process.calls}, {"tools"}) |
| 606 | + self._assert_only_the_initial_replay_ran(tasks, process, outcome) |
| 607 | + |
| 608 | + def test_text_route_timeouts_make_no_final_validation_call(self): |
| 609 | + # An empty reply from a failed call is not kept in the response cache, |
| 610 | + # so re-scoring the same documents would start the process again. |
| 611 | + tasks = _arithmetic_tasks() |
| 612 | + process, outcome = self._night(tasks, timeout=True) |
| 613 | + self.assertEqual({route for _phase, route in process.calls}, {"text"}) |
| 614 | + self._assert_only_the_initial_replay_ran(tasks, process, outcome) |
| 615 | + |
| 616 | + def test_text_route_answers_stay_free_cache_hits(self): |
| 617 | + # Control: here the final replay would have been served from cache. |
| 618 | + tasks = _arithmetic_tasks() |
| 619 | + process, outcome = self._night(tasks, timeout=False) |
| 620 | + self._assert_only_the_initial_replay_ran(tasks, process, outcome) |
| 621 | + |
| 622 | + |
| 623 | +class TokenBudgetFinalScoringTest(unittest.TestCase): |
| 624 | + """With the budget spent, final scoring reuses the replay of the kept edit.""" |
| 625 | + |
| 626 | + def test_kept_skill_edit_is_scored_from_its_own_validation_replay(self): |
| 627 | + tasks = _tool_tasks() |
| 628 | + n_val = sum(1 for t in tasks if t.split == "val") |
| 629 | + n_train = sum(1 for t in tasks if t.split == "train") |
| 630 | + backend, process = _synthetic_claude() |
| 631 | + |
| 632 | + def spent() -> bool: |
| 633 | + # The budget runs out once the skill step's reflect call started. |
| 634 | + return ("train", "text") in process.calls |
| 635 | + |
| 636 | + with mock.patch.dict(os.environ, {"SKILLOPT_SLEEP_WORKERS": "1"}), \ |
| 637 | + mock.patch("skillopt_sleep.backend.subprocess.run", side_effect=process): |
| 638 | + result = consolidate( |
| 639 | + backend, tasks, "# skill\n", "", budget_exhausted=spent |
| 640 | + ) |
| 641 | + |
| 642 | + self.assertEqual( |
| 643 | + process.calls, |
| 644 | + [("baseline_val", "tools")] * n_val |
| 645 | + + [("train", "tools")] * n_train |
| 646 | + + [("train", "text")] |
| 647 | + + [("gate_trial:skill", "tools")] * n_val, |
| 648 | + ) |
| 649 | + self.assertEqual(result.budget_skipped, ["memory edits", "final validation replay"]) |
| 650 | + self.assertTrue(result.accepted) |
| 651 | + self.assertIn(_SEARCH_RULE, result.new_skill) |
| 652 | + self.assertEqual(result.candidate_score, 1.0) |
| 653 | + trial, final = result.gate_trials |
| 654 | + self.assertEqual(trial["target"], "skill") |
| 655 | + self.assertEqual(final["target"], "final") |
| 656 | + self.assertEqual(final["candidate_score"], trial["candidate_score"]) |
| 657 | + |
| 658 | + |
466 | 659 | if __name__ == "__main__": |
467 | 660 | unittest.main() |
0 commit comments