|
| 1 | +"""Tests for per-skill-group consolidation (issue #120). |
| 2 | +
|
| 3 | +Pure-stdlib (unittest), deterministic MockBackend, no API key, no network. |
| 4 | +Run: python -m pytest tests/test_sleep_multi_skill.py |
| 5 | +""" |
| 6 | +from __future__ import annotations |
| 7 | + |
| 8 | +import unittest |
| 9 | + |
| 10 | +from skillopt_sleep.backend import MockBackend |
| 11 | +from skillopt_sleep.consolidate import consolidate |
| 12 | +from skillopt_sleep.experiments.personas import programmer_persona, researcher_persona |
| 13 | +from skillopt_sleep.memory import set_learned |
| 14 | +from skillopt_sleep.mine import assign_splits |
| 15 | +from skillopt_sleep.multi_skill import ( |
| 16 | + CONSOLIDATED, |
| 17 | + FAILED, |
| 18 | + SKIPPED, |
| 19 | + GroupConsolidation, |
| 20 | + SkillGroup, |
| 21 | + accepted_group_skills, |
| 22 | + consolidate_groups, |
| 23 | +) |
| 24 | + |
| 25 | + |
| 26 | +def _tasks(persona, seed=42): |
| 27 | + return assign_splits(persona(), holdout_fraction=0.34, seed=seed) |
| 28 | + |
| 29 | + |
| 30 | +class TestConsolidateGroups(unittest.TestCase): |
| 31 | + def test_no_groups_is_an_empty_mapping(self): |
| 32 | + self.assertEqual(consolidate_groups(MockBackend(), []), {}) |
| 33 | + |
| 34 | + def test_single_group_matches_the_existing_single_skill_run(self): |
| 35 | + skill = set_learned("", []) |
| 36 | + direct = consolidate(MockBackend(), _tasks(researcher_persona), skill, "", |
| 37 | + edit_budget=4, gate_metric="mixed", night=1, |
| 38 | + evolve_memory=False) |
| 39 | + grouped = consolidate_groups( |
| 40 | + MockBackend(), |
| 41 | + [SkillGroup("research-skill", skill, _tasks(researcher_persona))], |
| 42 | + edit_budget=4, gate_metric="mixed", night=1, |
| 43 | + ) |
| 44 | + self.assertEqual(list(grouped), ["research-skill"]) |
| 45 | + outcome = grouped["research-skill"] |
| 46 | + self.assertEqual(outcome.status, CONSOLIDATED) |
| 47 | + self.assertTrue(outcome.accepted) |
| 48 | + self.assertEqual(outcome.result.new_skill, direct.new_skill) |
| 49 | + self.assertEqual(outcome.result.gate_action, direct.gate_action) |
| 50 | + self.assertEqual(outcome.result.candidate_score, direct.candidate_score) |
| 51 | + |
| 52 | + def test_each_group_gets_its_own_decision_and_document(self): |
| 53 | + groups = [ |
| 54 | + SkillGroup("research-skill", set_learned("# research\n", []), |
| 55 | + _tasks(researcher_persona)), |
| 56 | + SkillGroup("programming-skill", set_learned("# programming\n", []), |
| 57 | + _tasks(programmer_persona, seed=1)), |
| 58 | + ] |
| 59 | + outcomes = consolidate_groups(MockBackend(), groups, edit_budget=4, night=1) |
| 60 | + self.assertEqual(list(outcomes), ["research-skill", "programming-skill"]) |
| 61 | + self.assertIn("# research", outcomes["research-skill"].result.new_skill) |
| 62 | + self.assertIn("# programming", outcomes["programming-skill"].result.new_skill) |
| 63 | + self.assertNotIn("# programming", outcomes["research-skill"].result.new_skill) |
| 64 | + |
| 65 | + def test_group_without_tasks_is_skipped_not_fatal(self): |
| 66 | + outcomes = consolidate_groups( |
| 67 | + MockBackend(), |
| 68 | + [ |
| 69 | + SkillGroup("empty-skill", set_learned("", []), []), |
| 70 | + SkillGroup("research-skill", set_learned("", []), _tasks(researcher_persona)), |
| 71 | + ], |
| 72 | + edit_budget=4, night=1, |
| 73 | + ) |
| 74 | + self.assertEqual(outcomes["empty-skill"].status, SKIPPED) |
| 75 | + self.assertIsNone(outcomes["empty-skill"].result) |
| 76 | + self.assertIn("no mined tasks", outcomes["empty-skill"].reason) |
| 77 | + self.assertEqual(outcomes["research-skill"].status, CONSOLIDATED) |
| 78 | + |
| 79 | + def test_group_without_a_name_is_skipped_and_reported(self): |
| 80 | + outcomes = consolidate_groups( |
| 81 | + MockBackend(), |
| 82 | + [SkillGroup(" ", set_learned("", []), _tasks(researcher_persona))], |
| 83 | + edit_budget=4, night=1, |
| 84 | + ) |
| 85 | + self.assertEqual(outcomes[""].status, SKIPPED) |
| 86 | + self.assertIn("no skill name", outcomes[""].reason) |
| 87 | + |
| 88 | + def test_repeated_group_name_runs_once(self): |
| 89 | + calls = [] |
| 90 | + |
| 91 | + def _fake(backend, tasks, skill, memory, **kwargs): |
| 92 | + calls.append(skill) |
| 93 | + return consolidate(backend, tasks, skill, memory, **kwargs) |
| 94 | + |
| 95 | + outcomes = consolidate_groups( |
| 96 | + MockBackend(), |
| 97 | + [ |
| 98 | + SkillGroup("research-skill", set_learned("# first\n", []), |
| 99 | + _tasks(researcher_persona)), |
| 100 | + SkillGroup("research-skill", set_learned("# second\n", []), |
| 101 | + _tasks(researcher_persona)), |
| 102 | + ], |
| 103 | + consolidate_fn=_fake, edit_budget=4, night=1, |
| 104 | + ) |
| 105 | + self.assertEqual(len(outcomes), 1) |
| 106 | + self.assertEqual(len(calls), 1) |
| 107 | + self.assertIn("# first", calls[0]) |
| 108 | + |
| 109 | + def test_one_failing_group_is_isolated_and_reported(self): |
| 110 | + def _fake(backend, tasks, skill, memory, **kwargs): |
| 111 | + if "boom" in skill: |
| 112 | + raise RuntimeError("backend exploded") |
| 113 | + return consolidate(backend, tasks, skill, memory, **kwargs) |
| 114 | + |
| 115 | + outcomes = consolidate_groups( |
| 116 | + MockBackend(), |
| 117 | + [ |
| 118 | + SkillGroup("broken-skill", "boom", _tasks(researcher_persona)), |
| 119 | + SkillGroup("research-skill", set_learned("", []), _tasks(researcher_persona)), |
| 120 | + ], |
| 121 | + consolidate_fn=_fake, edit_budget=4, night=1, |
| 122 | + ) |
| 123 | + self.assertEqual(outcomes["broken-skill"].status, FAILED) |
| 124 | + self.assertIsNone(outcomes["broken-skill"].result) |
| 125 | + self.assertIn("RuntimeError: backend exploded", outcomes["broken-skill"].reason) |
| 126 | + self.assertEqual(outcomes["research-skill"].status, CONSOLIDATED) |
| 127 | + self.assertTrue(outcomes["research-skill"].accepted) |
| 128 | + |
| 129 | + def test_group_runs_do_not_evolve_shared_memory(self): |
| 130 | + seen = {} |
| 131 | + |
| 132 | + def _fake(backend, tasks, skill, memory, **kwargs): |
| 133 | + seen.update(kwargs) |
| 134 | + seen["memory"] = memory |
| 135 | + return consolidate(backend, tasks, skill, memory, **kwargs) |
| 136 | + |
| 137 | + consolidate_groups( |
| 138 | + MockBackend(), |
| 139 | + [SkillGroup("research-skill", set_learned("", []), _tasks(researcher_persona))], |
| 140 | + "# shared memory\n", consolidate_fn=_fake, edit_budget=4, night=1, |
| 141 | + ) |
| 142 | + self.assertEqual(seen["memory"], "# shared memory\n") |
| 143 | + self.assertFalse(seen["evolve_memory"]) |
| 144 | + |
| 145 | + def test_accepted_group_skills_lists_only_accepted_updates(self): |
| 146 | + outcomes = { |
| 147 | + "kept": GroupConsolidation( |
| 148 | + "kept", CONSOLIDATED, |
| 149 | + result=consolidate(MockBackend(), _tasks(researcher_persona), |
| 150 | + set_learned("", []), "", edit_budget=4, night=1), |
| 151 | + ), |
| 152 | + "skipped": GroupConsolidation("skipped", SKIPPED, reason="no mined tasks"), |
| 153 | + "failed": GroupConsolidation("failed", FAILED, reason="RuntimeError: x"), |
| 154 | + } |
| 155 | + self.assertTrue(outcomes["kept"].accepted) |
| 156 | + self.assertEqual(list(accepted_group_skills(outcomes)), ["kept"]) |
| 157 | + self.assertFalse(outcomes["skipped"].accepted) |
| 158 | + self.assertFalse(outcomes["failed"].accepted) |
| 159 | + |
| 160 | + def test_consolidating_groups_writes_no_files(self): |
| 161 | + import os |
| 162 | + import tempfile |
| 163 | + |
| 164 | + with tempfile.TemporaryDirectory() as tmp: |
| 165 | + before = sorted(os.listdir(tmp)) |
| 166 | + consolidate_groups( |
| 167 | + MockBackend(), |
| 168 | + [SkillGroup("research-skill", set_learned("", []), _tasks(researcher_persona))], |
| 169 | + edit_budget=4, night=1, |
| 170 | + ) |
| 171 | + self.assertEqual(sorted(os.listdir(tmp)), before) |
| 172 | + |
| 173 | + |
| 174 | +if __name__ == "__main__": |
| 175 | + unittest.main() |
0 commit comments