|
| 1 | +"""Tests for per-skill-group gate reporting (issue #120). |
| 2 | +
|
| 3 | +Pure-stdlib (unittest), deterministic MockBackend, no API key, no network. |
| 4 | +Run: python -m pytest tests/test_sleep_skill_group_report.py |
| 5 | +""" |
| 6 | +from __future__ import annotations |
| 7 | + |
| 8 | +import unittest |
| 9 | + |
| 10 | +from skillopt_sleep.backend import MockBackend |
| 11 | +from skillopt_sleep.consolidate import ConsolidationResult |
| 12 | +from skillopt_sleep.experiments.personas import researcher_persona |
| 13 | +from skillopt_sleep.memory import set_learned |
| 14 | +from skillopt_sleep.mine import assign_splits |
| 15 | +from skillopt_sleep.multi_skill import ( |
| 16 | + CONSOLIDATED, |
| 17 | + FAILED, |
| 18 | + SKIPPED, |
| 19 | + GroupConsolidation, |
| 20 | + SkillGroup, |
| 21 | + consolidate_groups, |
| 22 | + skill_group_reports, |
| 23 | +) |
| 24 | +from skillopt_sleep.types import EditRecord, SkillGroupReport, SleepReport |
| 25 | + |
| 26 | + |
| 27 | +def _tasks(seed=42): |
| 28 | + return assign_splits(researcher_persona(), holdout_fraction=0.34, seed=seed) |
| 29 | + |
| 30 | + |
| 31 | +def _result(accepted, action, baseline, candidate, applied=0, rejected=0): |
| 32 | + return ConsolidationResult( |
| 33 | + accepted=accepted, gate_action=action, baseline_score=baseline, |
| 34 | + candidate_score=candidate, new_skill="", new_memory="", |
| 35 | + applied_edits=[EditRecord("skill", "add", f"rule {i}") for i in range(applied)], |
| 36 | + rejected_edits=[EditRecord("skill", "add", f"bad {i}") for i in range(rejected)], |
| 37 | + holdout_baseline=baseline, holdout_candidate=candidate, |
| 38 | + ) |
| 39 | + |
| 40 | + |
| 41 | +class TestSkillGroupReport(unittest.TestCase): |
| 42 | + def test_row_defaults_are_empty_evidence(self): |
| 43 | + row = SkillGroupReport(skill_name="example-skill") |
| 44 | + self.assertEqual( |
| 45 | + (row.status, row.accepted, row.gate_action, row.baseline_score, |
| 46 | + row.candidate_score, row.n_tasks, row.n_applied_edits, |
| 47 | + row.n_rejected_edits, row.reason), |
| 48 | + ("", False, "", 0.0, 0.0, 0, 0, 0, ""), |
| 49 | + ) |
| 50 | + self.assertEqual(row.to_dict()["skill_name"], "example-skill") |
| 51 | + |
| 52 | + |
| 53 | +class TestSkillGroupRows(unittest.TestCase): |
| 54 | + def test_each_group_keeps_its_own_scores_and_decision(self): |
| 55 | + outcomes = { |
| 56 | + "strong-skill": GroupConsolidation( |
| 57 | + "strong-skill", CONSOLIDATED, n_tasks=6, |
| 58 | + result=_result(True, "accept_new_best", 0.25, 0.75, applied=2), |
| 59 | + ), |
| 60 | + "weak-skill": GroupConsolidation( |
| 61 | + "weak-skill", CONSOLIDATED, n_tasks=3, |
| 62 | + result=_result(False, "reject", 0.5, 0.1, rejected=1), |
| 63 | + ), |
| 64 | + } |
| 65 | + rows = {r.skill_name: r for r in skill_group_reports(outcomes)} |
| 66 | + strong, weak = rows["strong-skill"], rows["weak-skill"] |
| 67 | + self.assertTrue(strong.accepted) |
| 68 | + self.assertEqual((strong.baseline_score, strong.candidate_score), (0.25, 0.75)) |
| 69 | + self.assertEqual((strong.n_tasks, strong.n_applied_edits), (6, 2)) |
| 70 | + self.assertFalse(weak.accepted) |
| 71 | + self.assertEqual((weak.baseline_score, weak.candidate_score), (0.5, 0.1)) |
| 72 | + self.assertEqual((weak.n_applied_edits, weak.n_rejected_edits), (0, 1)) |
| 73 | + self.assertEqual(weak.gate_action, "reject") |
| 74 | + |
| 75 | + def test_rows_follow_first_seen_group_order(self): |
| 76 | + outcomes = { |
| 77 | + "b-skill": GroupConsolidation("b-skill", SKIPPED, reason="no mined tasks"), |
| 78 | + "a-skill": GroupConsolidation("a-skill", SKIPPED, reason="no mined tasks"), |
| 79 | + } |
| 80 | + self.assertEqual([r.skill_name for r in skill_group_reports(outcomes)], |
| 81 | + ["b-skill", "a-skill"]) |
| 82 | + |
| 83 | + def test_skipped_and_failed_rows_carry_reasons_and_no_borrowed_scores(self): |
| 84 | + outcomes = { |
| 85 | + "strong-skill": GroupConsolidation( |
| 86 | + "strong-skill", CONSOLIDATED, n_tasks=6, |
| 87 | + result=_result(True, "accept_new_best", 0.25, 0.75, applied=2), |
| 88 | + ), |
| 89 | + "empty-skill": GroupConsolidation( |
| 90 | + "empty-skill", SKIPPED, reason="no mined tasks for this skill"), |
| 91 | + "broken-skill": GroupConsolidation( |
| 92 | + "broken-skill", FAILED, reason="RuntimeError: backend exploded", n_tasks=4), |
| 93 | + } |
| 94 | + rows = {r.skill_name: r for r in skill_group_reports(outcomes)} |
| 95 | + for name in ("empty-skill", "broken-skill"): |
| 96 | + self.assertFalse(rows[name].accepted) |
| 97 | + self.assertEqual(rows[name].gate_action, "") |
| 98 | + self.assertEqual((rows[name].baseline_score, rows[name].candidate_score), (0.0, 0.0)) |
| 99 | + self.assertTrue(rows[name].reason) |
| 100 | + self.assertEqual(rows["broken-skill"].status, FAILED) |
| 101 | + self.assertEqual(rows["broken-skill"].n_tasks, 4) |
| 102 | + self.assertTrue(rows["strong-skill"].accepted) |
| 103 | + |
| 104 | + def test_rows_from_a_real_mixed_night(self): |
| 105 | + outcomes = consolidate_groups( |
| 106 | + MockBackend(), |
| 107 | + [ |
| 108 | + SkillGroup("research-skill", set_learned("", []), _tasks()), |
| 109 | + SkillGroup("empty-skill", set_learned("", []), []), |
| 110 | + ], |
| 111 | + edit_budget=4, gate_metric="mixed", night=1, |
| 112 | + ) |
| 113 | + rows = skill_group_reports(outcomes) |
| 114 | + self.assertEqual([r.skill_name for r in rows], ["research-skill", "empty-skill"]) |
| 115 | + research = rows[0] |
| 116 | + self.assertEqual(research.status, CONSOLIDATED) |
| 117 | + self.assertTrue(research.accepted) |
| 118 | + self.assertGreater(research.candidate_score, research.baseline_score) |
| 119 | + self.assertEqual(research.n_tasks, len(_tasks())) |
| 120 | + self.assertEqual(rows[1].status, SKIPPED) |
| 121 | + self.assertFalse(rows[1].accepted) |
| 122 | + |
| 123 | + |
| 124 | +class TestSleepReportCompatibility(unittest.TestCase): |
| 125 | + def test_single_skill_report_has_no_group_rows(self): |
| 126 | + report = SleepReport(night=1, project="/repo/example", accepted=True, |
| 127 | + gate_action="accept_new_best") |
| 128 | + self.assertEqual(report.skill_groups, []) |
| 129 | + self.assertEqual(report.to_dict()["skill_groups"], []) |
| 130 | + |
| 131 | + def test_legacy_keyword_construction_still_works(self): |
| 132 | + legacy = { |
| 133 | + "night": 2, "project": "/repo/example", "n_sessions": 1, "n_tasks": 3, |
| 134 | + "baseline_score": 0.1, "candidate_score": 0.2, "accepted": True, |
| 135 | + "gate_action": "accept", "edits": [], "notes": ["ok"], |
| 136 | + } |
| 137 | + report = SleepReport(**legacy) |
| 138 | + self.assertEqual(report.skill_groups, []) |
| 139 | + self.assertEqual(report.gate_action, "accept") |
| 140 | + |
| 141 | + def test_group_rows_serialize_beside_the_single_skill_summary(self): |
| 142 | + report = SleepReport(night=3, project="/repo/example", accepted=True, |
| 143 | + gate_action="accept_new_best", candidate_score=0.75) |
| 144 | + report.skill_groups = skill_group_reports({ |
| 145 | + "research-skill": GroupConsolidation( |
| 146 | + "research-skill", CONSOLIDATED, n_tasks=6, |
| 147 | + result=_result(True, "accept_new_best", 0.25, 0.75, applied=2), |
| 148 | + ), |
| 149 | + }) |
| 150 | + payload = report.to_dict() |
| 151 | + # older consumers read the flat summary and ignore the new key |
| 152 | + self.assertEqual(payload["gate_action"], "accept_new_best") |
| 153 | + self.assertEqual(payload["candidate_score"], 0.75) |
| 154 | + self.assertEqual(payload["skill_groups"][0]["skill_name"], "research-skill") |
| 155 | + self.assertEqual(payload["skill_groups"][0]["n_applied_edits"], 2) |
| 156 | + |
| 157 | + |
| 158 | +if __name__ == "__main__": |
| 159 | + unittest.main() |
0 commit comments