Skip to content

Commit 8e78ba0

Browse files
feat(sleep): report per-skill-group gate decisions
Add SkillGroupReport plus SleepReport.skill_groups (default empty) and build the rows from each group's own baseline, candidate score, decision, and reason, so a weak group can neither block nor borrow a strong group's evidence and older single-group report consumers keep working. Refs microsoft#120
1 parent 9639719 commit 8e78ba0

3 files changed

Lines changed: 226 additions & 4 deletions

File tree

‎skillopt_sleep/multi_skill.py‎

Lines changed: 39 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515

1616
from skillopt_sleep.backend import Backend
1717
from skillopt_sleep.consolidate import ConsolidationResult, consolidate
18-
from skillopt_sleep.types import TaskRecord
18+
from skillopt_sleep.types import SkillGroupReport, TaskRecord
1919

2020
CONSOLIDATED = "consolidated"
2121
SKIPPED = "skipped"
@@ -39,6 +39,7 @@ class GroupConsolidation:
3939
status: str
4040
result: Optional[ConsolidationResult] = None
4141
reason: str = ""
42+
n_tasks: int = 0
4243

4344
@property
4445
def accepted(self) -> bool:
@@ -83,7 +84,9 @@ def consolidate_groups(
8384
if name in out:
8485
continue # first group wins; a repeated name is not a second night
8586
if not group.tasks:
86-
out[name] = GroupConsolidation(name, SKIPPED, reason="no mined tasks for this skill")
87+
out[name] = GroupConsolidation(
88+
name, SKIPPED, reason="no mined tasks for this skill"
89+
)
8790
continue
8891
try:
8992
result = consolidate_fn(
@@ -92,13 +95,45 @@ def consolidate_groups(
9295
)
9396
except Exception as exc: # one group's failure must not abort the night
9497
out[name] = GroupConsolidation(
95-
name, FAILED, reason=f"{type(exc).__name__}: {exc}"[:300]
98+
name, FAILED, reason=f"{type(exc).__name__}: {exc}"[:300],
99+
n_tasks=len(group.tasks),
96100
)
97101
continue
98-
out[name] = GroupConsolidation(name, CONSOLIDATED, result=result)
102+
out[name] = GroupConsolidation(
103+
name, CONSOLIDATED, result=result, n_tasks=len(group.tasks)
104+
)
99105
return out
100106

101107

108+
def skill_group_reports(
109+
outcomes: Dict[str, GroupConsolidation],
110+
) -> List[SkillGroupReport]:
111+
"""Build one report row per group, from that group's evidence only.
112+
113+
A skipped or failed group reports its reason and keeps zeroed scores rather
114+
than inheriting another group's numbers, and an accepted group's row is
115+
unaffected by its neighbours' decisions.
116+
"""
117+
rows: List[SkillGroupReport] = []
118+
for name, outcome in outcomes.items():
119+
row = SkillGroupReport(
120+
skill_name=name,
121+
status=outcome.status,
122+
reason=outcome.reason,
123+
n_tasks=outcome.n_tasks,
124+
)
125+
result = outcome.result
126+
if result is not None:
127+
row.accepted = result.accepted
128+
row.gate_action = result.gate_action
129+
row.baseline_score = result.baseline_score
130+
row.candidate_score = result.candidate_score
131+
row.n_applied_edits = len(result.applied_edits)
132+
row.n_rejected_edits = len(result.rejected_edits)
133+
rows.append(row)
134+
return rows
135+
136+
102137
def accepted_group_skills(outcomes: Dict[str, GroupConsolidation]) -> Dict[str, str]:
103138
"""Skill name -> new document, for groups whose gate accepted an update."""
104139
return {

‎skillopt_sleep/types.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,30 @@ class EditRecord:
128128
rationale: str = ""
129129

130130

131+
@dataclass
132+
class SkillGroupReport:
133+
"""One skill group's own gate evidence for the night.
134+
135+
Every field describes that group alone: its baseline, its candidate score,
136+
its decision, and why. Groups never share rows, so a weak group cannot
137+
borrow a strong group's evidence or block its acceptance.
138+
"""
139+
140+
skill_name: str
141+
status: str = "" # consolidated | skipped | failed
142+
accepted: bool = False
143+
gate_action: str = ""
144+
baseline_score: float = 0.0
145+
candidate_score: float = 0.0
146+
n_tasks: int = 0
147+
n_applied_edits: int = 0
148+
n_rejected_edits: int = 0
149+
reason: str = ""
150+
151+
def to_dict(self) -> Dict[str, Any]:
152+
return asdict(self)
153+
154+
131155
@dataclass
132156
class SleepReport:
133157
"""Everything one night produced — written to staging for review."""
@@ -154,6 +178,10 @@ class SleepReport:
154178
# list above — without them a night can report no edits while the optimizer
155179
# actually produced several.
156180
unmatched_edits: List[EditRecord] = field(default_factory=list)
181+
# Per-skill gate rows for a multi-skill night, in first-seen order. Empty on
182+
# a single-managed-skill night, so the fields above remain the whole story
183+
# for consumers that predate skill groups.
184+
skill_groups: List[SkillGroupReport] = field(default_factory=list)
157185
tokens_used: int = 0
158186
notes: List[str] = field(default_factory=list)
159187

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
"""Tests for per-skill-group gate reporting (issue #120).
2+
3+
Pure-stdlib (unittest), deterministic MockBackend, no API key, no network.
4+
Run: python -m pytest tests/test_sleep_skill_group_report.py
5+
"""
6+
from __future__ import annotations
7+
8+
import unittest
9+
10+
from skillopt_sleep.backend import MockBackend
11+
from skillopt_sleep.consolidate import ConsolidationResult
12+
from skillopt_sleep.experiments.personas import researcher_persona
13+
from skillopt_sleep.memory import set_learned
14+
from skillopt_sleep.mine import assign_splits
15+
from skillopt_sleep.multi_skill import (
16+
CONSOLIDATED,
17+
FAILED,
18+
SKIPPED,
19+
GroupConsolidation,
20+
SkillGroup,
21+
consolidate_groups,
22+
skill_group_reports,
23+
)
24+
from skillopt_sleep.types import EditRecord, SkillGroupReport, SleepReport
25+
26+
27+
def _tasks(seed=42):
28+
return assign_splits(researcher_persona(), holdout_fraction=0.34, seed=seed)
29+
30+
31+
def _result(accepted, action, baseline, candidate, applied=0, rejected=0):
32+
return ConsolidationResult(
33+
accepted=accepted, gate_action=action, baseline_score=baseline,
34+
candidate_score=candidate, new_skill="", new_memory="",
35+
applied_edits=[EditRecord("skill", "add", f"rule {i}") for i in range(applied)],
36+
rejected_edits=[EditRecord("skill", "add", f"bad {i}") for i in range(rejected)],
37+
holdout_baseline=baseline, holdout_candidate=candidate,
38+
)
39+
40+
41+
class TestSkillGroupReport(unittest.TestCase):
42+
def test_row_defaults_are_empty_evidence(self):
43+
row = SkillGroupReport(skill_name="example-skill")
44+
self.assertEqual(
45+
(row.status, row.accepted, row.gate_action, row.baseline_score,
46+
row.candidate_score, row.n_tasks, row.n_applied_edits,
47+
row.n_rejected_edits, row.reason),
48+
("", False, "", 0.0, 0.0, 0, 0, 0, ""),
49+
)
50+
self.assertEqual(row.to_dict()["skill_name"], "example-skill")
51+
52+
53+
class TestSkillGroupRows(unittest.TestCase):
54+
def test_each_group_keeps_its_own_scores_and_decision(self):
55+
outcomes = {
56+
"strong-skill": GroupConsolidation(
57+
"strong-skill", CONSOLIDATED, n_tasks=6,
58+
result=_result(True, "accept_new_best", 0.25, 0.75, applied=2),
59+
),
60+
"weak-skill": GroupConsolidation(
61+
"weak-skill", CONSOLIDATED, n_tasks=3,
62+
result=_result(False, "reject", 0.5, 0.1, rejected=1),
63+
),
64+
}
65+
rows = {r.skill_name: r for r in skill_group_reports(outcomes)}
66+
strong, weak = rows["strong-skill"], rows["weak-skill"]
67+
self.assertTrue(strong.accepted)
68+
self.assertEqual((strong.baseline_score, strong.candidate_score), (0.25, 0.75))
69+
self.assertEqual((strong.n_tasks, strong.n_applied_edits), (6, 2))
70+
self.assertFalse(weak.accepted)
71+
self.assertEqual((weak.baseline_score, weak.candidate_score), (0.5, 0.1))
72+
self.assertEqual((weak.n_applied_edits, weak.n_rejected_edits), (0, 1))
73+
self.assertEqual(weak.gate_action, "reject")
74+
75+
def test_rows_follow_first_seen_group_order(self):
76+
outcomes = {
77+
"b-skill": GroupConsolidation("b-skill", SKIPPED, reason="no mined tasks"),
78+
"a-skill": GroupConsolidation("a-skill", SKIPPED, reason="no mined tasks"),
79+
}
80+
self.assertEqual([r.skill_name for r in skill_group_reports(outcomes)],
81+
["b-skill", "a-skill"])
82+
83+
def test_skipped_and_failed_rows_carry_reasons_and_no_borrowed_scores(self):
84+
outcomes = {
85+
"strong-skill": GroupConsolidation(
86+
"strong-skill", CONSOLIDATED, n_tasks=6,
87+
result=_result(True, "accept_new_best", 0.25, 0.75, applied=2),
88+
),
89+
"empty-skill": GroupConsolidation(
90+
"empty-skill", SKIPPED, reason="no mined tasks for this skill"),
91+
"broken-skill": GroupConsolidation(
92+
"broken-skill", FAILED, reason="RuntimeError: backend exploded", n_tasks=4),
93+
}
94+
rows = {r.skill_name: r for r in skill_group_reports(outcomes)}
95+
for name in ("empty-skill", "broken-skill"):
96+
self.assertFalse(rows[name].accepted)
97+
self.assertEqual(rows[name].gate_action, "")
98+
self.assertEqual((rows[name].baseline_score, rows[name].candidate_score), (0.0, 0.0))
99+
self.assertTrue(rows[name].reason)
100+
self.assertEqual(rows["broken-skill"].status, FAILED)
101+
self.assertEqual(rows["broken-skill"].n_tasks, 4)
102+
self.assertTrue(rows["strong-skill"].accepted)
103+
104+
def test_rows_from_a_real_mixed_night(self):
105+
outcomes = consolidate_groups(
106+
MockBackend(),
107+
[
108+
SkillGroup("research-skill", set_learned("", []), _tasks()),
109+
SkillGroup("empty-skill", set_learned("", []), []),
110+
],
111+
edit_budget=4, gate_metric="mixed", night=1,
112+
)
113+
rows = skill_group_reports(outcomes)
114+
self.assertEqual([r.skill_name for r in rows], ["research-skill", "empty-skill"])
115+
research = rows[0]
116+
self.assertEqual(research.status, CONSOLIDATED)
117+
self.assertTrue(research.accepted)
118+
self.assertGreater(research.candidate_score, research.baseline_score)
119+
self.assertEqual(research.n_tasks, len(_tasks()))
120+
self.assertEqual(rows[1].status, SKIPPED)
121+
self.assertFalse(rows[1].accepted)
122+
123+
124+
class TestSleepReportCompatibility(unittest.TestCase):
125+
def test_single_skill_report_has_no_group_rows(self):
126+
report = SleepReport(night=1, project="/repo/example", accepted=True,
127+
gate_action="accept_new_best")
128+
self.assertEqual(report.skill_groups, [])
129+
self.assertEqual(report.to_dict()["skill_groups"], [])
130+
131+
def test_legacy_keyword_construction_still_works(self):
132+
legacy = {
133+
"night": 2, "project": "/repo/example", "n_sessions": 1, "n_tasks": 3,
134+
"baseline_score": 0.1, "candidate_score": 0.2, "accepted": True,
135+
"gate_action": "accept", "edits": [], "notes": ["ok"],
136+
}
137+
report = SleepReport(**legacy)
138+
self.assertEqual(report.skill_groups, [])
139+
self.assertEqual(report.gate_action, "accept")
140+
141+
def test_group_rows_serialize_beside_the_single_skill_summary(self):
142+
report = SleepReport(night=3, project="/repo/example", accepted=True,
143+
gate_action="accept_new_best", candidate_score=0.75)
144+
report.skill_groups = skill_group_reports({
145+
"research-skill": GroupConsolidation(
146+
"research-skill", CONSOLIDATED, n_tasks=6,
147+
result=_result(True, "accept_new_best", 0.25, 0.75, applied=2),
148+
),
149+
})
150+
payload = report.to_dict()
151+
# older consumers read the flat summary and ignore the new key
152+
self.assertEqual(payload["gate_action"], "accept_new_best")
153+
self.assertEqual(payload["candidate_score"], 0.75)
154+
self.assertEqual(payload["skill_groups"][0]["skill_name"], "research-skill")
155+
self.assertEqual(payload["skill_groups"][0]["n_applied_edits"], 2)
156+
157+
158+
if __name__ == "__main__":
159+
unittest.main()

0 commit comments

Comments
 (0)