Skip to content

Commit cbaf6a3

Browse files
feat(sleep): consolidate skill groups independently with failure isolation
Add consolidate_groups: each skill group runs through the existing consolidate() with its own tasks, document, and gate decision. Task-less, name-less, repeated, and raising groups are reported instead of aborting the night, shared memory is read-only, and adoption stays explicit. Refs microsoft#120
1 parent 8a4c96a commit cbaf6a3

2 files changed

Lines changed: 272 additions & 0 deletions

File tree

‎skillopt_sleep/multi_skill.py‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
"""SkillOpt-Sleep — consolidate several skill groups in one night.
2+
3+
A night normally consolidates one managed skill. When mined tasks have been
4+
grouped per skill, each group is an independent consolidation: its own tasks, its
5+
own document, its own gate decision. This module drives those runs so that a
6+
group with no tasks, an unusable name, or a failing backend is isolated and
7+
reported instead of aborting the groups around it.
8+
9+
Adoption is unchanged and still explicit: nothing here writes a skill file.
10+
"""
11+
from __future__ import annotations
12+
13+
from dataclasses import dataclass, field
14+
from typing import Callable, Dict, List, Optional, Sequence
15+
16+
from skillopt_sleep.backend import Backend
17+
from skillopt_sleep.consolidate import ConsolidationResult, consolidate
18+
from skillopt_sleep.types import TaskRecord
19+
20+
CONSOLIDATED = "consolidated"
21+
SKIPPED = "skipped"
22+
FAILED = "failed"
23+
24+
25+
@dataclass
26+
class SkillGroup:
27+
"""One skill's slice of the night: its name, its document, its tasks."""
28+
29+
skill_name: str
30+
skill: str = ""
31+
tasks: List[TaskRecord] = field(default_factory=list)
32+
33+
34+
@dataclass
35+
class GroupConsolidation:
36+
"""Per-group outcome. ``result`` is set only when the group actually ran."""
37+
38+
skill_name: str
39+
status: str
40+
result: Optional[ConsolidationResult] = None
41+
reason: str = ""
42+
43+
@property
44+
def accepted(self) -> bool:
45+
return bool(self.result and self.result.accepted)
46+
47+
48+
def consolidate_groups(
49+
backend: Backend,
50+
groups: Sequence[SkillGroup],
51+
memory: str = "",
52+
*,
53+
consolidate_fn: Callable[..., ConsolidationResult] = consolidate,
54+
**consolidate_kwargs: object,
55+
) -> Dict[str, GroupConsolidation]:
56+
"""Consolidate each group independently, in order, isolating failures.
57+
58+
Returns one entry per input group, keyed by skill name and ordered
59+
first-seen. A group is ``skipped`` when it is unusable (blank name, no
60+
tasks, repeated name) and ``failed`` when its own consolidation raised; both
61+
leave every other group's decision untouched.
62+
63+
``memory`` is the shared agent memory and is passed through read-only: group
64+
runs evolve skills only, so no group can rewrite another group's memory.
65+
"""
66+
out: Dict[str, GroupConsolidation] = {}
67+
for group in groups:
68+
name = (group.skill_name or "").strip()
69+
if not name:
70+
out.setdefault("", GroupConsolidation("", SKIPPED, reason="group has no skill name"))
71+
continue
72+
if name in out:
73+
continue # first group wins; a repeated name is not a second night
74+
if not group.tasks:
75+
out[name] = GroupConsolidation(name, SKIPPED, reason="no mined tasks for this skill")
76+
continue
77+
try:
78+
result = consolidate_fn(
79+
backend, list(group.tasks), group.skill, memory,
80+
evolve_memory=False, **consolidate_kwargs,
81+
)
82+
except Exception as exc: # one group's failure must not abort the night
83+
out[name] = GroupConsolidation(
84+
name, FAILED, reason=f"{type(exc).__name__}: {exc}"[:300]
85+
)
86+
continue
87+
out[name] = GroupConsolidation(name, CONSOLIDATED, result=result)
88+
return out
89+
90+
91+
def accepted_group_skills(outcomes: Dict[str, GroupConsolidation]) -> Dict[str, str]:
92+
"""Skill name -> new document, for groups whose gate accepted an update."""
93+
return {
94+
name: outcome.result.new_skill
95+
for name, outcome in outcomes.items()
96+
if outcome.accepted and outcome.result is not None
97+
}

‎tests/test_sleep_multi_skill.py‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
"""Tests for per-skill-group consolidation (issue #120).
2+
3+
Pure-stdlib (unittest), deterministic MockBackend, no API key, no network.
4+
Run: python -m pytest tests/test_sleep_multi_skill.py
5+
"""
6+
from __future__ import annotations
7+
8+
import unittest
9+
10+
from skillopt_sleep.backend import MockBackend
11+
from skillopt_sleep.consolidate import consolidate
12+
from skillopt_sleep.experiments.personas import programmer_persona, researcher_persona
13+
from skillopt_sleep.memory import set_learned
14+
from skillopt_sleep.mine import assign_splits
15+
from skillopt_sleep.multi_skill import (
16+
CONSOLIDATED,
17+
FAILED,
18+
SKIPPED,
19+
GroupConsolidation,
20+
SkillGroup,
21+
accepted_group_skills,
22+
consolidate_groups,
23+
)
24+
25+
26+
def _tasks(persona, seed=42):
27+
return assign_splits(persona(), holdout_fraction=0.34, seed=seed)
28+
29+
30+
class TestConsolidateGroups(unittest.TestCase):
31+
def test_no_groups_is_an_empty_mapping(self):
32+
self.assertEqual(consolidate_groups(MockBackend(), []), {})
33+
34+
def test_single_group_matches_the_existing_single_skill_run(self):
35+
skill = set_learned("", [])
36+
direct = consolidate(MockBackend(), _tasks(researcher_persona), skill, "",
37+
edit_budget=4, gate_metric="mixed", night=1,
38+
evolve_memory=False)
39+
grouped = consolidate_groups(
40+
MockBackend(),
41+
[SkillGroup("research-skill", skill, _tasks(researcher_persona))],
42+
edit_budget=4, gate_metric="mixed", night=1,
43+
)
44+
self.assertEqual(list(grouped), ["research-skill"])
45+
outcome = grouped["research-skill"]
46+
self.assertEqual(outcome.status, CONSOLIDATED)
47+
self.assertTrue(outcome.accepted)
48+
self.assertEqual(outcome.result.new_skill, direct.new_skill)
49+
self.assertEqual(outcome.result.gate_action, direct.gate_action)
50+
self.assertEqual(outcome.result.candidate_score, direct.candidate_score)
51+
52+
def test_each_group_gets_its_own_decision_and_document(self):
53+
groups = [
54+
SkillGroup("research-skill", set_learned("# research\n", []),
55+
_tasks(researcher_persona)),
56+
SkillGroup("programming-skill", set_learned("# programming\n", []),
57+
_tasks(programmer_persona, seed=1)),
58+
]
59+
outcomes = consolidate_groups(MockBackend(), groups, edit_budget=4, night=1)
60+
self.assertEqual(list(outcomes), ["research-skill", "programming-skill"])
61+
self.assertIn("# research", outcomes["research-skill"].result.new_skill)
62+
self.assertIn("# programming", outcomes["programming-skill"].result.new_skill)
63+
self.assertNotIn("# programming", outcomes["research-skill"].result.new_skill)
64+
65+
def test_group_without_tasks_is_skipped_not_fatal(self):
66+
outcomes = consolidate_groups(
67+
MockBackend(),
68+
[
69+
SkillGroup("empty-skill", set_learned("", []), []),
70+
SkillGroup("research-skill", set_learned("", []), _tasks(researcher_persona)),
71+
],
72+
edit_budget=4, night=1,
73+
)
74+
self.assertEqual(outcomes["empty-skill"].status, SKIPPED)
75+
self.assertIsNone(outcomes["empty-skill"].result)
76+
self.assertIn("no mined tasks", outcomes["empty-skill"].reason)
77+
self.assertEqual(outcomes["research-skill"].status, CONSOLIDATED)
78+
79+
def test_group_without_a_name_is_skipped_and_reported(self):
80+
outcomes = consolidate_groups(
81+
MockBackend(),
82+
[SkillGroup(" ", set_learned("", []), _tasks(researcher_persona))],
83+
edit_budget=4, night=1,
84+
)
85+
self.assertEqual(outcomes[""].status, SKIPPED)
86+
self.assertIn("no skill name", outcomes[""].reason)
87+
88+
def test_repeated_group_name_runs_once(self):
89+
calls = []
90+
91+
def _fake(backend, tasks, skill, memory, **kwargs):
92+
calls.append(skill)
93+
return consolidate(backend, tasks, skill, memory, **kwargs)
94+
95+
outcomes = consolidate_groups(
96+
MockBackend(),
97+
[
98+
SkillGroup("research-skill", set_learned("# first\n", []),
99+
_tasks(researcher_persona)),
100+
SkillGroup("research-skill", set_learned("# second\n", []),
101+
_tasks(researcher_persona)),
102+
],
103+
consolidate_fn=_fake, edit_budget=4, night=1,
104+
)
105+
self.assertEqual(len(outcomes), 1)
106+
self.assertEqual(len(calls), 1)
107+
self.assertIn("# first", calls[0])
108+
109+
def test_one_failing_group_is_isolated_and_reported(self):
110+
def _fake(backend, tasks, skill, memory, **kwargs):
111+
if "boom" in skill:
112+
raise RuntimeError("backend exploded")
113+
return consolidate(backend, tasks, skill, memory, **kwargs)
114+
115+
outcomes = consolidate_groups(
116+
MockBackend(),
117+
[
118+
SkillGroup("broken-skill", "boom", _tasks(researcher_persona)),
119+
SkillGroup("research-skill", set_learned("", []), _tasks(researcher_persona)),
120+
],
121+
consolidate_fn=_fake, edit_budget=4, night=1,
122+
)
123+
self.assertEqual(outcomes["broken-skill"].status, FAILED)
124+
self.assertIsNone(outcomes["broken-skill"].result)
125+
self.assertIn("RuntimeError: backend exploded", outcomes["broken-skill"].reason)
126+
self.assertEqual(outcomes["research-skill"].status, CONSOLIDATED)
127+
self.assertTrue(outcomes["research-skill"].accepted)
128+
129+
def test_group_runs_do_not_evolve_shared_memory(self):
130+
seen = {}
131+
132+
def _fake(backend, tasks, skill, memory, **kwargs):
133+
seen.update(kwargs)
134+
seen["memory"] = memory
135+
return consolidate(backend, tasks, skill, memory, **kwargs)
136+
137+
consolidate_groups(
138+
MockBackend(),
139+
[SkillGroup("research-skill", set_learned("", []), _tasks(researcher_persona))],
140+
"# shared memory\n", consolidate_fn=_fake, edit_budget=4, night=1,
141+
)
142+
self.assertEqual(seen["memory"], "# shared memory\n")
143+
self.assertFalse(seen["evolve_memory"])
144+
145+
def test_accepted_group_skills_lists_only_accepted_updates(self):
146+
outcomes = {
147+
"kept": GroupConsolidation(
148+
"kept", CONSOLIDATED,
149+
result=consolidate(MockBackend(), _tasks(researcher_persona),
150+
set_learned("", []), "", edit_budget=4, night=1),
151+
),
152+
"skipped": GroupConsolidation("skipped", SKIPPED, reason="no mined tasks"),
153+
"failed": GroupConsolidation("failed", FAILED, reason="RuntimeError: x"),
154+
}
155+
self.assertTrue(outcomes["kept"].accepted)
156+
self.assertEqual(list(accepted_group_skills(outcomes)), ["kept"])
157+
self.assertFalse(outcomes["skipped"].accepted)
158+
self.assertFalse(outcomes["failed"].accepted)
159+
160+
def test_consolidating_groups_writes_no_files(self):
161+
import os
162+
import tempfile
163+
164+
with tempfile.TemporaryDirectory() as tmp:
165+
before = sorted(os.listdir(tmp))
166+
consolidate_groups(
167+
MockBackend(),
168+
[SkillGroup("research-skill", set_learned("", []), _tasks(researcher_persona))],
169+
edit_budget=4, night=1,
170+
)
171+
self.assertEqual(sorted(os.listdir(tmp)), before)
172+
173+
174+
if __name__ == "__main__":
175+
unittest.main()

0 commit comments

Comments
 (0)