|
| 1 | +"""Tests for Claude ``Skill`` tool-use harvesting (issue #120). |
| 2 | +
|
| 3 | +Pure-stdlib (unittest), deterministic, no API key, no third-party deps. |
| 4 | +Run: python -m pytest tests/test_sleep_skill_harvest.py |
| 5 | +""" |
| 6 | +from __future__ import annotations |
| 7 | + |
| 8 | +import json |
| 9 | +import os |
| 10 | +import tempfile |
| 11 | +import unittest |
| 12 | + |
| 13 | +from skillopt_sleep.harvest import digest_transcript |
| 14 | +from skillopt_sleep.types import SessionDigest |
| 15 | + |
| 16 | + |
| 17 | +def _skill_block(skill, name="Skill", block_type="tool_use", extra=None): |
| 18 | + args = {"skill": skill} if skill is not None else {} |
| 19 | + if extra: |
| 20 | + args.update(extra) |
| 21 | + return {"type": block_type, "name": name, "input": args} |
| 22 | + |
| 23 | + |
| 24 | +def _assistant(content): |
| 25 | + return { |
| 26 | + "type": "assistant", |
| 27 | + "timestamp": "2026-07-28T10:00:00Z", |
| 28 | + "cwd": "/repo/example", |
| 29 | + "gitBranch": "main", |
| 30 | + "message": {"role": "assistant", "content": content}, |
| 31 | + } |
| 32 | + |
| 33 | + |
| 34 | +def _user(text): |
| 35 | + return { |
| 36 | + "type": "user", |
| 37 | + "timestamp": "2026-07-28T09:59:00Z", |
| 38 | + "cwd": "/repo/example", |
| 39 | + "message": {"role": "user", "content": text}, |
| 40 | + } |
| 41 | + |
| 42 | + |
| 43 | +class TestSessionDigestSkillsField(unittest.TestCase): |
| 44 | + def test_defaults_to_empty_list(self): |
| 45 | + digest = SessionDigest(session_id="s1", project="/repo/example") |
| 46 | + self.assertEqual(digest.skills_used, []) |
| 47 | + |
| 48 | + def test_to_dict_includes_empty_skills_used(self): |
| 49 | + digest = SessionDigest(session_id="s1", project="/repo/example") |
| 50 | + self.assertIn("skills_used", digest.to_dict()) |
| 51 | + self.assertEqual(digest.to_dict()["skills_used"], []) |
| 52 | + |
| 53 | + def test_legacy_payload_without_skills_used_still_loads(self): |
| 54 | + legacy = {"session_id": "s1", "project": "/repo/example", "tools_used": ["Bash"]} |
| 55 | + known = set(SessionDigest.__dataclass_fields__) |
| 56 | + digest = SessionDigest(**{k: v for k, v in legacy.items() if k in known}) |
| 57 | + self.assertEqual(digest.skills_used, []) |
| 58 | + self.assertEqual(digest.tools_used, ["Bash"]) |
| 59 | + |
| 60 | + def test_unknown_key_filter_can_ignore_new_field(self): |
| 61 | + payload = SessionDigest( |
| 62 | + session_id="s1", project="/repo/example", skills_used=["example-skill"] |
| 63 | + ).to_dict() |
| 64 | + older_known = { |
| 65 | + f for f in SessionDigest.__dataclass_fields__ if f != "skills_used" |
| 66 | + } |
| 67 | + digest = SessionDigest(**{k: v for k, v in payload.items() if k in older_known}) |
| 68 | + self.assertEqual(digest.skills_used, []) |
| 69 | + |
| 70 | + |
| 71 | +class TestSkillHarvest(unittest.TestCase): |
| 72 | + def _digest(self, records): |
| 73 | + with tempfile.TemporaryDirectory() as tmp: |
| 74 | + path = os.path.join(tmp, "session-example.jsonl") |
| 75 | + with open(path, "w", encoding="utf-8") as f: |
| 76 | + for record in records: |
| 77 | + f.write(json.dumps(record) + "\n") |
| 78 | + return digest_transcript(path) |
| 79 | + |
| 80 | + def test_harvests_skill_invocation(self): |
| 81 | + digest = self._digest([ |
| 82 | + _user("run the example skill"), |
| 83 | + _assistant([_skill_block("example-skill")]), |
| 84 | + ]) |
| 85 | + self.assertIsNotNone(digest) |
| 86 | + self.assertEqual(digest.skills_used, ["example-skill"]) |
| 87 | + # tools_used keeps its existing meaning, including the Skill tool itself |
| 88 | + self.assertEqual(digest.tools_used, ["Skill"]) |
| 89 | + |
| 90 | + def test_duplicates_collapse_in_first_seen_order(self): |
| 91 | + digest = self._digest([ |
| 92 | + _user("run both skills"), |
| 93 | + _assistant([ |
| 94 | + _skill_block("second-skill"), |
| 95 | + _skill_block("example-skill"), |
| 96 | + _skill_block("second-skill"), |
| 97 | + ]), |
| 98 | + ]) |
| 99 | + self.assertEqual(digest.skills_used, ["second-skill", "example-skill"]) |
| 100 | + |
| 101 | + def test_whitespace_trimmed_without_rewriting_name(self): |
| 102 | + digest = self._digest([ |
| 103 | + _user("run it"), |
| 104 | + _assistant([_skill_block(" Example-Skill:v2 ")]), |
| 105 | + ]) |
| 106 | + self.assertEqual(digest.skills_used, ["Example-Skill:v2"]) |
| 107 | + |
| 108 | + def test_ordinary_tools_never_populate_skills_used(self): |
| 109 | + digest = self._digest([ |
| 110 | + _user("read the file"), |
| 111 | + _assistant([ |
| 112 | + {"type": "tool_use", "name": "Read", "input": {"file_path": "/repo/a.py"}}, |
| 113 | + {"type": "text", "text": "I used the Skill tool description here"}, |
| 114 | + ]), |
| 115 | + ]) |
| 116 | + self.assertEqual(digest.skills_used, []) |
| 117 | + self.assertEqual(digest.tools_used, ["Read"]) |
| 118 | + |
| 119 | + def test_malformed_blocks_are_ignored(self): |
| 120 | + digest = self._digest([ |
| 121 | + _user("run it"), |
| 122 | + _assistant([ |
| 123 | + _skill_block("lowercase-tool", name="skill"), |
| 124 | + _skill_block("wrong-type", block_type="tool_result"), |
| 125 | + {"type": "tool_use", "name": "Skill"}, |
| 126 | + {"type": "tool_use", "name": "Skill", "input": "example-skill"}, |
| 127 | + {"type": "tool_use", "name": "Skill", "input": {"skill": 7}}, |
| 128 | + _skill_block(" "), |
| 129 | + _skill_block(None), |
| 130 | + "not-a-block", |
| 131 | + ]), |
| 132 | + ]) |
| 133 | + self.assertEqual(digest.skills_used, []) |
| 134 | + |
| 135 | + def test_malformed_jsonl_record_is_non_fatal(self): |
| 136 | + with tempfile.TemporaryDirectory() as tmp: |
| 137 | + path = os.path.join(tmp, "session-example.jsonl") |
| 138 | + with open(path, "w", encoding="utf-8") as f: |
| 139 | + f.write(json.dumps(_user("run it")) + "\n") |
| 140 | + f.write("{not json\n") |
| 141 | + f.write("\n") |
| 142 | + f.write(json.dumps(_assistant([_skill_block("example-skill")])) + "\n") |
| 143 | + digest = digest_transcript(path) |
| 144 | + self.assertIsNotNone(digest) |
| 145 | + self.assertEqual(digest.skills_used, ["example-skill"]) |
| 146 | + |
| 147 | + def test_other_tool_arguments_and_outputs_stay_out_of_the_digest(self): |
| 148 | + digest = self._digest([ |
| 149 | + _user("run it"), |
| 150 | + _assistant([ |
| 151 | + _skill_block("example-skill", extra={"secret_arg": "do-not-copy"}), |
| 152 | + {"type": "tool_result", "content": "output should not copy"}, |
| 153 | + ]), |
| 154 | + ]) |
| 155 | + serialized = json.dumps(digest.to_dict()) |
| 156 | + self.assertEqual(digest.skills_used, ["example-skill"]) |
| 157 | + self.assertNotIn("do-not-copy", serialized) |
| 158 | + self.assertNotIn("output should not copy", serialized) |
| 159 | + |
| 160 | + |
| 161 | +if __name__ == "__main__": |
| 162 | + unittest.main() |
0 commit comments