Skip to content

Commit 79124b3

Browse files
authored
Merge pull request #257 from RohithPariki/auto-fix-slow-update-robustness
fix(optimizer): robust metric resolution in slow-update longitudinal comparison
2 parents 3f1fee8 + fa44cfb commit 79124b3

2 files changed

Lines changed: 172 additions & 4 deletions

File tree

‎skillopt/optimizer/slow_update.py‎

Lines changed: 37 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import json
2222
import os
2323
import traceback
24+
from typing import Any
2425

2526
from skillopt.model import chat_optimizer
2627
from skillopt.prompts import load_prompt
@@ -156,6 +157,38 @@ def _read_trajectory(rollout_dir: str, task_id: str) -> str:
156157
# ── Structured comparison pairs ─────────────────────────────────────────────
157158

158159

160+
def _is_result_success(res: dict | Any) -> bool:
161+
"""Determine whether a rollout result indicates success.
162+
163+
Strictly requires the 'hard' key to be a finite numeric or boolean value in [0, 1].
164+
Fails closed for strings, non-finite values, out-of-range values, or missing fields.
165+
"""
166+
if not res or not isinstance(res, dict):
167+
return False
168+
hard_val = res.get("hard")
169+
if isinstance(hard_val, (bool, int, float)) and not isinstance(hard_val, str):
170+
import math
171+
if math.isfinite(hard_val) and 0.0 <= hard_val <= 1.0:
172+
return float(hard_val) >= 0.5
173+
return False
174+
175+
176+
def _get_soft_score(res: dict | Any) -> float:
177+
"""Safely extract the soft score from a rollout result.
178+
179+
Strictly requires the 'soft' key to be a finite numeric or boolean value in [0, 1].
180+
Fails closed to 0.0 for strings, non-finite values, out-of-range values, or missing fields.
181+
"""
182+
if not res or not isinstance(res, dict):
183+
return 0.0
184+
soft_val = res.get("soft")
185+
if isinstance(soft_val, (bool, int, float)) and not isinstance(soft_val, str):
186+
import math
187+
if math.isfinite(soft_val) and 0.0 <= soft_val <= 1.0:
188+
return float(soft_val)
189+
return 0.0
190+
191+
159192
def build_comparison_pairs(
160193
results_prev: list[dict],
161194
results_curr: list[dict],
@@ -183,8 +216,8 @@ def build_comparison_pairs(
183216
tid = str(item.get("id", ""))
184217
prev = prev_by_id.get(tid, {})
185218
curr = curr_by_id.get(tid, {})
186-
prev_ok = bool(prev.get("hard", 0))
187-
curr_ok = bool(curr.get("hard", 0))
219+
prev_ok = _is_result_success(prev)
220+
curr_ok = _is_result_success(curr)
188221

189222
if not prev_ok and curr_ok:
190223
category = "improved"
@@ -201,13 +234,13 @@ def build_comparison_pairs(
201234
"category": category,
202235
"prev": {
203236
"hard": int(prev_ok),
204-
"soft": float(prev.get("soft", 0.0)),
237+
"soft": _get_soft_score(prev),
205238
"predicted_answer": prev.get("predicted_answer", prev.get("answer", "N/A")),
206239
"fail_reason": prev.get("fail_reason", ""),
207240
},
208241
"curr": {
209242
"hard": int(curr_ok),
210-
"soft": float(curr.get("soft", 0.0)),
243+
"soft": _get_soft_score(curr),
211244
"predicted_answer": curr.get("predicted_answer", curr.get("answer", "N/A")),
212245
"fail_reason": curr.get("fail_reason", ""),
213246
},
Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
"""Tests for slow update field manipulation and longitudinal comparison robustness."""
2+
3+
from __future__ import annotations
4+
5+
import json
6+
import os
7+
import tempfile
8+
9+
from skillopt.optimizer.slow_update import (
10+
_is_result_success,
11+
_strip_all_slow_update_fields,
12+
build_comparison_pairs,
13+
extract_slow_update_field,
14+
has_slow_update_field,
15+
inject_empty_slow_update_field,
16+
replace_slow_update_field,
17+
save_comparison_pairs,
18+
)
19+
20+
21+
def test_is_result_success_strict_contract() -> None:
22+
# Explicit hard boolean / float (Valid)
23+
assert _is_result_success({"hard": 1}) is True
24+
assert _is_result_success({"hard": 0}) is False
25+
assert _is_result_success({"hard": True}) is True
26+
assert _is_result_success({"hard": False}) is False
27+
assert _is_result_success({"hard": 0.8}) is True
28+
assert _is_result_success({"hard": 0.2}) is False
29+
30+
# Fail closed for strings
31+
assert _is_result_success({"hard": "true"}) is False
32+
assert _is_result_success({"hard": "1"}) is False
33+
assert _is_result_success({"hard": "false"}) is False
34+
35+
# Fail closed for None and non-finite
36+
assert _is_result_success({"hard": None}) is False
37+
assert _is_result_success({"hard": float("nan")}) is False
38+
assert _is_result_success({"hard": float("inf")}) is False
39+
40+
# Fail closed for out of range
41+
assert _is_result_success({"hard": -1}) is False
42+
assert _is_result_success({"hard": 2}) is False
43+
44+
# Fail closed for other metric shapes (adapters must normalize)
45+
assert _is_result_success({"score": 1}) is False
46+
assert _is_result_success({"exact_match": 1}) is False
47+
assert _is_result_success({}) is False
48+
49+
50+
def test_build_comparison_pairs_categorization() -> None:
51+
items = [
52+
{"id": "task-1", "question": "Task 1 description"},
53+
{"id": "task-2", "question": "Task 2 description"},
54+
{"id": "task-3", "question": "Task 3 description"},
55+
{"id": "task-4", "question": "Task 4 description"},
56+
]
57+
58+
# task-1: improved (fail -> pass)
59+
# task-2: regressed (pass -> fail)
60+
# task-3: persistent_fail (fail -> fail)
61+
# task-4: stable_success (pass -> pass)
62+
results_prev = [
63+
{"id": "task-1", "hard": 0.0, "soft": 0.1, "predicted_answer": "wrong1"},
64+
{"id": "task-2", "hard": 1.0, "soft": 0.9, "predicted_answer": "correct2"},
65+
{"id": "task-3", "hard": 0, "soft": 0.0, "fail_reason": "timeout"},
66+
{"id": "task-4", "hard": 1, "soft": 1.0, "predicted_answer": "correct4"},
67+
]
68+
results_curr = [
69+
{"id": "task-1", "hard": 1.0, "soft": 0.95, "predicted_answer": "correct1"},
70+
{"id": "task-2", "hard": 0.0, "soft": 0.2, "predicted_answer": "wrong2"},
71+
{"id": "task-3", "hard": 0, "soft": 0.05, "fail_reason": "wrong_syntax"},
72+
{"id": "task-4", "hard": 1, "soft": 1.0, "predicted_answer": "correct4"},
73+
]
74+
75+
pairs = build_comparison_pairs(results_prev, results_curr, items)
76+
assert len(pairs) == 4
77+
78+
by_id = {p["id"]: p for p in pairs}
79+
assert by_id["task-1"]["category"] == "improved"
80+
assert by_id["task-1"]["prev"]["hard"] == 0
81+
assert by_id["task-1"]["curr"]["hard"] == 1
82+
83+
assert by_id["task-2"]["category"] == "regressed"
84+
assert by_id["task-2"]["prev"]["hard"] == 1
85+
assert by_id["task-2"]["curr"]["hard"] == 0
86+
87+
assert by_id["task-3"]["category"] == "persistent_fail"
88+
assert by_id["task-3"]["prev"]["hard"] == 0
89+
assert by_id["task-3"]["curr"]["hard"] == 0
90+
91+
assert by_id["task-4"]["category"] == "stable_success"
92+
assert by_id["task-4"]["prev"]["hard"] == 1
93+
assert by_id["task-4"]["curr"]["hard"] == 1
94+
95+
96+
def test_slow_update_field_lifecycle() -> None:
97+
skill = "# Main Skill\n\nRule 1: Always verify assumptions."
98+
99+
assert not has_slow_update_field(skill)
100+
injected = inject_empty_slow_update_field(skill)
101+
assert has_slow_update_field(injected)
102+
assert extract_slow_update_field(injected) == ""
103+
104+
# Idempotent inject
105+
assert inject_empty_slow_update_field(injected) == injected
106+
107+
# Replace field with guidance
108+
guidance = "Avoid premature tool exit on partial output."
109+
updated = replace_slow_update_field(injected, guidance)
110+
assert has_slow_update_field(updated)
111+
assert extract_slow_update_field(updated) == guidance
112+
113+
# Stripping all fields
114+
stripped = _strip_all_slow_update_fields(updated)
115+
assert not has_slow_update_field(stripped)
116+
assert stripped == skill.rstrip()
117+
118+
119+
def test_save_comparison_pairs_writes_valid_json() -> None:
120+
pairs = [
121+
{
122+
"id": "item-1",
123+
"task": "Test task",
124+
"category": "improved",
125+
"prev": {"hard": 0},
126+
"curr": {"hard": 1},
127+
}
128+
]
129+
with tempfile.TemporaryDirectory() as tmpdir:
130+
out_file = os.path.join(tmpdir, "comparison.json")
131+
save_comparison_pairs(pairs, out_file)
132+
assert os.path.exists(out_file)
133+
with open(out_file, encoding="utf-8") as f:
134+
loaded = json.load(f)
135+
assert loaded == pairs

0 commit comments

Comments
 (0)