Skip to content

Commit 4d93cad

Browse files
committed
fix(sleep): render baseline-relative probe scores (LG-40917)
1 parent 67b2f16 commit 4d93cad

2 files changed

Lines changed: 64 additions & 6 deletions

File tree

‎skillopt_sleep/cycle.py‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -400,19 +400,25 @@ def _render_report_md(report: SleepReport, cfg: SleepConfig) -> str:
400400
"closed for this candidate."
401401
)
402402
lines.append("")
403-
lines.append("| Source | Probe | Variant | Source | Probe | Delta | Status |")
404-
lines.append("|---|---|---|---:|---:|---:|---|")
403+
lines.append(
404+
"| Source | Probe | Variant | Baseline source | Baseline probe | "
405+
"Candidate source | Candidate probe | Gap change | Status |"
406+
)
407+
lines.append("|---|---|---|---:|---:|---:|---:|---:|---|")
405408
for row in probe.get("rows", []):
406409
source_id = _markdown_table_text(row.get("source_task_id", ""))
407410
probe_id = _markdown_table_text(row.get("probe_task_id", ""))
408411
kind = _markdown_table_text(row.get("probe_kind", ""))
409-
source_score = _report_score(row.get("source_score"))
410-
probe_score = _report_score(row.get("probe_score"))
411-
delta = _report_score(row.get("delta"))
412+
baseline_source = _report_score(row.get("baseline_source_score"))
413+
baseline_probe = _report_score(row.get("baseline_probe_score"))
414+
candidate_source = _report_score(row.get("candidate_source_score"))
415+
candidate_probe = _report_score(row.get("candidate_probe_score"))
416+
gap_change = _report_score(row.get("gap_change"))
412417
status = _markdown_table_text(row.get("status", ""))
413418
lines.append(
414419
f"| `{source_id}` | `{probe_id}` | {kind} | "
415-
f"{source_score} | {probe_score} | {delta} | {status} |"
420+
f"{baseline_source} | {baseline_probe} | "
421+
f"{candidate_source} | {candidate_probe} | {gap_change} | {status} |"
416422
)
417423
lines.append("")
418424
if report.edits:

‎tests/test_adversarial_dream.py‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -509,6 +509,58 @@ def test_multi_skill_cycle_keeps_blocking_probe_rollouts(tmp_path) -> None:
509509
assert group.accepted is True
510510

511511

512+
@pytest.mark.parametrize("blocking", [False, True], ids=["advisory", "blocking"])
513+
@pytest.mark.parametrize(
514+
("backend_type", "probe_score", "gap_change", "status"),
515+
[
516+
(_CandidateBackend, 0.0, -1.0, "brittle"),
517+
(_RobustCandidateBackend, 1.0, 0.0, "stable"),
518+
],
519+
ids=["brittle", "robust"],
520+
)
521+
def test_cycle_report_renders_baseline_relative_probe_scores(
522+
tmp_path, blocking, backend_type, probe_score, gap_change, status
523+
) -> None:
524+
project = tmp_path / "project"
525+
project.mkdir()
526+
config = load_config(
527+
invoked_project=str(project),
528+
projects="invoked",
529+
backend="mock",
530+
state_dir=str(tmp_path / "state"),
531+
claude_home=str(tmp_path / ".claude"),
532+
evolve_memory=False,
533+
dream_adversarial=1,
534+
dream_adversarial_blocking=blocking,
535+
dream_adversarial_rollouts=2,
536+
auto_adopt=False,
537+
)
538+
539+
outcome = run_sleep_cycle(
540+
config, seed_tasks=_candidate_tasks(), backend=backend_type()
541+
)
542+
543+
row = outcome.report.gate_trials[0]["adversarial_probe"]["rows"][0]
544+
assert row["baseline_source_score"] == 0.0
545+
assert row["baseline_probe_score"] == 0.0
546+
assert row["candidate_source_score"] == 1.0
547+
assert row["candidate_probe_score"] == probe_score
548+
assert row["gap_change"] == gap_change
549+
assert row["status"] == status
550+
with open(
551+
os.path.join(outcome.staging_dir, "report.md"), encoding="utf-8"
552+
) as handle:
553+
markdown = handle.read()
554+
assert (
555+
"| Source | Probe | Variant | Baseline source | Baseline probe | "
556+
"Candidate source | Candidate probe | Gap change | Status |"
557+
) in markdown
558+
assert (
559+
"| `train` | `train_adversarial_request-frame` | request-frame | "
560+
f"0.000 | 0.000 | 1.000 | {probe_score:.3f} | {gap_change:.3f} | {status} |"
561+
) in markdown
562+
563+
512564
class _EquallySensitiveBackend(_CandidateBackend):
513565
"""Frame sensitivity exists identically with and without the candidate."""
514566

0 commit comments

Comments
 (0)