Skip to content

Commit 850d584

Browse files
committed
Honor judge BLOCK when score is low
1 parent 2221d98 commit 850d584

6 files changed

Lines changed: 139 additions & 5 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@ All notable changes to the `unplug-ai` SDK.
44

55
## [Unreleased]
66

7+
### Fixed
8+
9+
- Judge `action=block` (and `review`/`allow`) now clamps finding scores so score-driven policy honors the declared verdict even when the LLM returns an inconsistent score
10+
711
## [0.5.2] — 2026-07-20
812

913
### Added

‎sdk/docs/LIMITS_AND_JUDGE.md‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,18 @@ Optional Stage-3 classifier for borderline cases. Pass any object with
6565
| `judge_low` / `judge_high` on `GuardConfig` | `0.3` / `0.8` | Invoke when max scanner score is in `[low, high)` **or** ML emitted ABSTAIN |
6666
| Failures / timeouts | — | Fail closed (`Action.BLOCK`, stage `llm_judge`) |
6767

68-
`judge_enabled` in TOML is **deprecated and ignored**. Provide a real `judge=`
69-
instance in code.
68+
Judge JSON `action` is authoritative for that finding's contribution to
69+
score-driven policy. Inconsistent score/action pairs are clamped:
70+
71+
| Judge `action` | Score handling |
72+
|----------------|----------------|
73+
| `block` | Raised to at least `block_threshold` so enforcement blocks |
74+
| `review` | Clamped into the review band (`>= review_threshold`, `< redact_threshold`) |
75+
| `allow` | Capped below `review_threshold` so the judge finding alone cannot escalate |
76+
77+
Other scanner findings can still raise the overall action above the judge's
78+
verdict. `judge_enabled` in TOML is **deprecated and ignored**. Provide a real
79+
`judge=` instance in code.
7080

7181
### Python
7282

‎sdk/src/unplug/core/judge.py‎

Lines changed: 45 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
from __future__ import annotations
44

55
import asyncio
6+
import math
67
from typing import Any, Protocol, runtime_checkable
78

89
from pydantic import BaseModel, Field
@@ -41,14 +42,56 @@ class JudgeResult(BaseModel):
4142
score: float = Field(ge=0.0, le=1.0, default=0.5)
4243
reason: str = ""
4344

44-
def to_finding(self, text_length: int) -> Finding:
45+
def enforced_score(
46+
self,
47+
*,
48+
block_threshold: float = 0.8,
49+
redact_threshold: float = 0.5,
50+
review_threshold: float = 0.3,
51+
) -> float:
52+
"""Clamp score so score-driven policy honors the declared action.
53+
54+
Inconsistent pairs are resolved in favor of ``action``:
55+
- ``BLOCK`` + low score → raise to at least ``block_threshold``
56+
- ``REVIEW`` → land in ``[review_threshold, min(redact, block))``
57+
- ``ALLOW`` + high score → cap below ``review_threshold``
58+
"""
59+
score = self.score
60+
if self.action == Action.BLOCK:
61+
return min(1.0, max(score, block_threshold))
62+
if self.action == Action.REVIEW:
63+
upper = min(redact_threshold, block_threshold)
64+
if upper <= review_threshold:
65+
return min(1.0, max(score, review_threshold))
66+
floored = max(score, review_threshold)
67+
if floored >= upper:
68+
return max(review_threshold, math.nextafter(upper, 0.0))
69+
return floored
70+
if self.action == Action.ALLOW:
71+
if score >= review_threshold:
72+
return max(0.0, math.nextafter(review_threshold, 0.0))
73+
return score
74+
return score
75+
76+
def to_finding(
77+
self,
78+
text_length: int,
79+
*,
80+
block_threshold: float = 0.8,
81+
redact_threshold: float = 0.5,
82+
review_threshold: float = 0.3,
83+
) -> Finding:
4584
return Finding(
4685
category=self.category,
4786
subcategory="llm_judge",
4887
stage="llm_judge",
4988
span_start=0,
5089
span_end=text_length,
51-
score=self.score,
90+
score=self.enforced_score(
91+
block_threshold=block_threshold,
92+
redact_threshold=redact_threshold,
93+
review_threshold=review_threshold,
94+
),
5295
evidence=self.reason,
5396
)
5497

‎sdk/src/unplug/pipelines/input.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -174,4 +174,11 @@ def _maybe_judge(
174174
evidence=f"Judge failed: {type(exc).__name__}",
175175
)
176176
]
177-
return [result.to_finding(len(input_data.text))]
177+
return [
178+
result.to_finding(
179+
len(input_data.text),
180+
block_threshold=policy.block_threshold,
181+
redact_threshold=policy.redact_threshold,
182+
review_threshold=policy.review_threshold,
183+
)
184+
]

‎sdk/tests/integration/test_guard_limits.py‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,3 +107,40 @@ async def broken_judge(prompt: str) -> str:
107107
result = guard.scan("ignore previous instructions")
108108
assert result.action == Action.BLOCK
109109
assert any(f.stage == "llm_judge" for f in result.findings)
110+
111+
def test_low_score_judge_block_still_blocks(self) -> None:
112+
async def fake_judge(prompt: str) -> str:
113+
_ = prompt
114+
return (
115+
'{"action": "block", "category": "injection", '
116+
'"score": 0.05, "reason": "explicit block"}'
117+
)
118+
119+
guard = Guard(
120+
scanners=["injection"],
121+
judge=CallableJudge(fake_judge),
122+
config=GuardConfig(judge_low=0.0, judge_high=1.0),
123+
)
124+
result = guard.scan("What is the weather in Paris?")
125+
assert result.action == Action.BLOCK
126+
assert result.safe is False
127+
judge_findings = [f for f in result.findings if f.stage == "llm_judge"]
128+
assert judge_findings
129+
assert judge_findings[0].score >= 0.8
130+
131+
def test_high_score_judge_allow_does_not_self_block(self) -> None:
132+
async def fake_judge(prompt: str) -> str:
133+
_ = prompt
134+
return '{"action": "allow", "category": "safe", "score": 0.99, "reason": "benign"}'
135+
136+
guard = Guard(
137+
scanners=["injection"],
138+
judge=CallableJudge(fake_judge),
139+
config=GuardConfig(judge_low=0.0, judge_high=1.0),
140+
)
141+
result = guard.scan("What is the weather in Paris?")
142+
judge_findings = [f for f in result.findings if f.stage == "llm_judge"]
143+
assert judge_findings
144+
assert judge_findings[0].score < 0.3
145+
assert result.action == Action.ALLOW
146+
assert result.safe is True

‎sdk/tests/unit/core/test_judge.py‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,39 @@ def test_to_finding(self):
2929
assert f.span_end == 100
3030
assert f.score == 0.95
3131

32+
def test_block_action_raises_low_score_to_threshold(self):
33+
r = JudgeResult(action=Action.BLOCK, category="injection", score=0.1, reason="block")
34+
assert r.enforced_score(block_threshold=0.8) == 0.8
35+
f = r.to_finding(text_length=50, block_threshold=0.8)
36+
assert f.score == 0.8
37+
38+
def test_block_action_keeps_high_score(self):
39+
r = JudgeResult(action=Action.BLOCK, category="injection", score=0.95, reason="block")
40+
assert r.enforced_score(block_threshold=0.8) == 0.95
41+
42+
def test_block_string_action_raises_low_score(self):
43+
r = JudgeResult.model_validate(
44+
{"action": "block", "category": "injection", "score": 0.05, "reason": "no"}
45+
)
46+
assert r.action == Action.BLOCK
47+
assert r.enforced_score(block_threshold=0.9) == 0.9
48+
49+
def test_allow_action_caps_high_score(self):
50+
r = JudgeResult(action=Action.ALLOW, category="safe", score=0.95, reason="ok")
51+
capped = r.enforced_score(review_threshold=0.3)
52+
assert capped < 0.3
53+
54+
def test_review_action_lands_in_review_band(self):
55+
r = JudgeResult(action=Action.REVIEW, category="injection", score=0.05, reason="maybe")
56+
score = r.enforced_score(block_threshold=0.8, redact_threshold=0.5, review_threshold=0.3)
57+
assert score >= 0.3
58+
assert score < 0.5
59+
60+
def test_review_action_clamps_high_score_below_redact(self):
61+
r = JudgeResult(action=Action.REVIEW, category="injection", score=0.99, reason="maybe")
62+
score = r.enforced_score(block_threshold=0.8, redact_threshold=0.5, review_threshold=0.3)
63+
assert 0.3 <= score < 0.5
64+
3265

3366
class TestJudgeContext:
3467
def test_empty_context(self):

0 commit comments

Comments
 (0)