Skip to content

Commit fa813f6

Browse files
mogulOpenCode Agent
andcommitted
fix(opencode): allow long refusals in the text-only image control
The 500-character reply stop is right where only a short token is expected (unset and explicit text+image), but the explicit text-only negative control legitimately produces verbose refusals; a live smoke run halted on one. Use an 8000-character cap there, still stopping on credential-shaped content, and truncate stored replies to 500 characters after classification and leak detection. Co-authored-by: OpenCode Agent <[email protected]>
1 parent 049465e commit fa813f6

4 files changed

Lines changed: 88 additions & 8 deletions

File tree

‎integrations/isolation/acq-kits/opencode/docs/synthetic-image-validation.md‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -191,10 +191,16 @@ interrupt.
191191
The script enforces these stop conditions and halts rather than continuing: a
192192
literal credential in the rendered provider block, or the key variable unset;
193193
a provider `baseURL` that is not `https://api.gsa.usai.gov`; a reply that is
194-
oversize or carries credential-shaped data; an explicit `["text"]` run that
194+
longer than 500 characters (unset and explicit `["text","image"]` runs, where
195+
only a six-character token is expected) or carries credential-shaped data; an
196+
explicit `["text"]` run that
195197
returns the token; three consecutive models failing the text control; and an
196198
OpenCode that cannot be run. A model whose own text control fails is recorded
197-
and skipped (auth or routing, not an image result).
199+
and skipped (auth or routing, not an image result). In the explicit
200+
`["text"]` negative control a long, harmless refusal is the expected outcome, so
201+
that run's length cap is 8000 characters instead (credential-shaped content
202+
still stops the run); stored replies are truncated to 500 characters after
203+
classification.
198204

199205
The script never edits `catalog.json`, a schema, or any committed capability
200206
value, and its `candidate` column applies the decision rule above mechanically (and, stricter than the prose, also requires the text-only control samples to be plain non-token replies, so an outage cannot pass as a negative control).

‎integrations/isolation/acq-kits/opencode/scripts/verify_image_capability.py‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,9 @@
4848
MIN_SAMPLES = 3
4949
STEP_TIMEOUT = 300
5050
MAX_REPLY_CHARS = 500
51+
# The explicit text-only run is the negative control: a long, harmless refusal is the GOOD outcome there,
52+
# so it gets a much larger stop cap; stored replies are still truncated to MAX_REPLY_CHARS.
53+
MAX_CONTROL_REPLY_CHARS = 8000
5154
MAX_CONSECUTIVE_CONTROL_FAILURES = 3
5255
TOKEN_LEN = 6
5356
# Unambiguous glyphs only (no 0/O, 1/I/L, 2/Z, 5/S, 8/B) so a misread is a model fault, not an OCR coin flip.
@@ -273,10 +276,10 @@ def key_env_name(block: dict) -> str:
273276
return match.group(1)
274277

275278

276-
def check_reply_content(reply: str, secret_values: tuple[str, ...]) -> None:
279+
def check_reply_content(reply: str, secret_values: tuple[str, ...], limit: int = MAX_REPLY_CHARS) -> None:
277280
"""Stop condition: a reply that is oversize or carries secret-shaped data is not a token reply."""
278-
if len(reply) > MAX_REPLY_CHARS:
279-
raise StopRun("UNEXPECTED_CONTENT", f"reply longer than {MAX_REPLY_CHARS} characters")
281+
if len(reply) > limit:
282+
raise StopRun("UNEXPECTED_CONTENT", f"reply longer than {limit} characters")
280283
if redact(reply, secret_values) != reply:
281284
raise StopRun("UNEXPECTED_CONTENT", "reply contains credential-shaped data (withheld)")
282285

@@ -569,7 +572,15 @@ class Settings:
569572
status: Callable[[str], None] = print
570573

571574

572-
def probe_step(cfg: Settings, tree: Path, cwd: Path, model_id: str, prompt: str, image: Path | None) -> StepOutcome:
575+
def probe_step(
576+
cfg: Settings,
577+
tree: Path,
578+
cwd: Path,
579+
model_id: str,
580+
prompt: str,
581+
image: Path | None,
582+
limit: int = MAX_REPLY_CHARS,
583+
) -> StepOutcome:
573584
outcome = run_step(
574585
cfg.opencode,
575586
f"{cfg.provider}/{model_id}",
@@ -580,7 +591,7 @@ def probe_step(cfg: Settings, tree: Path, cwd: Path, model_id: str, prompt: str,
580591
cfg.timeout,
581592
)
582593
if outcome.kind == "reply":
583-
check_reply_content(outcome.text, cfg.secret)
594+
check_reply_content(outcome.text, cfg.secret, limit)
584595
outcome.text = outcome.text.strip()
585596
outcome.detail = redact(outcome.detail, cfg.secret)[:MAX_REPLY_CHARS]
586597
return outcome
@@ -609,9 +620,11 @@ def verify_model(cfg: Settings, model_id: str, cwd: Path, images: Path, results:
609620
paths.append(path)
610621
for name, _ in CONFIGS:
611622
for index, (token, path) in enumerate(zip(tokens, paths, strict=True)):
612-
outcome = probe_step(cfg, trees[name], cwd, model_id, PROMPT_IMAGE, path)
623+
limit = MAX_CONTROL_REPLY_CHARS if name == "text-only" else MAX_REPLY_CHARS
624+
outcome = probe_step(cfg, trees[name], cwd, model_id, PROMPT_IMAGE, path, limit)
613625
sample = Sample(name, index, token, outcome.text, classify(outcome, token), outcome.detail)
614626
sample.leaked = outcome.kind == "reply" and token.lower() in outcome.text.lower()
627+
sample.reply = sample.reply[:MAX_REPLY_CHARS]
615628
result.samples.append(sample)
616629
cfg.status(f" {name} #{index + 1}: {sample.classification}")
617630
if name == "text-only" and sample.leaked:

‎scripts/tests/fixtures/fake_opencode.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,14 @@ def emit_error(status, message):
6767
emit_error(400, "image input is not supported")
6868
if model_id == "blind":
6969
emit_text("ZZZZZZ")
70+
if model_id == "verbose":
71+
emit_text(token if image_allowed else "I am unable to view images in this conversation. " * 60)
72+
if model_id == "lateleak":
73+
emit_text(token if image_allowed else "I cannot see any image. " * 40 + token)
74+
if model_id == "longall":
75+
emit_text("x" * 600)
76+
if model_id == "longsecret":
77+
emit_text("I cannot see an image. " * 60 + "Bearer abcdefghijklmnop12345" if not image_allowed else token)
7078
if model_id == "good":
7179
emit_text(token if image_allowed else "I cannot see any image")
7280
emit_text("I cannot see any image")

‎scripts/tests/test_verify_image_capability.py‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -348,6 +348,59 @@ def test_default_output_dir_is_under_cwd_tmp(sandbox, monkeypatch, tmp_path):
348348
assert len(runs) == 1 and (runs[0] / "evidence.md").exists()
349349

350350

351+
def set_models(config, names):
352+
data = json.loads(config.read_text().split("\n", 1)[1])
353+
data["provider"]["usai"]["models"] = {n: {"name": n} for n in names}
354+
config.write_text(json.dumps(data))
355+
356+
357+
def test_long_refusal_on_text_only_is_classified_and_run_continues(sandbox):
358+
set_models(sandbox[2], ["verbose", "good"])
359+
code, out = run_main(sandbox)
360+
assert code == 0
361+
doc = json.loads((out / "results.json").read_text())
362+
assert [m["model"] for m in doc["models"]] == ["verbose", "good"]
363+
verbose = doc["models"][0]
364+
control = [s for s in verbose["samples"] if s["config"] == "text-only"]
365+
assert len(control) == 3 and {s["classification"] for s in control} == {v.WRONG}
366+
assert all(len(s["reply"]) == v.MAX_REPLY_CHARS for s in control)
367+
assert verbose["candidate_image"] is True and doc["stopped"] == ""
368+
369+
370+
def test_long_reply_in_token_configurations_still_stops(sandbox):
371+
set_models(sandbox[2], ["longall", "good"])
372+
code, out = run_main(sandbox)
373+
assert code == 2
374+
doc = json.loads((out / "results.json").read_text())
375+
assert "UNEXPECTED_CONTENT" in doc["stopped"] and "500" in doc["stopped"]
376+
assert [m["model"] for m in doc["models"]] == ["longall"]
377+
378+
379+
def test_secret_shaped_long_reply_on_text_only_still_stops(sandbox, capsys):
380+
set_models(sandbox[2], ["longsecret"])
381+
code, out = run_main(sandbox)
382+
assert code == 2
383+
doc = json.loads((out / "results.json").read_text())
384+
assert "UNEXPECTED_CONTENT" in doc["stopped"] and doc["summary"]["qualifying"] == []
385+
assert "abcdefghijklmnop12345" not in (out / "results.json").read_text()
386+
assert "abcdefghijklmnop12345" not in capsys.readouterr().out
387+
388+
389+
def test_token_after_truncation_point_is_still_detected_as_leak(sandbox):
390+
set_models(sandbox[2], ["lateleak"])
391+
code, out = run_main(sandbox)
392+
assert code == 2
393+
doc = json.loads((out / "results.json").read_text())
394+
assert "NEGATIVE_CONTROL_LEAK" in doc["stopped"] and doc["summary"]["qualifying"] == []
395+
assert doc["summary"]["text_only_token_leaks"] == 1
396+
397+
398+
def test_control_reply_over_the_larger_cap_stops():
399+
with pytest.raises(v.StopRun):
400+
v.check_reply_content("x" * (v.MAX_CONTROL_REPLY_CHARS + 1), (), v.MAX_CONTROL_REPLY_CHARS)
401+
v.check_reply_content("x" * v.MAX_CONTROL_REPLY_CHARS, (), v.MAX_CONTROL_REPLY_CHARS)
402+
403+
351404
def test_negative_control_leak_stops_the_run(sandbox):
352405
tmp, fake, config = sandbox
353406
data = json.loads(config.read_text().split("\n", 1)[1])

0 commit comments

Comments
 (0)