Skip to content

Commit b5f0322

Browse files
mogulOpenCode Agent
andcommitted
fix(opencode): own the scratch dir, honor --samples, skip unavailable models
Address review of the image-verification script: - Create a unique owned scratch dir only after preflight and remove only it, so a pre-existing <output-dir>/.work survives --dry-run, preflight stops, and complete runs. - Pass the requested sample count into candidate decisions so a model only qualifies when every configuration has all requested samples. - Classify text-control failures: a message naming the single model (blocked, unknown, no tool-use streaming) is reported as SKIPPED (unavailable), does not count toward the consecutive-failure stop, and does not change the exit status; 401s, bare 4xx, timeouts and connection errors stay run-level, and a run where no model passes still stops. Co-authored-by: OpenCode Agent <[email protected]>
1 parent fa813f6 commit b5f0322

4 files changed

Lines changed: 262 additions & 17 deletions

File tree

‎integrations/isolation/acq-kits/opencode/docs/synthetic-image-validation.md‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,7 @@ classification, text-only control result, date) to the output directory, and
179179
prints one line to paste back:
180180

181181
```text
182-
RESULT unset=correct:N/wrong:N/rejected:N/error:N text-image=... text-only=... models=N candidates=N token_leaks=N stopped=no opencode=<version> date=<YYYY-MM-DD> dir=<path>
182+
RESULT unset=correct:N/wrong:N/rejected:N/error:N text-image=... text-only=... models=N candidates=N token_leaks=N skipped=N(<models>) stopped=no opencode=<version> date=<YYYY-MM-DD> dir=<path>
183183
```
184184

185185
Paste back the `RESULT` line and the contents of `evidence.md` (it holds
@@ -194,9 +194,17 @@ a provider `baseURL` that is not `https://api.gsa.usai.gov`; a reply that is
194194
longer than 500 characters (unset and explicit `["text","image"]` runs, where
195195
only a six-character token is expected) or carries credential-shaped data; an
196196
explicit `["text"]` run that
197-
returns the token; three consecutive models failing the text control; and an
197+
returns the token; three consecutive models failing the text control for auth, routing, timeout or
198+
connection reasons (or no model passing it at all); and an
198199
OpenCode that cannot be run. A model whose own text control fails is recorded
199-
and skipped (auth or routing, not an image result). In the explicit
200+
and skipped (not an image result). When the gateway message names that one model (`model_blocked_globally`,
201+
an unknown or nonexistent model, or "doesn't support tool use") it is reported
202+
as `SKIPPED (unavailable)`, does not count toward the three-in-a-row stop, and
203+
does not change the exit status; a complete run with such models exits 0 and
204+
lists them as `skipped=N(model,...)` in the `RESULT` line. A plain 401 or 403, a
205+
bare 4xx with no model-specific text, a timeout, or a connection error stays a
206+
run-level failure, and if no model at all passes its text control the run stops
207+
(a run cannot succeed with every model "unavailable"). In the explicit
200208
`["text"]` negative control a long, harmless refusal is the expected outcome, so
201209
that run's length cap is 8000 characters instead (credential-shaped content
202210
still stops the run); stored replies are truncated to 500 characters after

‎integrations/isolation/acq-kits/opencode/scripts/verify_image_capability.py‎

Lines changed: 58 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import struct
3434
import subprocess
3535
import sys
36+
import tempfile
3637
import zlib
3738
from collections.abc import Callable
3839
from dataclasses import dataclass, field
@@ -313,6 +314,7 @@ class ModelResult:
313314
control_detail: str = ""
314315
samples: list[Sample] = field(default_factory=list)
315316
skipped: str = ""
317+
unavailable: bool = False
316318

317319

318320
def normalize_reply(text: str) -> str:
@@ -322,6 +324,26 @@ def normalize_reply(text: str) -> str:
322324
return text
323325

324326

327+
# A text-control failure is a per-model fact only when the gateway names the model itself (blocked, unknown,
328+
# or lacking a feature OpenCode needs). Deliberately narrow: generic phrases such as "not supported" or
329+
# "does not exist" also appear in credential and routing errors, which must stay run-level failures.
330+
UNAVAILABLE_RE = re.compile(
331+
r"model[_ -]?blocked|blocked[_ -]?globally|model[_ -]?not[_ -]?found|unknown model"
332+
r"|model\b[^.]{0,60}\bdoes not exist|(?:does not|doesn['\u2019]?t) support tool use",
333+
re.IGNORECASE,
334+
)
335+
336+
337+
def control_unavailable(outcome: StepOutcome) -> bool:
338+
"""True when a failed text control means this one model is unusable, so it is skipped, not a run-level fault.
339+
340+
Never true for a 401, a timeout, a connection error, or a bare 4xx with no model-specific text.
341+
"""
342+
if outcome.kind not in ("rejected", "error") or outcome.detail.startswith("HTTP 401"):
343+
return False
344+
return bool(UNAVAILABLE_RE.search(outcome.detail))
345+
346+
325347
def classify(outcome: StepOutcome, token: str) -> str:
326348
if outcome.kind == "rejected":
327349
return REJECTED
@@ -335,9 +357,11 @@ def decide(result: ModelResult, required: int = MIN_SAMPLES) -> tuple[bool, list
335357
text-only control never returned the token. Returns (qualifies, reasons it does not)."""
336358
reasons: list[str] = []
337359
if not result.control_ok:
338-
reasons.append("text control failed" if not result.skipped else result.skipped)
360+
return (False, [result.skipped or "text control failed"])
339361
by_config = {name: [s for s in result.samples if s.config == name] for name, _ in CONFIGS}
340-
explicit, control = by_config["text-image"], by_config["text-only"]
362+
unset, explicit, control = by_config["unset"], by_config["text-image"], by_config["text-only"]
363+
if len(unset) < required:
364+
reasons.append(f"fewer than {required} unset samples")
341365
if len(explicit) < required:
342366
reasons.append(f"fewer than {required} text-image samples")
343367
reasons += [
@@ -352,14 +376,21 @@ def decide(result: ModelResult, required: int = MIN_SAMPLES) -> tuple[bool, list
352376
return (not reasons, reasons)
353377

354378

355-
def summarize(results: list[ModelResult]) -> dict:
379+
def summarize(results: list[ModelResult], required: int = MIN_SAMPLES) -> dict:
356380
counts = {name: dict.fromkeys(CLASSES, 0) for name, _ in CONFIGS}
357381
for result in results:
358382
for sample in result.samples:
359383
counts[sample.config][sample.classification] += 1
360-
qualifying = [r.model for r in results if decide(r)[0]]
384+
qualifying = [r.model for r in results if decide(r, required)[0]]
361385
leaks = sum(1 for r in results for s in r.samples if s.config == "text-only" and s.leaked)
362-
return {"models": len(results), "qualifying": qualifying, "counts": counts, "text_only_token_leaks": leaks}
386+
skipped = [r.model for r in results if r.unavailable]
387+
return {
388+
"models": len(results),
389+
"qualifying": qualifying,
390+
"counts": counts,
391+
"text_only_token_leaks": leaks,
392+
"skipped_unavailable": skipped,
393+
}
363394

364395

365396
# --- running OpenCode ------------------------------------------------------------------------------
@@ -610,7 +641,9 @@ def verify_model(cfg: Settings, model_id: str, cwd: Path, images: Path, results:
610641
result.control_ok = control.kind == "reply" and bool(control.text)
611642
result.control_detail = control.text[:80] if result.control_ok else (control.detail or control.kind)
612643
if not result.control_ok:
613-
result.skipped = f"text control failed: {result.control_detail}"
644+
result.unavailable = control_unavailable(control)
645+
label = "SKIPPED (unavailable)" if result.unavailable else "text control failed"
646+
result.skipped = f"{label}: {result.control_detail}"
614647
return result
615648
tokens = make_tokens(cfg.seed, model_id, cfg.samples)
616649
paths = []
@@ -640,12 +673,16 @@ def run_all(cfg: Settings, models: list[str], cwd: Path, images: Path, results:
640673
for number, model_id in enumerate(models, start=1):
641674
cfg.status(f"[{number}/{len(models)}] {model_id}")
642675
result = verify_model(cfg, model_id, cwd, images, results)
676+
if result.unavailable:
677+
continue
643678
consecutive_control_failures = 0 if result.control_ok else consecutive_control_failures + 1
644679
if consecutive_control_failures >= MAX_CONSECUTIVE_CONTROL_FAILURES:
645680
raise StopRun(
646681
"CONTROL_FAILING",
647682
f"{consecutive_control_failures} consecutive models failed the text control (auth or routing?)",
648683
)
684+
if results and not any(r.control_ok for r in results):
685+
raise StopRun("CONTROL_FAILING", "no model passed the text control (auth, routing, or provider-wide fault?)")
649686

650687

651688
# --- outputs ----------------------------------------------------------------------------------------
@@ -659,14 +696,15 @@ def tsv_field(text: object) -> str:
659696
return re.sub(r"[\t\r\n]+", " ", str(text))
660697

661698

662-
def model_record(result: ModelResult) -> dict:
663-
qualifies, reasons = decide(result)
699+
def model_record(result: ModelResult, required: int = MIN_SAMPLES) -> dict:
700+
qualifies, reasons = decide(result, required)
664701
return {
665702
"model": result.model,
666703
"config_facts": result.facts,
667704
"text_control_ok": result.control_ok,
668705
"text_control_detail": result.control_detail,
669706
"skipped": result.skipped,
707+
"skipped_unavailable": result.unavailable,
670708
"candidate_image": qualifies,
671709
"reasons_not_candidate": reasons,
672710
"samples": [
@@ -688,8 +726,9 @@ def model_record(result: ModelResult) -> dict:
688726

689727

690728
def build_document(meta: dict, results: list[ModelResult]) -> dict:
691-
records = [model_record(r) for r in results]
692-
summary = summarize(results)
729+
required = max(int(meta.get("samples", MIN_SAMPLES)), MIN_SAMPLES)
730+
records = [model_record(r, required) for r in results]
731+
summary = summarize(results, required)
693732
if any(code in str(meta.get("stopped", "")) for code in INVALIDATING_STOPS):
694733
# The harness itself is in doubt, so nothing from this run may be treated as a candidate.
695734
for record in records:
@@ -777,8 +816,9 @@ def evidence_markdown(document: dict) -> str:
777816
for model in document["models"]:
778817
facts = model["config_facts"]
779818
step1 = f"{fact_cell(facts.get('modalities.input'))} / {fact_cell(facts.get('reasoning'))}"
819+
failure = cell(model["skipped"] or model["text_control_detail"])
780820
control = (
781-
"ok" if model["text_control_ok"] else f"FAILED: {cell(model['skipped'] or model['text_control_detail'])}"
821+
"ok" if model["text_control_ok"] else (failure if model["skipped_unavailable"] else f"FAILED: {failure}")
782822
)
783823
decision = "yes" if model["candidate_image"] else "no: " + cell("; ".join(model["reasons_not_candidate"]))
784824
lines.append(
@@ -812,6 +852,7 @@ def evidence_markdown(document: dict) -> str:
812852

813853
def result_line(document: dict, outdir: Path) -> str:
814854
counts = document["summary"]["counts"]
855+
skipped = document["summary"].get("skipped_unavailable", [])
815856
parts = ["RESULT"]
816857
for name, _ in CONFIGS:
817858
c = counts[name]
@@ -820,6 +861,7 @@ def result_line(document: dict, outdir: Path) -> str:
820861
f"models={document['summary']['models']}",
821862
f"candidates={len(document['summary']['qualifying'])}",
822863
f"token_leaks={document['summary']['text_only_token_leaks']}",
864+
f"skipped={len(skipped)}" + (f"({','.join(skipped)})" if skipped else ""),
823865
f"stopped={document.get('stopped') or 'no'}",
824866
f"opencode={document['opencode_version']}",
825867
f"date={document['date']}",
@@ -895,7 +937,7 @@ def main(argv: list[str] | None = None) -> int:
895937
"host": args.expected_host,
896938
"stopped": "",
897939
}
898-
work = outdir / ".work"
940+
work: Path | None = None
899941
old_handlers = {s: signal.signal(s, _raise_interrupted) for s in (signal.SIGINT, signal.SIGTERM)}
900942
code = 0
901943
try:
@@ -923,6 +965,8 @@ def main(argv: list[str] | None = None) -> int:
923965
return 0
924966
outdir.mkdir(parents=True, exist_ok=True)
925967
outdir.chmod(0o700)
968+
# Owned scratch dir: unique per run, created only after preflight, and the only thing cleanup removes.
969+
work = Path(tempfile.mkdtemp(prefix=".work-", dir=outdir))
926970
cwd, images = work / "cwd", work / "images"
927971
for directory in (cwd, images):
928972
directory.mkdir(parents=True, exist_ok=True)
@@ -951,7 +995,8 @@ def main(argv: list[str] | None = None) -> int:
951995
finally:
952996
for sig_number, handler in old_handlers.items():
953997
signal.signal(sig_number, handler)
954-
shutil.rmtree(work, ignore_errors=True)
998+
if work is not None:
999+
shutil.rmtree(work, ignore_errors=True)
9551000
outdir.mkdir(parents=True, exist_ok=True)
9561001
outdir.chmod(0o700)
9571002
document = build_document(meta, results)

‎scripts/tests/fixtures/fake_opencode.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import os
77
import re
88
import sys
9+
import time
910
from pathlib import Path
1011

1112
args = sys.argv[1:]
@@ -55,6 +56,16 @@ def emit_error(status, message):
5556

5657

5758
if image is None:
59+
if model_id.startswith("bare404"):
60+
emit_error(404, "Not Found")
61+
if model_id.startswith("blocked"):
62+
emit_error(403, "Not authorized. model_blocked_globally")
63+
if model_id.startswith("nostream"):
64+
emit_error(400, "This model doesn't support tool use in streaming mode.")
65+
if model_id.startswith("auth403"):
66+
emit_error(403, "Not authorized.")
67+
if model_id.startswith("timeout"):
68+
time.sleep(5)
5869
if model_id.startswith("nocontrol"):
5970
emit_error(401, "unauthorized")
6071
emit_text("OK")

0 commit comments

Comments
 (0)