Skip to content

Commit fd63be9

Browse files
anandgupta42claude
andcommitted
feat: verifier-gated model router (run cheap → verify → escalate)
Run a cheap model first, verify the result deterministically (`dbt build`), and escalate to a stronger model only when verification fails. Flag-gated (`ALTIMATE_ROUTER`), default off — the normal single-model path is unchanged. - `router/verifier.ts` — deterministic `Verdict` from `dbt build`/`dbt test`; spoof-proof (last-summary + exitCode backstop); pluggable `Impl`; honest `unverifiable` state when a gate cannot run - `router/router.ts` — escalation ladder; per-tier exception handling so a transient tier failure escalates instead of aborting; env ladder override - `router/policy.ts` — static default ladder or an altimate-API-served policy (key-gated, `AbortSignal` timeouts, `sanitizeTiers` validation + cap) - `router/verdict.ts` — verdict envelope (schemaVersion, evidence hash, signer seam) - `cli/cmd/run.ts` — orchestrator: routes only verifiable (dbt) workspaces; non-dbt projects run once with the user's model (no silent downgrade) - 51 unit tests + 3 env-gated E2E suites (real dbt / real model / real network) Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
1 parent 56259a3 commit fd63be9

13 files changed

Lines changed: 1329 additions & 2 deletions

File tree

‎packages/opencode/src/cli/cmd/run.ts‎

Lines changed: 74 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,12 @@ import { Filesystem } from "../../util/filesystem"
1010
import { createOpencodeClient, type Message, type OpencodeClient, type ToolPart } from "@opencode-ai/sdk/v2"
1111
import { Server } from "../../server/server"
1212
import { Provider } from "../../provider/provider"
13+
// altimate_change start — verifier-gated router (run cheap, verify, escalate)
14+
import { Router } from "../../router/router"
15+
import { Verifier } from "../../router/verifier"
16+
import { Verdict } from "../../router/verdict"
17+
import { Policy } from "../../router/policy"
18+
// altimate_change end
1319
import { Agent } from "../../agent/agent"
1420
import { PermissionNext } from "../../permission/next"
1521
import { Tool } from "../../tool/tool"
@@ -866,6 +872,67 @@ You are speaking to a non-technical business executive. Follow these rules stric
866872
}
867873
}
868874

875+
// altimate_change start — verifier-gated router orchestration
876+
// Deterministic-verify the dbt workspace in cwd (`dbt build`, judged by Verifier).
877+
// Only gates real dbt projects; with nothing to prove it returns ok (no escalation).
878+
async function verifyWorkspace(): Promise<Verifier.Verdict> {
879+
const root = process.cwd()
880+
if (!(await Filesystem.exists(path.join(root, "dbt_project.yml"))))
881+
return { ok: true, unverifiable: true, reason: "no dbt project to verify", checks: [] }
882+
try {
883+
const proc = Bun.spawn(["dbt", "build"], { cwd: root, stdout: "pipe", stderr: "pipe" })
884+
// Hard timeout so a hung dbt (lock, prompt, runaway query) can't stall the run.
885+
let timedOut = false
886+
const timer = setTimeout(() => {
887+
timedOut = true
888+
proc.kill()
889+
}, 300_000)
890+
const out = (await new Response(proc.stdout).text()) + (await new Response(proc.stderr).text())
891+
const code = await proc.exited
892+
clearTimeout(timer)
893+
if (timedOut) return { ok: false, reason: "dbt build timed out after 300s", checks: [] }
894+
return Verifier.fromDbt(out, code)
895+
} catch (e) {
896+
// dbt binary missing / spawn failure → can't verify; mark unverifiable (fail-open, but honest).
897+
return { ok: true, unverifiable: true, reason: `verify skipped: ${String(e)}`, checks: [] }
898+
}
899+
}
900+
901+
// Run the tier ladder: cheap → verify → escalate with failing-check context, stop at first pass.
902+
// Each tier re-invokes the existing single-run path with that model (and the escalation note
903+
// prepended) in the SAME workspace, so a later tier fixes the prior attempt rather than restarting.
904+
async function runRouted(sdk: OpencodeClient) {
905+
// Only route when the workspace is verifiable. Without a deterministic gate, routing
906+
// would accept the cheapest tier with no way to verify or escalate — silently
907+
// downgrading quality. In a non-dbt project, run once with the user's model instead.
908+
if (!(await Filesystem.exists(path.join(process.cwd(), "dbt_project.yml")))) {
909+
await execute(sdk)
910+
return
911+
}
912+
const baseMessage = message
913+
const policy = Policy.resolve()
914+
const tiers = await policy.tiers({ prompt: baseMessage })
915+
const result = await Router.route({
916+
tiers,
917+
runAgent: async (model, note) => {
918+
args.model = model
919+
message = note ? `${note}\n\n${baseMessage}` : baseMessage
920+
await execute(sdk)
921+
},
922+
verify: verifyWorkspace,
923+
})
924+
message = baseMessage
925+
const envelope = Verdict.build(result, { now: new Date().toISOString() })
926+
if (args.format === "json") {
927+
process.stdout.write(JSON.stringify({ type: "verdict", timestamp: Date.now(), ...envelope }) + EOL)
928+
} else {
929+
const tag = envelope.solved ? `✓ verified by ${envelope.solvedBy}` : "✗ unverified after all tiers"
930+
UI.println(UI.Style.TEXT_INFO_BOLD + `~ router: ${tag} (policy: ${policy.source})`)
931+
}
932+
await Policy.reportOutcome(envelope)
933+
}
934+
// altimate_change end
935+
869936
if (args.attach) {
870937
const headers = (() => {
871938
const password = args.password ?? process.env.OPENCODE_SERVER_PASSWORD
@@ -875,7 +942,9 @@ You are speaking to a non-technical business executive. Follow these rules stric
875942
return { Authorization: auth }
876943
})()
877944
const sdk = createOpencodeClient({ baseUrl: args.attach, directory, headers })
878-
return await execute(sdk)
945+
// altimate_change start — route when enabled, else single run
946+
return Router.enabled() ? await runRouted(sdk) : await execute(sdk)
947+
// altimate_change end
879948
}
880949

881950
await bootstrap(process.cwd(), async () => {
@@ -884,7 +953,10 @@ You are speaking to a non-technical business executive. Follow these rules stric
884953
return Server.Default().fetch(request)
885954
}) as typeof globalThis.fetch
886955
const sdk = createOpencodeClient({ baseUrl: "http://altimate-code.internal", fetch: fetchFn })
887-
await execute(sdk)
956+
// altimate_change start — route when enabled, else single run
957+
if (Router.enabled()) await runRouted(sdk)
958+
else await execute(sdk)
959+
// altimate_change end
888960
})
889961
},
890962
})
Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# Verifier-gated router
2+
3+
Run a cheap model first, verify the result deterministically, and escalate to a
4+
stronger model only when verification fails. Most runs finish at the cheap tier;
5+
the rest get a stronger attempt that receives the exact failing checks as context.
6+
Flag-gated (`ALTIMATE_ROUTER`), default off — the normal single-model path is unchanged.
7+
8+
## Modules (pure, unit-tested)
9+
- **`verifier.ts`** — `Verifier`: a deterministic `Verdict` from `dbt build`/`dbt test`
10+
output (`fromDbt`, `parseDbtSummary`, `failingNodes`). `Impl` is the pluggable
11+
verifier interface; the default `dbtVerifier(run)` shells dbt (runner injected,
12+
fail-open). `ALLOW_ALL` passes everything when no verifier is configured.
13+
- **`router.ts`** — `Router`: the escalation mechanism. `route({tiers, runAgent, verify})`
14+
runs each tier, verifies, escalates on a failed verdict with the failing checks
15+
(`escalationContext`), stops at the first pass. `DEFAULT_LADDER` is ordered
16+
cheapest → strongest; override via `ALTIMATE_ROUTER_LADDER`.
17+
- **`policy.ts`** — `Policy`: where the ladder comes from. `STATIC` is the built-in
18+
default; `altimate(key)` fetches a per-context ladder from the altimate API when
19+
`ALTIMATE_API_KEY` is set (degrades to static on any failure); `resolve()` picks
20+
between them; `reportOutcome()` posts verified outcomes back (key-gated, best-effort).
21+
`sanitizeTiers` validates + caps any ladder from the API.
22+
- **`verdict.ts`** — `Verdict.Envelope`: a machine-checkable record of the result
23+
(which tier, per-attempt history, checks, evidence hash, timestamp, optional signature).
24+
25+
## Configuration
26+
- `ALTIMATE_ROUTER=1` — enable routing (default off).
27+
- `ALTIMATE_ROUTER_LADDER` — comma-separated `provider/model` ids to override the default ladder.
28+
- `ALTIMATE_API_KEY` / `ALTIMATE_API_URL` — use the altimate API for the routing policy
29+
and outcome reporting instead of the static ladder.
30+
31+
## Integration
32+
`src/cli/cmd/run.ts` (`RunCommand`): when `Router.enabled()`, the run resolves a policy,
33+
runs each tier by re-invoking the existing run path with that model (escalation note
34+
prepended) in the same workspace, verifies with `dbt build` between tiers, and emits a
35+
verdict envelope. The default (non-router) path is untouched.
36+
37+
## Tests
38+
- **Unit** — `test/router/{verifier,router,verdict,policy}.test.ts`. Pure logic, incl.
39+
adversarial cases (dbt summary-line injection, ANSI/huge/multi-summary output,
40+
endpoint response validation/capping).
41+
- **E2E** (`test/router/*.e2e.test.ts`, env-gated — require docker + a dbt image +
42+
network, excluded from default CI):
43+
- `verifier.e2e` — real `dbt build` (pass / compile-error / failing-test) and that a
44+
model emitting a fake summary does not change the verdict. `E2E_IMG=<image> bun test verifier.e2e`.
45+
- `router.e2e` — real model calls + real dbt: cheap tier solves; an unsatisfiable
46+
workspace escalates through tiers, caps, and threads failing-check context.
47+
`OPENROUTER_API_KEY=… E2E_IMG=… bun test router.e2e`.
48+
- `policy.e2e` — real network: live local server (incl. error/malformed/oversized
49+
responses) and an unreachable endpoint, all degrade gracefully. `bun test policy.e2e`.
Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
/**
2+
* Routing policy — the decision of WHAT to route to, kept separate from the
3+
* mechanism that executes it.
4+
*
5+
* Two sources:
6+
* - `STATIC`: the built-in default ladder, always available.
7+
* - `altimate(key)`: when an altimate API key is configured, the routing policy is
8+
* fetched per-context from the altimate API and used instead, and verified
9+
* outcomes are reported back so the policy can be tuned over time.
10+
*
11+
* The client executes whatever policy it is handed. The SaaS policy activates only
12+
* when `ALTIMATE_API_KEY` is present, otherwise the static ladder is used.
13+
* Network/transport failures degrade to STATIC.
14+
*/
15+
import { Router } from "./router"
16+
import type { Verdict } from "./verdict"
17+
18+
export namespace Policy {
19+
/** Signals available for routing decisions (extended over time). */
20+
export interface RoutingContext {
21+
prompt?: string
22+
projectType?: string
23+
taskId?: string
24+
}
25+
26+
export interface RoutingPolicy {
27+
source: "static" | "altimate"
28+
tiers(ctx: RoutingContext): Promise<Router.Tier[]>
29+
}
30+
31+
type Fetch = typeof globalThis.fetch
32+
33+
/** Defensive cap: a bad/compromised policy endpoint must not inject a cost-bomb ladder. */
34+
export const MAX_TIERS = 8
35+
36+
/**
37+
* Validate + cap a ladder returned by the policy endpoint. Keeps only entries with a
38+
* non-empty string `model`, derives a label when missing, caps to MAX_TIERS. Returns
39+
* null if nothing usable (caller falls back to the static ladder).
40+
*/
41+
/** A model id must look like `provider/model[/...]` — plain chars only, no whitespace/control. */
42+
const MODEL_RE = /^[A-Za-z0-9._-]+(?:\/[A-Za-z0-9._-]+)+$/
43+
44+
export function sanitizeTiers(raw: unknown): Router.Tier[] | null {
45+
if (!Array.isArray(raw)) return null
46+
const out: Router.Tier[] = []
47+
for (const t of raw) {
48+
const model = (t as any)?.model
49+
if (typeof model !== "string") continue
50+
const m = model.trim()
51+
if (!m || m.length > 200 || !MODEL_RE.test(m)) continue
52+
const rawLabel = typeof (t as any)?.label === "string" && (t as any).label ? (t as any).label : m.split("/").pop() || m
53+
// Strip non-printable/ANSI — the label is printed to the terminal.
54+
const label = String(rawLabel).replace(/[^\x20-\x7E]/g, "").slice(0, 100) || m
55+
out.push({ model: m, label })
56+
if (out.length >= MAX_TIERS) break
57+
}
58+
return out.length ? out : null
59+
}
60+
61+
export function apiKey(): string | undefined {
62+
return process.env["ALTIMATE_API_KEY"] || undefined
63+
}
64+
65+
export function baseUrl(): string {
66+
return process.env["ALTIMATE_API_URL"] || "https://api.altimate.ai"
67+
}
68+
69+
/** Built-in default ladder (env-overridable via ALTIMATE_ROUTER_LADDER). */
70+
export const STATIC: RoutingPolicy = {
71+
source: "static",
72+
async tiers() {
73+
return Router.ladder()
74+
},
75+
}
76+
77+
/**
78+
* Customer routing policy served by the altimate API. Resolves the per-context
79+
* ladder for this account; degrades to the static ladder if the service is
80+
* unreachable or returns nothing usable.
81+
*/
82+
export function altimate(key: string, base: string = baseUrl(), fetchImpl: Fetch = fetch): RoutingPolicy {
83+
return {
84+
source: "altimate",
85+
async tiers(ctx: RoutingContext): Promise<Router.Tier[]> {
86+
try {
87+
const res = await fetchImpl(`${base}/v1/router/policy`, {
88+
method: "POST",
89+
headers: { "Content-Type": "application/json", Authorization: `Bearer ${key}` },
90+
body: JSON.stringify(ctx),
91+
signal: AbortSignal.timeout(3000),
92+
})
93+
if (!res.ok) return Router.ladder()
94+
const data = (await res.json()) as { tiers?: unknown }
95+
return sanitizeTiers(data?.tiers) ?? Router.ladder()
96+
} catch {
97+
return Router.ladder()
98+
}
99+
},
100+
}
101+
}
102+
103+
/** The active policy: customer policy when an altimate key is set, else the static ladder. */
104+
export function resolve(fetchImpl: Fetch = fetch): RoutingPolicy {
105+
const key = apiKey()
106+
return key ? altimate(key, baseUrl(), fetchImpl) : STATIC
107+
}
108+
109+
/**
110+
* Report a verified outcome back to the altimate service so the customer's policy
111+
* improves. Best-effort and key-gated — a no-op without a key, and never throws.
112+
*/
113+
export async function reportOutcome(
114+
envelope: Verdict.Envelope,
115+
base: string = baseUrl(),
116+
fetchImpl: Fetch = fetch,
117+
): Promise<void> {
118+
const key = apiKey()
119+
if (!key) return
120+
try {
121+
await fetchImpl(`${base}/v1/router/outcomes`, {
122+
method: "POST",
123+
headers: { "Content-Type": "application/json", Authorization: `Bearer ${key}` },
124+
body: JSON.stringify(envelope),
125+
signal: AbortSignal.timeout(3000),
126+
})
127+
} catch {
128+
/* best-effort: outcome reporting must never break the run */
129+
}
130+
}
131+
}
Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
/**
2+
* Verifier-gated model router — the escalation ladder.
3+
*
4+
* Run the CHEAP tier first; verify the workspace deterministically (Verifier);
5+
* if the verdict is not ok, escalate to the next stronger tier, handing it the
6+
* exact failing checks so it fixes rather than restarts blind. Stop at the first
7+
* passing verdict (or the top of the ladder).
8+
*
9+
* Because the cheap tier handles most tasks, escalation is rare. The default ladder
10+
* is ordered cheapest → strongest and can be overridden per deployment.
11+
*
12+
* Pure orchestration: `runAgent` + `verify` are injected → unit-testable without
13+
* a live model or dbt. Flag-gated (`ALTIMATE_ROUTER`); default off.
14+
*/
15+
import { Verifier } from "./verifier"
16+
17+
export namespace Router {
18+
export interface Tier {
19+
model: string
20+
label: string
21+
}
22+
23+
/**
24+
* Default ladder, ordered cheapest → strongest. A tier is only reached when the
25+
* previous tier's output fails verification, so most runs complete at the cheap tier.
26+
* Override per deployment via `ALTIMATE_ROUTER_LADDER` or an injected policy.
27+
*/
28+
export const DEFAULT_LADDER: Tier[] = [
29+
{ model: "openrouter/deepseek/deepseek-v4-flash", label: "deepseek-v4-flash" },
30+
{ model: "openrouter/z-ai/glm-5.1", label: "glm-5.1" },
31+
{ model: "openrouter/anthropic/claude-opus-4.8", label: "claude-opus-4.8" },
32+
]
33+
34+
export function enabled(): boolean {
35+
return process.env["ALTIMATE_ROUTER"] === "1"
36+
}
37+
38+
/** Ladder from `ALTIMATE_ROUTER_LADDER` (comma-separated provider/model ids) or the default. */
39+
export function ladder(): Tier[] {
40+
const env = process.env["ALTIMATE_ROUTER_LADDER"]
41+
if (!env) return DEFAULT_LADDER
42+
const tiers = env
43+
.split(",")
44+
.map((s) => s.trim())
45+
.filter(Boolean)
46+
.map((model) => ({ model, label: model.split("/").pop() || model }))
47+
return tiers.length ? tiers : DEFAULT_LADDER
48+
}
49+
50+
/** Escalate iff the verdict failed AND a stronger tier remains. */
51+
export function shouldEscalate(verdict: Verifier.Verdict, tierIndex: number, tiers: Tier[]): boolean {
52+
return !verdict.ok && tierIndex < tiers.length - 1
53+
}
54+
55+
/** The note handed to the next tier — names the exact failing checks so it fixes them. */
56+
export function escalationContext(prev: Tier, verdict: Verifier.Verdict): string {
57+
const failing = verdict.checks.filter((c) => !c.ok).map((c) => c.name)
58+
const lines = [
59+
`A previous attempt (by ${prev.label}) did not pass verification.`,
60+
verdict.reason ? `Verifier reason: ${verdict.reason}` : "",
61+
failing.length ? `Failing checks to fix: ${failing.join(", ")}.` : "",
62+
`The prior changes are in the workspace — fix these specific failures; do not start over.`,
63+
]
64+
return lines.filter(Boolean).join("\n")
65+
}
66+
67+
export interface Attempt {
68+
tier: Tier
69+
verdict: Verifier.Verdict
70+
}
71+
72+
export interface RouteResult {
73+
solved: boolean
74+
solvedBy?: Tier
75+
attempts: Attempt[]
76+
}
77+
78+
/**
79+
* Drive the ladder: run each tier, verify, escalate on failure with context,
80+
* stop at the first ok verdict. `runAgent(model, escalationNote?)` performs the
81+
* agent run in the shared workspace; `verify()` judges the post-run workspace.
82+
*/
83+
export async function route(params: {
84+
tiers?: Tier[]
85+
runAgent: (model: string, escalationNote?: string) => Promise<void>
86+
verify: () => Promise<Verifier.Verdict>
87+
}): Promise<RouteResult> {
88+
const tiers = params.tiers ?? ladder()
89+
const attempts: Attempt[] = []
90+
let note: string | undefined
91+
for (let i = 0; i < tiers.length; i++) {
92+
const tier = tiers[i]
93+
// A thrown agent/verify error is treated as a failed attempt so the ladder can
94+
// escalate, rather than aborting the whole run on a transient failure in one tier.
95+
let verdict: Verifier.Verdict
96+
try {
97+
await params.runAgent(tier.model, note)
98+
verdict = await params.verify()
99+
} catch (e) {
100+
verdict = { ok: false, reason: `tier error: ${String(e)}`, checks: [] }
101+
}
102+
attempts.push({ tier, verdict })
103+
if (verdict.ok) return { solved: true, solvedBy: tier, attempts }
104+
if (!shouldEscalate(verdict, i, tiers)) break
105+
note = escalationContext(tier, verdict)
106+
}
107+
return { solved: false, attempts }
108+
}
109+
}

0 commit comments

Comments
 (0)