Skip to content

Commit 56f17f4

Browse files
committed
feat(routing): execute capability-aware policy profiles (RI-05)
1 parent 00e1c4a commit 56f17f4

9 files changed

Lines changed: 450 additions & 10 deletions

File tree

‎devlog/_plan/260804_router_intelligence/001_pr_stack_status.md‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ other; closing one is a maintainer decision and neither is stale.
4343
| RI-02 | `feat/ri-02-request-history-index` | `b5a8e7c4c` (RI-01 head) | pending | pending | pending | in progress |
4444
| RI-03 | `feat/ri-03-routing-analytics` | `7efb6e842` (RI-02 head) | pending | pending | pending | in progress |
4545
| RI-04 | `feat/ri-04-policy-profile-core` | `2069e724e` (RI-03 head) | pending | pending | pending | in progress |
46-
| RI-05 | `feat/ri-05-capability-aware-routing` | `feat/ri-04` head | pending | pending | pending | queued |
46+
| RI-05 | `feat/ri-05-capability-aware-routing` | `00e1c4ae5` (RI-04 head) | pending | pending | pending | in progress |
4747
| RI-06 | `feat/ri-06-health-aware-routing` | `feat/ri-05` head | pending | pending | pending | queued |
4848
| RI-07 | `feat/ri-07-quota-aware-routing` | `feat/ri-06` head | pending | pending | pending | queued |
4949
| RI-08 | `feat/ri-08-cost-aware-routing` | `feat/ri-07` head | pending | pending | pending | queued |
@@ -158,3 +158,25 @@ other; closing one is a maintainer decision and neither is stale.
158158
- `tests/config.test.ts`: 109/115 pass; the 6 symlink failures reproduce
159159
identically on the pristine base (Windows symlink EPERM, environmental)
160160
- Remaining Low findings: none
161+
162+
### RI-05 - feat/ri-05-capability-aware-routing
163+
164+
- Base SHA: `00e1c4ae5df32cdf6d72957c1a36b334fe4dc0a6` (RI-04 head)
165+
- Reviewed commit: same as final (author self-review before push)
166+
- Findings (self-review): 2 fixed pre-push - (1) request evidence was wired
167+
but unused by the evaluator - request requirements (`request-tools`,
168+
`request-image-input`) now constrain candidates when the body provably
169+
needs them; (2) trace-score plumbing verified end-to-end (score was added
170+
to trace candidates in RI-04).
171+
- Final commit: pending (recorded after commit)
172+
- PR: pending
173+
- Verification:
174+
- `bun x tsc --noEmit`: PASSED (0 errors)
175+
- `bun run test tests/policy-execution.test.ts`: 8/8 pass - explicit
176+
policy/<id> + alias execution, precedence unchanged (explicit/combo/
177+
native/default), all-excluded error, unknown-capability per profile,
178+
request evidence (image/tools) constraints, determinism
179+
- Focused regression suites: 231/231 pass across 8 files (incl. combo
180+
e2e + codex-routing)
181+
- `bun run privacy:scan`: passed
182+
- Remaining Low findings: none

‎src/router.ts‎

Lines changed: 51 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,16 @@ import {
2727
type RouteDecisionTraceV1,
2828
type TraceCandidateInput,
2929
} from "./routing/trace";
30+
import { getRoutingProfile, resolvePolicyProfileId } from "./routing/profile";
31+
import { evaluatePolicyProfile, type PolicyRequestEvidence } from "./routing/evaluator";
32+
import { candidateCapabilityEvidence } from "./routing/capability";
33+
34+
export class NoEligiblePolicyCandidateError extends Error {
35+
constructor(readonly profileId: string) {
36+
super(`No eligible candidates for policy profile: ${profileId}`);
37+
this.name = "NoEligiblePolicyCandidateError";
38+
}
39+
}
3040

3141
export interface RouteResult {
3242
providerName: string;
@@ -420,8 +430,39 @@ function comboRouteCandidates(config: OcxConfig, route: RouteResult): TraceCandi
420430
});
421431
}
422432

423-
function routeModelInternal(config: OcxConfig, modelId: string, bypassCombos: boolean): RouteResult {
433+
function routeModelInternal(
434+
config: OcxConfig,
435+
modelId: string,
436+
bypassCombos: boolean,
437+
policyEvidence?: PolicyRequestEvidence,
438+
): RouteResult {
424439
const slash = modelId.indexOf("/");
440+
// Policy namespace is system-reserved: an explicit `policy/<id>` or a
441+
// configured profile alias executes the policy evaluator and routes the
442+
// selected candidate. Only explicit requests reach this branch.
443+
const policyId = resolvePolicyProfileId(config, modelId);
444+
if (policyId) {
445+
const profile = getRoutingProfile(config, policyId);
446+
if (!profile) throw new Error(`Unknown routing profile: ${policyId}`);
447+
const candidateEvidence = profile.candidates.map(candidate => ({
448+
provider: candidate.provider,
449+
model: candidate.model,
450+
capability: candidateCapabilityEvidence(config, candidate.provider, candidate.model),
451+
}));
452+
const evaluation = evaluatePolicyProfile(config, policyId, policyEvidence ?? {}, candidateEvidence);
453+
if (evaluation.selectedIndex === null) {
454+
throw new NoEligiblePolicyCandidateError(policyId);
455+
}
456+
const selected = evaluation.candidates[evaluation.selectedIndex]!;
457+
const concrete = `${selected.provider}/${selected.model}`;
458+
const routed = routeModelInternal(config, concrete, true, undefined);
459+
return {
460+
...routed,
461+
routeKind: "policy" as const,
462+
routeReason: "policy-selected",
463+
routeDecision: evaluation.trace,
464+
};
465+
}
425466
if (slash > 0) {
426467
const namespace = modelId.slice(0, slash);
427468
const binding = codexAccountNamespaceEntries(config)
@@ -460,7 +501,7 @@ function routeModelInternal(config: OcxConfig, modelId: string, bypassCombos: bo
460501
const concrete = `${combo.target.provider}/${combo.target.model}`;
461502
// The selected target is already a concrete provider/model reference. Resolve it without
462503
// consulting combo aliases again, otherwise an alias that shadows the target can recurse.
463-
const routed = routeModelInternal(config, concrete, true);
504+
const routed = routeModelInternal(config, concrete, true, undefined);
464505
return { ...routed, combo, routeKind: "combo" as const, routeReason: "combo-pick" };
465506
}
466507
}
@@ -535,8 +576,14 @@ function routeModelInternal(config: OcxConfig, modelId: string, bypassCombos: bo
535576
throw new Error(`No provider configured for model: ${modelId}`);
536577
}
537578

538-
export function routeModel(config: OcxConfig, modelId: string): RouteResult {
539-
const route = routeModelInternal(config, modelId, false);
579+
export function routeModel(
580+
config: OcxConfig,
581+
modelId: string,
582+
policyEvidence?: PolicyRequestEvidence,
583+
): RouteResult {
584+
const route = routeModelInternal(config, modelId, false, policyEvidence);
585+
// Policy routes carry a full evaluation trace already; never rebuild it.
586+
if (route.routeDecision) return route;
540587
const accountRef = route.codexAccountNamespace;
541588
route.routeDecision = buildRouteDecisionTrace({
542589
requestedModel: modelId,

‎src/routing/capability.ts‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
/**
2+
* Candidate capability evidence for policy routing (RI-05).
3+
*
4+
* Evidence comes from canonical local sources only - provider config maps,
5+
* the provider registry, the cached Codex catalog file, and the native-model
6+
* metadata helpers. No live network fetch happens at routing time.
7+
*
8+
* "Unknown is not zero": any dimension without canonical evidence stays
9+
* `undefined` (unknown) and the profile's `unknownEvidence` policy decides
10+
* how that affects eligibility.
11+
*/
12+
13+
import type { OcxConfig } from "../types";
14+
import { isCanonicalOpenAiForwardProvider } from "../providers/openai-tiers";
15+
import { PROVIDER_REGISTRY } from "../providers/registry";
16+
import {
17+
nativeInputModalities,
18+
nativeOpenAiContextWindow,
19+
nativeParallelToolCalls,
20+
nativeReasoningEfforts,
21+
} from "../codex/catalog/metadata";
22+
import { readCatalog, readCodexCatalogPath } from "../codex/catalog/parsing";
23+
import type { RouteCapabilityEvidence } from "./trace";
24+
25+
function cachedCatalogModels(): Array<{ provider: string; id: string; contextWindow?: number; inputModalities?: string[]; reasoningEfforts?: string[]; capabilities?: string[] }> {
26+
try {
27+
const catalog = readCatalog(readCodexCatalogPath());
28+
const models = catalog?.models;
29+
if (!Array.isArray(models)) return [];
30+
return models
31+
.filter((model): model is Record<string, unknown> & { id: string; provider: string } =>
32+
typeof model === "object" && model !== null && typeof model.id === "string" && typeof model.provider === "string")
33+
.map(model => ({
34+
provider: model.provider,
35+
id: model.id,
36+
...(typeof model.contextWindow === "number" ? { contextWindow: model.contextWindow } : {}),
37+
...(Array.isArray(model.inputModalities)
38+
? { inputModalities: model.inputModalities.filter((value): value is string => typeof value === "string") }
39+
: {}),
40+
...(Array.isArray(model.reasoningEfforts)
41+
? { reasoningEfforts: model.reasoningEfforts.filter((value): value is string => typeof value === "string") }
42+
: {}),
43+
...(Array.isArray(model.capabilities)
44+
? { capabilities: model.capabilities.filter((value): value is string => typeof value === "string") }
45+
: {}),
46+
}));
47+
} catch {
48+
return [];
49+
}
50+
}
51+
52+
function isLocalHostname(hostname: string): boolean {
53+
const normalized = hostname.trim().toLowerCase().replace(/\.$/, "");
54+
return normalized === "localhost" || normalized === "127.0.0.1"
55+
|| normalized === "::1" || normalized === "[::1]"
56+
|| normalized.endsWith(".localhost");
57+
}
58+
59+
function isPrivateHostname(hostname: string): boolean {
60+
return hostname.startsWith("10.") || hostname.startsWith("192.168.")
61+
|| /^172\.(1[6-9]|2\d|3[01])\./.test(hostname);
62+
}
63+
64+
function localRemoteEvidence(baseUrl: string | undefined): Pick<RouteCapabilityEvidence, "localOnly" | "remoteAllowed"> {
65+
if (typeof baseUrl !== "string" || baseUrl.length === 0) return {};
66+
try {
67+
const hostname = new URL(baseUrl).hostname;
68+
if (!hostname) return {};
69+
if (isLocalHostname(hostname) || isPrivateHostname(hostname)) return { localOnly: true };
70+
return { remoteAllowed: true };
71+
} catch {
72+
return {};
73+
}
74+
}
75+
76+
/**
77+
* Assemble canonical capability evidence for one `provider/model` candidate.
78+
* Sources (in priority order): provider config maps, provider registry hints,
79+
* cached Codex catalog row, native-model metadata.
80+
*/
81+
export function candidateCapabilityEvidence(
82+
config: OcxConfig,
83+
providerName: string,
84+
modelId: string,
85+
): RouteCapabilityEvidence {
86+
const provider = config.providers[providerName];
87+
const registryEntry = PROVIDER_REGISTRY.find(entry => entry.id === providerName);
88+
const catalogRow = cachedCatalogModels().find(model => model.provider === providerName && model.id === modelId);
89+
const isNative = providerName === "openai" && !modelId.includes("/");
90+
91+
const contextWindow = provider?.modelContextWindows?.[modelId]
92+
?? provider?.contextWindow
93+
?? registryEntry?.modelContextWindows?.[modelId]
94+
?? catalogRow?.contextWindow
95+
?? (isNative ? nativeOpenAiContextWindow(modelId) : undefined);
96+
97+
const modalities = provider?.modelInputModalities?.[modelId]
98+
?? registryEntry?.modelInputModalities?.[modelId]
99+
?? catalogRow?.inputModalities
100+
?? (isNative ? nativeInputModalities(modelId) : undefined);
101+
const image = Array.isArray(modalities)
102+
? modalities.includes("image")
103+
: undefined;
104+
105+
const capabilities = catalogRow?.capabilities ?? [];
106+
const tools = capabilities.includes("tools")
107+
|| (isNative ? true : provider?.parallelToolCalls === true)
108+
|| undefined;
109+
110+
const reasoningEfforts = provider?.modelReasoningEfforts?.[modelId]
111+
?? registryEntry?.modelReasoningEfforts?.[modelId]
112+
?? catalogRow?.reasoningEfforts
113+
?? (isNative ? nativeReasoningEfforts(modelId) : undefined);
114+
115+
const tierSupport = provider?.supportsServiceTier
116+
?? registryEntry?.supportsServiceTier;
117+
const serviceTier = tierSupport === true
118+
? "supported"
119+
: tierSupport === false ? "unsupported" : "unknown";
120+
121+
const localRemote = localRemoteEvidence(provider?.baseUrl);
122+
const encryptedCodexTasks = isCanonicalOpenAiForwardProvider(
123+
provider ?? { adapter: "", authMode: undefined, baseUrl: undefined },
124+
);
125+
126+
return {
127+
...(typeof contextWindow === "number" ? { contextWindow } : {}),
128+
...(typeof image === "boolean" ? { image } : {}),
129+
...(typeof tools === "boolean" ? { tools } : {}),
130+
...(reasoningEfforts !== undefined && reasoningEfforts.length > 0 ? { reasoningEfforts } : {}),
131+
...(serviceTier !== "unknown" ? { serviceTier } : {}),
132+
...localRemote,
133+
encryptedCodexTasks,
134+
};
135+
}

‎src/routing/evaluator.ts‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,27 @@ function requirementFor(
144144
return requirements;
145145
}
146146

147+
/**
148+
* Request-side capability constraints (RI-05): a request that provably needs
149+
* tools or image input imposes the same requirement on candidates as the
150+
* profile's hard requirements. Undefined request evidence adds nothing.
151+
*/
152+
function requestRequirements(
153+
requestEvidence: PolicyRequestEvidence,
154+
capability: RouteCapabilityEvidence | undefined,
155+
): RouteRequirementEvidence[] {
156+
const requirements: RouteRequirementEvidence[] = [];
157+
if (requestEvidence.toolsRequired === true) {
158+
const tools = booleanRequirement("request-tools", true, capability?.tools);
159+
if (tools) requirements.push(tools);
160+
}
161+
if (requestEvidence.imageInputRequired === true) {
162+
const image = booleanRequirement("request-image-input", true, capability?.image);
163+
if (image) requirements.push(image);
164+
}
165+
return requirements;
166+
}
167+
147168
function unsatisfiedOrUnknown(requirements: RouteRequirementEvidence[]): RouteRequirementEvidence[] {
148169
return requirements.filter(requirement => requirement.outcome !== "satisfied");
149170
}
@@ -173,7 +194,10 @@ export function evaluatePolicyProfile(
173194
const evidence = candidateEvidence.find(
174195
candidate => candidate.provider === declared.provider && candidate.model === declared.model,
175196
) ?? { provider: declared.provider, model: declared.model };
176-
const requirements = requirementFor(profile.require, evidence.capability);
197+
const requirements = [
198+
...requirementFor(profile.require, evidence.capability),
199+
...requestRequirements(requestEvidence, evidence.capability),
200+
];
177201
const exclusions: RouteExclusionReason[] = [];
178202
const bad = unsatisfiedOrUnknown(requirements);
179203
for (const requirement of bad) {

‎src/routing/request-evidence.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
/**
2+
* Cheap request-side evidence extraction for policy routing (RI-05).
3+
*
4+
* Extracts only what the request body can prove: whether the caller asked for
5+
* tools and whether the input contains image parts. Context-window size is
6+
* left unknown at routing time (documented limitation) - the dry-run API/CLI
7+
* remains the evidence-inspection surface for context-sensitive profiles.
8+
*/
9+
10+
import type { PolicyRequestEvidence } from "./evaluator";
11+
12+
function inputContainsImage(input: unknown): boolean {
13+
if (typeof input === "string") return false;
14+
if (!Array.isArray(input)) return false;
15+
return input.some(part => {
16+
if (!part || typeof part !== "object" || Array.isArray(part)) return false;
17+
const record = part as Record<string, unknown>;
18+
if (record.type === "image" || record.type === "input_image") return true;
19+
if (record.image_url !== undefined || record.image !== undefined) return true;
20+
return false;
21+
});
22+
}
23+
24+
export function evidenceFromBody(body: unknown): PolicyRequestEvidence {
25+
if (!body || typeof body !== "object" || Array.isArray(body)) return {};
26+
const record = body as Record<string, unknown>;
27+
const tools = Array.isArray(record.tools) && record.tools.length > 0;
28+
const image = inputContainsImage(record.input);
29+
return {
30+
...(tools ? { toolsRequired: true } : {}),
31+
...(image ? { imageInputRequired: true } : {}),
32+
};
33+
}

‎src/server/chat-completions.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import { redactSecretString } from "../lib/redact";
1919
import { resolveClientRetryAfter } from "../lib/retry-after";
2020
import { estimateTokens } from "../lib/token-estimate";
2121
import { routeModel } from "../router";
22+
import { evidenceFromBody } from "../routing/request-evidence";
2223
import { resolveWireProtocolOverride } from "./adapter-resolve";
2324
import type { OcxConfig } from "../types";
2425
import { readJsonRequestBody } from "./request-decompress";
@@ -110,7 +111,7 @@ async function handleChatCompletionsWithBudget(
110111
let nativeRoute = false;
111112
let directRoute = false;
112113
try {
113-
const route = routeModel(config, internalBody.model as string);
114+
const route = routeModel(config, internalBody.model as string, evidenceFromBody(internalBody));
114115
// Settle the wire once so every branch below reads the adapter this model will
115116
// actually use, not the provider-wide default (#404).
116117
route.provider = resolveWireProtocolOverride(route.providerName, route.modelId, route.provider, "chat");

‎src/server/claude-messages.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import {
2626
import { clearableDeadline, idleDeadline } from "../lib/abort";
2727
import { estimateTokens } from "../lib/token-estimate";
2828
import { routeModel } from "../router";
29+
import { evidenceFromBody } from "../routing/request-evidence";
2930
import { resolveWireProtocolOverride } from "./adapter-resolve";
3031
import type { OcxConfig } from "../types";
3132
import { readJsonRequestBody } from "./request-decompress";
@@ -628,7 +629,7 @@ async function handleClaudeMessagesWithBudget(
628629
// verified live 2026-07-11). Strip them for that route; routed providers keep them.
629630
let nativeRoute = false;
630631
try {
631-
const route = routeModel(config, internalBody.model as string);
632+
const route = routeModel(config, internalBody.model as string, evidenceFromBody(internalBody));
632633
// Settle the wire once so the sampling decision below reads the effective
633634
// adapter rather than the provider-wide default (#404).
634635
route.provider = resolveWireProtocolOverride(route.providerName, route.modelId, route.provider, "anthropic");

‎src/server/responses/core.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import {
1717
rememberResponseState,
1818
} from "../../responses/state";
1919
import { routeModel, type RouteResult } from "../../router";
20+
import { evidenceFromBody } from "../../routing/request-evidence";
2021
import {
2122
advanceComboAfterFailure,
2223
comboDefaultEffort,
@@ -1372,7 +1373,7 @@ async function handleResponsesInner(
13721373

13731374
let route: RouteResult;
13741375
try {
1375-
route = routeModel(config, parsed.modelId);
1376+
route = routeModel(config, parsed.modelId, evidenceFromBody(parsed._rawBody));
13761377
} catch (err) {
13771378
if (err instanceof NoAvailableComboTargetsError) {
13781379
return comboUnavailableResponse(err.message);
@@ -1444,7 +1445,7 @@ async function handleResponsesInner(
14441445

14451446
if (fallback?.to && !slugsEquivalent(fallback.to, route.modelId)) {
14461447
try {
1447-
route = routeModel(config, fallback.to);
1448+
route = routeModel(config, fallback.to, evidenceFromBody(parsed._rawBody));
14481449
logCtx.routeDecision = route.routeDecision;
14491450
} catch (err) {
14501451
if (err instanceof NoAvailableComboTargetsError) {

0 commit comments

Comments
 (0)