Skip to content

Commit d887c12

Browse files
committed
feat(control-plane): expose route decision explanations (RI-09)
1 parent 410db97 commit d887c12

7 files changed

Lines changed: 354 additions & 43 deletions

File tree

‎devlog/_plan/260804_router_intelligence/001_pr_stack_status.md‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,9 +46,9 @@ other; closing one is a maintainer decision and neither is stale.
4646
| RI-04 | `feat/ri-04-policy-profile-core` | `dev` (post-#1005 merge) | `31c9f0b28` | #1011 | https://github.com/lidge-jun/opencodex/pull/1011 | MERGED |
4747
| RI-05 | `feat/ri-05-capability-aware-routing` | `dev` (post-#1011 merge) | `088194a3a` | #1012 | https://github.com/lidge-jun/opencodex/pull/1012 | MERGED |
4848
| RI-06 | `feat/ri-06-health-aware-routing` | `dev` (post-#1012 merge) | `af692bb7a` | #1013 | https://github.com/lidge-jun/opencodex/pull/1013 | MERGED |
49-
| RI-07 | `feat/ri-07-quota-aware-routing` | `dev` (post-#1013 merge) | `0c5100271` (pre-restack) | #1014 | https://github.com/lidge-jun/opencodex/pull/1014 | in progress |
50-
| RI-08 | `feat/ri-08-cost-aware-routing` | `feat/ri-07` head | pending | pending | pending | queued |
51-
| RI-09 | `feat/ri-09-route-explainability-api` | `feat/ri-08` head | pending | pending | pending | queued |
49+
| RI-07 | `feat/ri-07-quota-aware-routing` | `dev` (post-#1013 merge) | `1f07c00b8` | #1014 | https://github.com/lidge-jun/opencodex/pull/1014 | MERGED |
50+
| RI-08 | `feat/ri-08-cost-aware-routing` | `dev` (post-#1014 merge) | `410db97e4` | #1015 | https://github.com/lidge-jun/opencodex/pull/1015 | MERGED |
51+
| RI-09 | `feat/ri-09-route-explainability-api` | `dev` (post-#1015 merge) | pending | #1016 | https://github.com/lidge-jun/opencodex/pull/1016 | in progress |
5252
| RI-10 | `feat/ri-10-routing-intelligence-ui` | `feat/ri-09` head | pending | pending | pending | queued |
5353

5454
## Per-PR acceptance log
@@ -258,6 +258,24 @@ other; closing one is a maintainer decision and neither is stale.
258258
- Base sync deferred: waiting for RI-05 (#1012) to merge before updating
259259
these branches from `dev`.
260260

261+
### RI-09 - feat/ri-09-route-explainability-api
262+
263+
- Base SHA: `66ae44aca4aff3d7d9f8317f7badcfb485f7f736` (RI-08 head)
264+
- Reviewed commit: same as final (author self-review before push)
265+
- Findings (self-review): 2 fixed pre-push - (1) variable shadowing in the
266+
dry-run handler (`profile` string vs profile config) - renamed to
267+
`profileConfig`; (2) CLI USAGE template syntax broken by the patch - fixed.
268+
- Final commit: pending (recorded after commit)
269+
- PR: pending
270+
- Verification:
271+
- `bun x tsc --noEmit`: PASSED (0 errors)
272+
- `bun run test tests/route-explainability.test.ts`: 4/4 pass -
273+
trace+attempts+outcome merge, 404 unknown ids, pre-trace rows, dry-run
274+
auto-evidence
275+
- Focused regression suites: 70/70 pass across 8 files
276+
- `bun run privacy:scan`: passed
277+
- Remaining Low findings: none
278+
261279
## Baseline note
262280

263281
The full-suite baseline on this Windows machine did not complete within the

‎src/cli/observe.ts‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import {
1414
const USAGE = `Usage:
1515
ocx observe logs [--provider <name>] [--model <id>] [--status <code>]
1616
[--limit <n>] [--follow] [--json|--jsonl]
17+
ocx logs explain <request-id> [--json]
1718
ocx logs rebuild-index
1819
ocx logs index-status
1920
ocx observe usage [--range <7d|30d|all>] [--surface <all|codex|claude|grok>] [--json]
@@ -81,6 +82,17 @@ async function logs(argv: string[], deps: RuntimeApiDeps): Promise<void> {
8182
} while (true);
8283
}
8384

85+
async function explain(argv: string[], deps: RuntimeApiDeps): Promise<void> {
86+
const args = [...argv];
87+
const requestId = args.shift();
88+
const wantsJson = takeFlag(args, "--json");
89+
if (!requestId) throw new CliUsageError("request id is required", USAGE);
90+
rejectArgs(args, USAGE);
91+
const encoded = encodeURIComponent(requestId);
92+
const result = await runtimeRequest(`/api/request-history/${encoded}/route-decision`, {}, deps);
93+
printData(result, wantsJson, wantsJson ? undefined : [JSON.stringify(result, null, 2)]);
94+
}
95+
8496
async function rebuildIndex(argv: string[], deps: RuntimeApiDeps): Promise<void> {
8597
const args = [...argv];
8698
const wantsJson = takeFlag(args, "--json");
@@ -140,7 +152,8 @@ export async function handleObserveCommand(argv: string[], deps: RuntimeApiDeps
140152
const [sub = "logs", ...rest] = argv;
141153
if (sub === "logs") {
142154
const action = rest[0];
143-
if (action === "rebuild-index") await rebuildIndex(rest.slice(1), deps);
155+
if (action === "explain") await explain(rest.slice(1), deps);
156+
else if (action === "rebuild-index") await rebuildIndex(rest.slice(1), deps);
144157
else if (action === "index-status") await indexStatus(rest.slice(1), deps);
145158
else await logs(rest, deps);
146159
}

‎src/cli/route-policy.ts‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ const USAGE = `Usage:
1313
ocx route policy list [--json]
1414
ocx route policy show <id> [--json]
1515
ocx route policy dry-run <id> [--model-context <tokens>] [--tools]
16+
[--image] [--structured-output] [--json]
17+
ocx route policy evaluate <id> [--model-context <tokens>] [--tools]
1618
[--image] [--structured-output] [--json]`;
1719

1820
interface ProfileRow {
@@ -40,7 +42,7 @@ async function show(argv: string[], deps: RuntimeApiDeps): Promise<void> {
4042
const args = [...argv];
4143
const id = args.shift();
4244
const wantsJson = takeFlag(args, "--json");
43-
if (!id) throw new CliUsageError("profile id is required", USAGE);
45+
if (!id || id.startsWith("-")) throw new CliUsageError("profile id is required", USAGE);
4446
rejectArgs(args, USAGE);
4547
const result = await runtimeRequest<{ profiles?: ProfileRow[] }>("/api/routing-profiles", {}, deps);
4648
const profile = (result.profiles ?? []).find(candidate => candidate.id === id);
@@ -52,7 +54,7 @@ async function dryRun(argv: string[], deps: RuntimeApiDeps): Promise<void> {
5254
const args = [...argv];
5355
const id = args.shift();
5456
const wantsJson = takeFlag(args, "--json");
55-
if (!id) throw new CliUsageError("profile id is required", USAGE);
57+
if (!id || id.startsWith("-")) throw new CliUsageError("profile id is required", USAGE);
5658
const modelContext = takeIntegerOption(args, "--model-context", { min: 1 });
5759
const tools = takeFlag(args, "--tools");
5860
const image = takeFlag(args, "--image");
@@ -81,10 +83,10 @@ async function dryRun(argv: string[], deps: RuntimeApiDeps): Promise<void> {
8183
export async function handleRoutePolicyCommand(argv: string[], deps: RuntimeApiDeps = {}): Promise<number> {
8284
return runCliAction(async () => {
8385
const [sub, ...rest] = argv;
84-
if (!sub) throw new CliUsageError("route policy requires a subcommand (list, show, dry-run)", USAGE);
86+
if (!sub) throw new CliUsageError("route policy requires a subcommand (list, show, dry-run, evaluate)", USAGE);
8587
if (sub === "list") await list(rest, deps);
8688
else if (sub === "show") await show(rest, deps);
87-
else if (sub === "dry-run") await dryRun(rest, deps);
89+
else if (sub === "dry-run" || sub === "evaluate") await dryRun(rest, deps);
8890
else throw new CliUsageError(`unknown route policy command: ${sub}`, USAGE);
8991
});
9092
}

‎src/routing/capability.ts‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -185,9 +185,12 @@ export function candidateCapabilityEvidence(
185185
: tierSupport === false ? "unsupported" : "unknown";
186186

187187
const localRemote = localRemoteEvidence(provider?.baseUrl);
188-
const encryptedCodexTasks = isCanonicalOpenAiForwardProvider(
189-
provider ?? { adapter: "", authMode: undefined, baseUrl: undefined },
190-
);
188+
// Only emit a definitive encryptedCodexTasks value when the provider is
189+
// present. An absent/unconfigured provider must stay unknown so
190+
// require.encryptedCodexTasks does not fail closed on missing config.
191+
const encryptedCodexTasks = provider === undefined
192+
? undefined
193+
: isCanonicalOpenAiForwardProvider(provider);
191194

192195
return {
193196
...(typeof contextWindow === "number" ? { contextWindow } : {}),
@@ -196,6 +199,6 @@ export function candidateCapabilityEvidence(
196199
...(reasoningEfforts !== undefined && reasoningEfforts.length > 0 ? { reasoningEfforts } : {}),
197200
...(serviceTier !== "unknown" ? { serviceTier } : {}),
198201
...localRemote,
199-
encryptedCodexTasks,
202+
...(typeof encryptedCodexTasks === "boolean" ? { encryptedCodexTasks } : {}),
200203
};
201204
}

‎src/server/management/request-history-routes.ts‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
*
44
* - `GET /api/request-history` - keyset-paginated rows with filters
55
* - `GET /api/request-history/:requestId` - one canonical row
6+
* - `GET /api/request-history/:requestId/route-decision` - why-this-route
7+
* explanation (RI-09): durable trace + attempts + outcome
68
*
79
* The index is a derived projection of `usage.jsonl`; every response carries
810
* an `index` status block so callers can see schema version, indexed rows and
@@ -19,6 +21,7 @@ import { requestLogEntryFromPersistedUsage } from "../request-log";
1921
import { requestLogDto } from "./shared";
2022
import { jsonResponse } from "../auth-cors";
2123
import type { ManagementContext } from "./context";
24+
import type { PersistedUsageEntry } from "../../usage/log";
2225

2326
function parseQueryInt(raw: string | null): number | undefined | "invalid" {
2427
if (raw === null) return undefined;
@@ -28,6 +31,17 @@ function parseQueryInt(raw: string | null): number | undefined | "invalid" {
2831
return Number.isInteger(value) ? value : "invalid";
2932
}
3033

34+
function finalAttemptTarget(entry: PersistedUsageEntry): { provider: string; model: string } {
35+
const attempts = entry.attempts;
36+
if (Array.isArray(attempts) && attempts.length > 0) {
37+
const last = attempts[attempts.length - 1];
38+
if (last && typeof last.provider === "string" && typeof last.model === "string") {
39+
return { provider: last.provider, model: last.model };
40+
}
41+
}
42+
return { provider: entry.provider, model: entry.model };
43+
}
44+
3145
export async function handleRequestHistoryRoutes(ctx: ManagementContext): Promise<Response | null> {
3246
const { url, req, config } = ctx;
3347
if (!url.pathname.startsWith("/api/request-history")) return null;
@@ -113,6 +127,50 @@ export async function handleRequestHistoryRoutes(ctx: ManagementContext): Promis
113127
}
114128

115129
if (url.pathname.startsWith("/api/request-history/") && req.method === "GET") {
130+
// Why-this-route explanation (RI-09): trace + attempts + outcome.
131+
if (url.pathname.endsWith("/route-decision")) {
132+
let requestId: string;
133+
try {
134+
requestId = decodeURIComponent(
135+
url.pathname.slice("/api/request-history/".length, -"/route-decision".length),
136+
);
137+
} catch {
138+
return jsonResponse({ error: { code: "not_found", message: "unknown request" } }, 404, req, config);
139+
}
140+
if (!requestId || requestId.includes("/")) {
141+
return jsonResponse({ error: { code: "not_found", message: "unknown request" } }, 404, req, config);
142+
}
143+
const entry = await requestHistoryRowById(requestId);
144+
if (!entry) {
145+
return jsonResponse({ error: { code: "not_found", message: "unknown request" } }, 404, req, config);
146+
}
147+
const trace = entry.routeDecision ?? null;
148+
const final = finalAttemptTarget(entry);
149+
return jsonResponse({
150+
requestId,
151+
routeDecision: trace,
152+
attemptSequence: entry.attempts ?? [],
153+
outcome: {
154+
status: entry.status,
155+
...(entry.terminalStatus ? { terminalStatus: entry.terminalStatus } : {}),
156+
...(entry.closeReason ? { closeReason: entry.closeReason } : {}),
157+
...(entry.errorCode ? { errorCode: entry.errorCode } : {}),
158+
...(entry.durationMs !== undefined ? { durationMs: entry.durationMs } : {}),
159+
...(entry.usageStatus ? { usageStatus: entry.usageStatus } : {}),
160+
},
161+
summary: {
162+
requestedModel: entry.requestedModel ?? entry.model,
163+
routeKind: trace?.routeKind ?? null,
164+
...(trace?.profile ? { profileId: trace.profile.id, revision: trace.profile.revision } : {}),
165+
selected: trace?.selected
166+
? { provider: trace.selected.provider, model: trace.selected.model }
167+
: null,
168+
finalProvider: final.provider,
169+
finalModel: final.model,
170+
},
171+
}, 200, req, config);
172+
}
173+
116174
let requestId: string;
117175
try {
118176
requestId = decodeURIComponent(url.pathname.slice("/api/request-history/".length));

‎src/server/management/routing-profile-routes.ts‎

Lines changed: 46 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import { evaluatePolicyProfile, type PolicyCandidateEvidence, type PolicyRequest
1111
import { candidateCapabilityEvidence } from "../../routing/capability";
1212
import { policyCandidateHealthEvidence } from "../../routing/health";
1313
import { quotaEvidenceForCandidate } from "../../routing/quota";
14+
import { costEvidenceForCandidate } from "../../routing/cost";
1415
import { providerCodexAccountMode } from "../../providers/registry";
1516
import { getEffectiveActiveCodexAccountId } from "../../codex/routing";
1617
import { getAccountSet } from "../../oauth/store";
@@ -19,6 +20,7 @@ import { isPlainRecord } from "./shared";
1920
import { readManagementJsonBody, rethrowManagementBodyTooLarge } from "./body";
2021
import { jsonResponse } from "../auth-cors";
2122
import type { ManagementContext } from "./context";
23+
import type { OcxConfig } from "../../types";
2224

2325
function profileDto(config: Parameters<typeof getRoutingProfile>[0], id: string): Record<string, unknown> | null {
2426
const profile = getRoutingProfile(config, id);
@@ -86,6 +88,49 @@ function parseCandidateEvidence(raw: unknown): PolicyCandidateEvidence[] | null
8688
return out;
8789
}
8890

91+
function assembleCandidateEvidence(
92+
config: OcxConfig,
93+
profile: NonNullable<ReturnType<typeof getRoutingProfile>>,
94+
): PolicyCandidateEvidence[] {
95+
// Match execution: fill the same candidate evidence the router would
96+
// assemble, so dry-run/evaluate reports the same eligibility as real routing
97+
// instead of treating every capability as unknown. Cost is always present so
98+
// evaluate mode can surface the profile limit even without a usage estimate.
99+
return profile.candidates.map(candidate => ({
100+
provider: candidate.provider,
101+
model: candidate.model,
102+
capability: candidateCapabilityEvidence(config, candidate.provider, candidate.model),
103+
health: policyCandidateHealthEvidence(config, candidate),
104+
quota: quotaEvidenceForCandidate({
105+
provider: candidate.provider,
106+
model: candidate.model,
107+
...(candidate.provider === OPENAI_CODEX_PROVIDER_ID
108+
&& providerCodexAccountMode(
109+
OPENAI_CODEX_PROVIDER_ID,
110+
config.providers[OPENAI_CODEX_PROVIDER_ID],
111+
) === "pool"
112+
? (() => {
113+
const codexAccountId = getEffectiveActiveCodexAccountId(config);
114+
return {
115+
codexAccountId,
116+
codexAccountPlan: codexAccountId
117+
? config.codexAccounts?.find(account => account.id === codexAccountId)?.plan
118+
: undefined,
119+
};
120+
})()
121+
: {}),
122+
accountRef: candidate.provider === "anthropic"
123+
? getAccountSet("anthropic")?.activeAccountId
124+
: undefined,
125+
}),
126+
cost: costEvidenceForCandidate({
127+
provider: candidate.provider,
128+
model: candidate.model,
129+
limitUsd: profile.limits.maxEstimatedCostUsd,
130+
}),
131+
}));
132+
}
133+
89134
export async function handleRoutingProfileRoutes(ctx: ManagementContext): Promise<Response | null> {
90135
const { req, url, config } = ctx;
91136

@@ -119,37 +164,7 @@ export async function handleRoutingProfileRoutes(ctx: ManagementContext): Promis
119164
return jsonResponse({ error: { code: "invalid_evidence", message: "evidence must be an object" } }, 400, req, config);
120165
}
121166
const candidateEvidence = body.candidates === undefined
122-
// Match execution: fill the same candidate evidence the router would
123-
// assemble, so dry-run reports the same eligibility as real routing
124-
// instead of treating every capability as unknown.
125-
? resolvedProfile.candidates.map(candidate => ({
126-
provider: candidate.provider,
127-
model: candidate.model,
128-
capability: candidateCapabilityEvidence(config, candidate.provider, candidate.model),
129-
health: policyCandidateHealthEvidence(config, candidate),
130-
quota: quotaEvidenceForCandidate({
131-
provider: candidate.provider,
132-
model: candidate.model,
133-
...(candidate.provider === OPENAI_CODEX_PROVIDER_ID
134-
&& providerCodexAccountMode(
135-
OPENAI_CODEX_PROVIDER_ID,
136-
config.providers[OPENAI_CODEX_PROVIDER_ID],
137-
) === "pool"
138-
? (() => {
139-
const codexAccountId = getEffectiveActiveCodexAccountId(config);
140-
return {
141-
codexAccountId,
142-
codexAccountPlan: codexAccountId
143-
? config.codexAccounts?.find(account => account.id === codexAccountId)?.plan
144-
: undefined,
145-
};
146-
})()
147-
: {}),
148-
accountRef: candidate.provider === "anthropic"
149-
? getAccountSet("anthropic")?.activeAccountId
150-
: undefined,
151-
}),
152-
}))
167+
? assembleCandidateEvidence(config, resolvedProfile)
153168
: parseCandidateEvidence(body.candidates);
154169
if (candidateEvidence === null) {
155170
return jsonResponse({ error: { code: "invalid_candidates", message: "candidates must be an array of evidence objects" } }, 400, req, config);

0 commit comments

Comments
 (0)