Skip to content

Commit 410db97

Browse files
authored
Merge pull request #1015 from Wibias/feat/ri-08-cost-aware-routing
feat(routing): add cost-aware policy scoring and limits
2 parents 80be427 + 24e705e commit 410db97

9 files changed

Lines changed: 306 additions & 20 deletions

File tree

‎devlog/_plan/260804_router_intelligence/000_master_plan.md‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -194,8 +194,7 @@ Evidence shapes:
194194
incompleteStreamRate?, recentLatencyMs?, sampleCount?, recencyWeight? }`.
195195
- `quota`: `{ known, headroomTokens?, exhausted?, resetAtMs?,
196196
reauthOrCooling?, reservedHeadroomTokens?, source }`.
197-
- `cost`: `{ estimatedUsd?, priceSource?, incomplete?, limitUsd?,
198-
excludedByLimit? }`.
197+
- `cost`: `{ estimatedUsd?, priceSource?, incomplete?, limitUsd? }`.
199198
- `score`: `{ total, components: { capability?, health?, quota?, cost?,
200199
latency?, configuredPriority? } }`.
201200

‎docs-site/src/content/docs/reference/configuration/routing.md‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ namespace, or reserved bare native families (`gpt-*`, `o1-*`, `o3-*`, `o4-*`, `c
104104
| `candidates` | `{ provider: string; model: string }[]` | required | Explicit allowlist of `provider/model` refs. No implicit expansion. |
105105
| `alias?` | `string` | — | Optional public model id in place of `policy/<id>`. |
106106
| `require?` | object | `{}` | Hard capability requirements evaluated before scoring (see below). |
107-
| `optimize?` | object | latency 0.55, health 0.25, cost 0.10, quota 0.10 | Scoring weights, normalized deterministically. Only `health` and `quota` have score dimensions; the configured-priority share is `1 - health - quota` (default 0.65), and `latency`/`cost` fold into that priority share rather than scoring independently. |
107+
| `optimize?` | object | latency 0.55, health 0.25, cost 0.10, quota 0.10 | Scoring weights, normalized deterministically. `health`, `quota`, and `cost` have score dimensions; the configured-priority share is `1 - health - quota - cost` (default 0.55), and `latency` folds into that priority share rather than scoring independently. |
108108
| `limits?` | object | — | Hard limits, e.g. `maxEstimatedCostUsd` (enforced by the dry-run evaluator when candidate cost evidence is known). |
109109
| `unknownEvidence?` | object | capability `exclude`, health/quota/cost `penalize` | How unknown evidence is treated per dimension: `allow`, `penalize`, or `exclude`. Unknown never becomes zero. |
110110

@@ -171,11 +171,11 @@ candidate evidence: `candidates[].codexAccountId` (Codex pool, provider `openai`
171171
requirements filter first, then deterministic scoring ranks the survivors.
172172

173173
Both are virtual namespaces with aliases and collision validation; they differ in *how* a candidate
174-
is chosen. Profile scoring combines the configured-priority component with the health (RI-06) and
175-
quota (RI-07) score dimensions where evidence is present; the `latency` and `cost` weights fold into
176-
the priority share rather than scoring independently, and cost is also enforced through the
177-
`limits.maxEstimatedCostUsd` cap. Per-request route-decision traces are recorded when a policy
178-
profile executes.
174+
is chosen. Profile scoring combines the configured-priority component with the health (RI-06),
175+
quota (RI-07), and cost (RI-08) score dimensions where evidence is present; the `latency` weight
176+
folds into the priority share rather than scoring independently. Cost is also enforced through the
177+
`limits.maxEstimatedCostUsd` cap (a candidate whose estimated cost exceeds the cap is excluded).
178+
Per-request route-decision traces are recorded when a policy profile executes.
179179

180180
### Catalog eligibility
181181

‎src/router.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import { evaluatePolicyProfile, type PolicyRequestEvidence } from "./routing/eva
3535
import { candidateCapabilityEvidence } from "./routing/capability";
3636
import { policyCandidateHealthEvidence } from "./routing/health";
3737
import { quotaEvidenceForCandidate } from "./routing/quota";
38+
import { costEvidenceForCandidate } from "./routing/cost";
3839

3940
export class NoEligiblePolicyCandidateError extends Error {
4041
/** Evaluation trace (with per-candidate exclusions) when nothing qualified. */
@@ -531,6 +532,11 @@ function routeModelInternal(
531532
? getAccountSet("anthropic")?.activeAccountId
532533
: undefined,
533534
}),
535+
cost: costEvidenceForCandidate({
536+
provider: candidate.provider,
537+
model: candidate.model,
538+
limitUsd: profile.limits.maxEstimatedCostUsd,
539+
}),
534540
}));
535541
const evaluation = evaluatePolicyProfile(config, policyId, policyEvidence ?? {}, candidateEvidence, now);
536542
if (evaluation.selectedIndex === null) {

‎src/routing/cost.ts‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
/**
2+
* Cost-aware policy scoring and limits (RI-08).
3+
*
4+
* Reuses the canonical price/cost normalization (`src/usage/cost.ts`).
5+
* Evidence distinguishes estimated vs authoritative usage and registry vs
6+
* expected prices. Unknown prices stay unknown (never free); the profile's
7+
* `unknownEvidence.cost` policy decides how unknown evidence is handled.
8+
*
9+
* No billing, invoicing, or hidden budgets - a hard per-request ceiling
10+
* (`limits.maxEstimatedCostUsd`) is evaluated deterministically.
11+
*/
12+
13+
import type { OcxUsage } from "../types";
14+
import type { UsageStatus } from "../usage/log";
15+
import { estimateRequestCost, type ServiceTierInput } from "../usage/cost";
16+
import type { RouteCostEvidence } from "./trace";
17+
18+
/** Reference cost (USD) for the relative cost score when no limit is set. */
19+
export const COST_SCORE_REFERENCE_USD = 1.0;
20+
21+
export interface CostEvidenceInput {
22+
provider: string;
23+
model: string;
24+
usage?: OcxUsage;
25+
usageStatus?: UsageStatus;
26+
serviceTier?: ServiceTierInput;
27+
limitUsd?: number;
28+
}
29+
30+
/**
31+
* Assemble cost evidence from the canonical price model. Returns unknown-ish
32+
* evidence (`incomplete: true`, no estimate) when usage or a price is
33+
* missing - never a fabricated zero.
34+
*/
35+
export function costEvidenceForCandidate(input: CostEvidenceInput): RouteCostEvidence {
36+
const limitUsd = input.limitUsd;
37+
if (!input.usage) {
38+
return {
39+
...(limitUsd !== undefined ? { limitUsd } : {}),
40+
incomplete: true,
41+
};
42+
}
43+
const estimate = estimateRequestCost({
44+
provider: input.provider,
45+
model: input.model,
46+
usage: input.usage,
47+
usageStatus: input.usageStatus ?? "estimated",
48+
...(input.serviceTier ? { serviceTier: input.serviceTier } : {}),
49+
});
50+
if (!estimate) {
51+
return {
52+
...(limitUsd !== undefined ? { limitUsd } : {}),
53+
incomplete: true,
54+
priceSource: "unmatched",
55+
};
56+
}
57+
const priceSource = estimate.price?.source ?? "registry";
58+
return {
59+
estimatedUsd: estimate.cost.total,
60+
priceSource,
61+
incomplete: estimate.estimated || priceSource === "expected",
62+
...(limitUsd !== undefined ? { limitUsd } : {}),
63+
};
64+
}
65+
66+
/**
67+
* Deterministic cost score in [0,1]: cheaper is better, relative to
68+
* `COST_SCORE_REFERENCE_USD` (or the profile limit when set). Unknown
69+
* estimates return null so the profile's unknownEvidence policy applies.
70+
*/
71+
export function costScore(evidence: RouteCostEvidence | undefined): number | null {
72+
if (evidence?.estimatedUsd === undefined || !Number.isFinite(evidence.estimatedUsd)) return null;
73+
const reference = typeof evidence.limitUsd === "number" && evidence.limitUsd > 0
74+
? evidence.limitUsd
75+
: COST_SCORE_REFERENCE_USD;
76+
return Math.max(0, Math.min(1, 1 - evidence.estimatedUsd / reference));
77+
}

‎src/routing/evaluator.ts‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,16 @@ import {
2323
import { getRoutingProfile, policyModelId, type NormalizedRoutingProfile } from "./profile";
2424
import { healthScore } from "./health";
2525
import { quotaScore } from "./quota";
26+
import { costScore } from "./cost";
2627

2728
/** Unknown health under "penalize": a low-but-not-zero deterministic floor. */
2829
export const HEALTH_UNKNOWN_PENALTY_SCORE = 0.3;
2930
/** Unknown health under "allow": neutral midpoint of the [0,1] health scale. */
3031
export const HEALTH_UNKNOWN_NEUTRAL_SCORE = 0.5;
3132
/** Unknown quota under "penalize": deterministic low floor. */
3233
export const QUOTA_UNKNOWN_PENALTY_SCORE = 0.3;
34+
/** Unknown cost under "penalize": deterministic low floor. */
35+
export const COST_UNKNOWN_PENALTY_SCORE = 0.3;
3336

3437
export interface PolicyRequestEvidence {
3538
/** Required context window for this request (tokens). */
@@ -342,17 +345,30 @@ export function evaluatePolicyProfile(
342345
}
343346
}
344347

348+
// Cost scoring (RI-08): the hard per-request ceiling was already checked
349+
// above; unknown cost follows the profile's unknownEvidence policy.
350+
const cost = evidence.cost;
351+
let costValue = cost ? costScore(cost) : null;
352+
if (costValue === null && profile.unknownEvidence.cost === "exclude") {
353+
exclusions.push({ code: "unknown-price" });
354+
eligible = false;
355+
} else if (costValue === null && profile.unknownEvidence.cost === "penalize") {
356+
costValue = COST_UNKNOWN_PENALTY_SCORE;
357+
}
358+
345359
const priorityScore = configuredPriorityScore(index, profile.candidates.length);
346360
const healthWeight = profile.optimize.health;
347361
const quotaWeight = profile.optimize.quota;
362+
const costWeight = profile.optimize.cost;
348363
// Only spend a dimension's weight when a value is actually present:
349-
// "allow" leaves missing health/quota components null, so subtracting
364+
// "allow" leaves missing health/quota/cost components null, so subtracting
350365
// their weights would shrink the priority share for evidence the profile
351366
// explicitly permits to be absent. Renormalize those weights back into
352367
// priority instead of silently changing the ranking semantics.
353368
const spentHealth = healthValue !== null ? healthWeight : 0;
354369
const spentQuota = quotaValue !== null ? quotaWeight : 0;
355-
const priorityWeight = Math.max(0, 1 - spentHealth - spentQuota);
370+
const spentCost = costValue !== null ? costWeight : 0;
371+
const priorityWeight = Math.max(0, 1 - spentHealth - spentQuota - spentCost);
356372
const components: RouteScoreEvidence["components"] = { configuredPriority: priorityScore };
357373
let total = priorityWeight * priorityScore;
358374
if (healthWeight > 0 && healthValue !== null) {
@@ -363,6 +379,10 @@ export function evaluatePolicyProfile(
363379
total += quotaWeight * quotaValue;
364380
components.quota = quotaValue;
365381
}
382+
if (costWeight > 0 && costValue !== null) {
383+
total += costWeight * costValue;
384+
components.cost = costValue;
385+
}
366386
const score: RouteScoreEvidence = { total, components };
367387
const evaluated: PolicyEvaluationCandidate = {
368388
provider: evidence.provider,

‎src/routing/trace.ts‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,6 @@ export interface RouteCostEvidence {
8383
priceSource?: string;
8484
incomplete?: boolean;
8585
limitUsd?: number;
86-
excludedByLimit?: boolean;
8786
}
8887

8988
export interface RouteScoreEvidence {
@@ -533,7 +532,6 @@ function parseCost(raw: unknown, caps: ParseCaps): RouteCostEvidence | undefined
533532
}
534533
if (typeof raw.incomplete === "boolean") out.incomplete = raw.incomplete;
535534
if (finiteNumber(raw.limitUsd)) out.limitUsd = raw.limitUsd;
536-
if (typeof raw.excludedByLimit === "boolean") out.excludedByLimit = raw.excludedByLimit;
537535
return Object.keys(out).length > 0 ? out : undefined;
538536
}
539537

‎tests/cost-scoring.test.ts‎

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
2+
import { mkdtempSync, rmSync } from "node:fs";
3+
import { tmpdir } from "node:os";
4+
import { join } from "node:path";
5+
import { costEvidenceForCandidate, costScore } from "../src/routing/cost";
6+
import { evaluatePolicyProfile, COST_UNKNOWN_PENALTY_SCORE } from "../src/routing/evaluator";
7+
import type { OcxConfig } from "../src/types";
8+
9+
let testDir = "";
10+
let previousHome: string | undefined;
11+
12+
beforeEach(() => {
13+
previousHome = process.env.OPENCODEX_HOME;
14+
testDir = mkdtempSync(join(tmpdir(), "ocx-cost-"));
15+
process.env.OPENCODEX_HOME = testDir;
16+
});
17+
18+
afterEach(() => {
19+
if (previousHome === undefined) delete process.env.OPENCODEX_HOME;
20+
else process.env.OPENCODEX_HOME = previousHome;
21+
if (testDir) rmSync(testDir, { recursive: true, force: true });
22+
});
23+
24+
function config(overrides: Partial<OcxConfig> = {}): OcxConfig {
25+
return {
26+
port: 10100,
27+
defaultProvider: "a",
28+
providers: {
29+
a: { adapter: "openai-chat", baseUrl: "https://a.example/v1", apiKey: "ka", models: ["m1", "m2"] },
30+
anthropic: { adapter: "anthropic", baseUrl: "https://api.anthropic.com/v1", apiKey: "kan", models: ["claude-opus-5", "claude-sonnet-5"] },
31+
},
32+
routingProfiles: {
33+
cost: {
34+
candidates: [
35+
{ provider: "anthropic", model: "claude-opus-5" },
36+
{ provider: "anthropic", model: "claude-sonnet-5" },
37+
],
38+
optimize: { cost: 0.8 },
39+
},
40+
},
41+
...overrides,
42+
};
43+
}
44+
45+
const USAGE = { inputTokens: 1000, outputTokens: 100, estimated: true };
46+
47+
describe("cost-aware scoring (RI-08)", () => {
48+
test("known price produces an estimate with provenance", async () => {
49+
const evidence = costEvidenceForCandidate({
50+
provider: "anthropic",
51+
model: "claude-opus-5",
52+
usage: { inputTokens: 1000, outputTokens: 100 },
53+
usageStatus: "reported",
54+
});
55+
expect(evidence.estimatedUsd).toBeGreaterThan(0);
56+
// Canonical jawcode pricing is authoritative; the expected-price overlay
57+
// is the fallback. Both carry a stable source code.
58+
expect(["jawcode", "expected"]).toContain(evidence.priceSource);
59+
expect(evidence.incomplete).toBe(evidence.priceSource === "expected");
60+
expect(costScore(evidence)).not.toBeNull();
61+
});
62+
63+
test("unknown price and missing usage stay unknown - never zero", async () => {
64+
const noPrice = costEvidenceForCandidate({ provider: "a", model: "m1", usage: USAGE });
65+
expect(noPrice.estimatedUsd).toBeUndefined();
66+
expect(noPrice.priceSource).toBe("unmatched");
67+
expect(noPrice.incomplete).toBe(true);
68+
expect(costScore(noPrice)).toBeNull();
69+
70+
const noUsage = costEvidenceForCandidate({ provider: "anthropic", model: "claude-opus-5" });
71+
expect(noUsage.estimatedUsd).toBeUndefined();
72+
expect(noUsage.incomplete).toBe(true);
73+
expect(costScore(noUsage)).toBeNull();
74+
});
75+
76+
test("hard maximum estimated cost excludes candidates and records the limit", async () => {
77+
const limited = config({
78+
routingProfiles: {
79+
cost: {
80+
candidates: [{ provider: "anthropic", model: "claude-opus-5" }],
81+
optimize: { cost: 0.8 },
82+
limits: { maxEstimatedCostUsd: 0.000001 },
83+
unknownEvidence: { capability: "allow", health: "penalize", quota: "penalize", cost: "penalize" },
84+
},
85+
},
86+
});
87+
const evidence = costEvidenceForCandidate({
88+
provider: "anthropic",
89+
model: "claude-opus-5",
90+
usage: USAGE,
91+
limitUsd: 0.000001,
92+
});
93+
expect(evidence.limitUsd).toBe(0.000001);
94+
95+
const result = evaluatePolicyProfile(limited, "cost", {}, [
96+
{ provider: "anthropic", model: "claude-opus-5", capability: { contextWindow: 200000 }, cost: evidence },
97+
]);
98+
expect(result.candidates[0]!.eligible).toBe(false);
99+
expect(result.candidates[0]!.exclusions.some(exclusion => exclusion.code === "cost-limit")).toBe(true);
100+
expect(result.selectedIndex).toBeNull();
101+
});
102+
103+
test("unknown cost follows the profile policy (exclude / penalize / allow)", async () => {
104+
const strict = config({
105+
routingProfiles: {
106+
c: {
107+
candidates: [{ provider: "a", model: "m1" }],
108+
unknownEvidence: { capability: "allow", health: "penalize", quota: "penalize", cost: "exclude" },
109+
},
110+
},
111+
});
112+
const excluded = evaluatePolicyProfile(strict, "c", {}, [
113+
{ provider: "a", model: "m1", capability: { contextWindow: 200000 } },
114+
]);
115+
expect(excluded.candidates[0]!.eligible).toBe(false);
116+
expect(excluded.candidates[0]!.exclusions.some(exclusion => exclusion.code === "unknown-price")).toBe(true);
117+
118+
const penalizing = config({
119+
routingProfiles: {
120+
c: {
121+
candidates: [{ provider: "a", model: "m1" }],
122+
unknownEvidence: { capability: "allow", health: "penalize", quota: "penalize", cost: "penalize" },
123+
},
124+
},
125+
});
126+
const penalized = evaluatePolicyProfile(penalizing, "c", {}, [
127+
{ provider: "a", model: "m1", capability: { contextWindow: 200000 } },
128+
]);
129+
expect(penalized.candidates[0]!.eligible).toBe(true);
130+
expect(penalized.candidates[0]!.score!.components.cost).toBe(COST_UNKNOWN_PENALTY_SCORE);
131+
132+
const allowing = config({
133+
routingProfiles: {
134+
c: {
135+
candidates: [{ provider: "a", model: "m1" }],
136+
unknownEvidence: { capability: "allow", health: "penalize", quota: "penalize", cost: "allow" },
137+
},
138+
},
139+
});
140+
const allowed = evaluatePolicyProfile(allowing, "c", {}, [
141+
{ provider: "a", model: "m1", capability: { contextWindow: 200000 } },
142+
]);
143+
expect(allowed.candidates[0]!.eligible).toBe(true);
144+
// Allowed-unknown cost: no cost component, and the unspent cost weight
145+
// folds back into priority instead of scoring unknown as zero.
146+
expect(allowed.candidates[0]!.score!.components.cost).toBeUndefined();
147+
expect(allowed.candidates[0]!.score!.total).toBeGreaterThan(0);
148+
});
149+
150+
test("cheaper candidates win when cost is weighted", async () => {
151+
const expensive = costEvidenceForCandidate({
152+
provider: "anthropic",
153+
model: "claude-opus-5",
154+
usage: { inputTokens: 200_000, outputTokens: 20_000, estimated: true },
155+
});
156+
const cheap = costEvidenceForCandidate({
157+
provider: "anthropic",
158+
model: "claude-sonnet-5",
159+
usage: { inputTokens: 2_000, outputTokens: 200, estimated: true },
160+
});
161+
const result = evaluatePolicyProfile(config(), "cost", {}, [
162+
{ provider: "anthropic", model: "claude-opus-5", capability: { contextWindow: 200000 }, cost: expensive },
163+
{ provider: "anthropic", model: "claude-sonnet-5", capability: { contextWindow: 200000 }, cost: cheap },
164+
]);
165+
expect(result.selectedIndex).toBe(1);
166+
expect(result.candidates[1]!.score!.components.cost).toBeGreaterThan(result.candidates[0]!.score!.components.cost!);
167+
});
168+
169+
test("trace carries cost evidence and the cost component", async () => {
170+
const evidence = costEvidenceForCandidate({ provider: "anthropic", model: "claude-opus-5", usage: USAGE });
171+
const result = evaluatePolicyProfile(config(), "cost", {}, [
172+
{ provider: "anthropic", model: "claude-opus-5", capability: { contextWindow: 200000 }, cost: evidence },
173+
{ provider: "anthropic", model: "claude-sonnet-5", capability: { contextWindow: 200000 } },
174+
]);
175+
expect(result.trace.candidates[0]!.cost).toBeDefined();
176+
expect(result.trace.candidates[0]!.score!.components.cost).toBeDefined();
177+
});
178+
});

0 commit comments

Comments
 (0)