|
| 1 | +import { describe, expect, test } from "bun:test"; |
| 2 | +import { candidateCapabilityEvidence } from "../src/routing/capability"; |
| 3 | +import { PROVIDER_REGISTRY } from "../src/providers/registry"; |
| 4 | +import { modelRecordValue } from "../src/reasoning-effort"; |
| 5 | +import { isModelTextOnly } from "../src/vision"; |
| 6 | +import type { OcxConfig, OcxProviderConfig } from "../src/types"; |
| 7 | + |
| 8 | +/** |
| 9 | + * `candidateCapabilityEvidence` describes what the resolver will do with a candidate, |
| 10 | + * so it has to match the resolver. Every runtime reader of `modelContextWindows`, |
| 11 | + * `modelInputModalities` and `modelReasoningEfforts` goes through `modelRecordValue` |
| 12 | + * (`src/reasoning-effort.ts:108`, `src/server/effort-policy.ts:122`, |
| 13 | + * `src/vision/index.ts:34`, `src/codex/catalog/provider-fetch.ts:612`), which accepts a |
| 14 | + * family entry for a tagged id. This file pins the evidence to that same rule. |
| 15 | + * |
| 16 | + * The window matters most: a bare lookup did not degrade to unknown there, it fell |
| 17 | + * through to the provider-wide `contextWindow` — a definite wrong answer, which the |
| 18 | + * module's own "unknown is not zero" contract is written to avoid. |
| 19 | + */ |
| 20 | + |
| 21 | +function providerWithFamilyEntries(): OcxProviderConfig { |
| 22 | + return { |
| 23 | + adapter: "openai-chat", |
| 24 | + baseUrl: "https://example.test/v1", |
| 25 | + contextWindow: 8_000, |
| 26 | + models: ["gpt-oss:120b"], |
| 27 | + modelContextWindows: { "gpt-oss": 131_072 }, |
| 28 | + modelInputModalities: { "gpt-oss": ["text"] }, |
| 29 | + modelReasoningEfforts: { "gpt-oss": ["low", "high"] }, |
| 30 | + } as unknown as OcxProviderConfig; |
| 31 | +} |
| 32 | + |
| 33 | +function configFor(provider: OcxProviderConfig): OcxConfig { |
| 34 | + return { providers: { custom: provider } } as unknown as OcxConfig; |
| 35 | +} |
| 36 | + |
| 37 | +describe("candidateCapabilityEvidence model matching", () => { |
| 38 | + test("a family entry covers its tagged siblings, as the resolver does", () => { |
| 39 | + const provider = providerWithFamilyEntries(); |
| 40 | + |
| 41 | + // Ground truth first: what the runtime itself resolves off this config. |
| 42 | + expect(modelRecordValue(provider.modelContextWindows, "gpt-oss:120b")).toBe(131_072); |
| 43 | + expect(isModelTextOnly(provider, "gpt-oss:120b")).toBe(true); |
| 44 | + |
| 45 | + const evidence = candidateCapabilityEvidence(configFor(provider), "custom", "gpt-oss:120b"); |
| 46 | + expect(evidence.contextWindow).toBe(131_072); |
| 47 | + expect(evidence.image).toBe(false); |
| 48 | + expect(evidence.reasoningEfforts).toEqual(["low", "high"]); |
| 49 | + }); |
| 50 | + |
| 51 | + test("the window does not fall through to the provider-wide value", () => { |
| 52 | + // The specific regression: the provider-wide 8_000 is not "unknown", it is a |
| 53 | + // definite answer belonging to a different model, and routing would act on it. |
| 54 | + // Asserted as the exact expected number rather than `not.toBe(8_000)`, which |
| 55 | + // would also pass for `undefined` or any other wrong value. |
| 56 | + const evidence = candidateCapabilityEvidence( |
| 57 | + configFor(providerWithFamilyEntries()), |
| 58 | + "custom", |
| 59 | + "gpt-oss:120b", |
| 60 | + ); |
| 61 | + expect(evidence.contextWindow).toBe(131_072); |
| 62 | + }); |
| 63 | + |
| 64 | + test("an exact entry still wins over the family entry", () => { |
| 65 | + const provider = { |
| 66 | + ...providerWithFamilyEntries(), |
| 67 | + modelContextWindows: { "gpt-oss": 131_072, "gpt-oss:20b": 32_000 }, |
| 68 | + } as unknown as OcxProviderConfig; |
| 69 | + expect(candidateCapabilityEvidence(configFor(provider), "custom", "gpt-oss:20b").contextWindow) |
| 70 | + .toBe(32_000); |
| 71 | + }); |
| 72 | + |
| 73 | + test("an unrelated model still falls back to the provider-wide window", () => { |
| 74 | + const evidence = candidateCapabilityEvidence( |
| 75 | + configFor(providerWithFamilyEntries()), |
| 76 | + "custom", |
| 77 | + "some-other-model", |
| 78 | + ); |
| 79 | + expect(evidence.contextWindow).toBe(8_000); |
| 80 | + expect(evidence.reasoningEfforts).toBeUndefined(); |
| 81 | + }); |
| 82 | + |
| 83 | + test("a registry entry covers its tagged siblings with no provider configured", () => { |
| 84 | + // The three registry lookups (capability.ts lines 170/180/206) are a separate branch |
| 85 | + // from the configured-provider ones above: they are only reached when the provider is |
| 86 | + // absent from the config, which every other case here supplies. |
| 87 | + const registryEntry = PROVIDER_REGISTRY.find(entry => entry.id === "xai"); |
| 88 | + if (!registryEntry) throw new Error("fixture drift: no `xai` entry in PROVIDER_REGISTRY"); |
| 89 | + |
| 90 | + // Pin the fixture's shape rather than its values, so registry churn does not turn |
| 91 | + // into a false failure here while real drift still does. |
| 92 | + const family = "grok-4.6"; |
| 93 | + expect(registryEntry.modelContextWindows?.[family]).toBeNumber(); |
| 94 | + expect(registryEntry.modelInputModalities?.[family]).toBeArray(); |
| 95 | + expect(registryEntry.modelReasoningEfforts?.[family]).toBeArray(); |
| 96 | + |
| 97 | + const emptyConfig = { providers: {} } as unknown as OcxConfig; |
| 98 | + const evidence = candidateCapabilityEvidence(emptyConfig, "xai", `${family}:latest`); |
| 99 | + |
| 100 | + expect(evidence.contextWindow).toBe(registryEntry.modelContextWindows![family]); |
| 101 | + expect(evidence.image).toBe(registryEntry.modelInputModalities![family].includes("image")); |
| 102 | + expect(evidence.reasoningEfforts).toEqual(registryEntry.modelReasoningEfforts![family]); |
| 103 | + }); |
| 104 | + |
| 105 | + test("noVisionModels beats an exact modality entry, as isModelTextOnly does", () => { |
| 106 | + // `isModelTextOnly` matches the no-vision list and returns true before it ever |
| 107 | + // reads `modelInputModalities`, so the `gpt-oss` no-vision entry wins over an |
| 108 | + // exact `gpt-oss:120b` entry listing "image". Evidence that disagrees here is |
| 109 | + // worse than a wrong window: routing selects the candidate for image work and |
| 110 | + // execution then refuses it. |
| 111 | + const provider = { |
| 112 | + ...providerWithFamilyEntries(), |
| 113 | + noVisionModels: ["gpt-oss"], |
| 114 | + modelInputModalities: { "gpt-oss:120b": ["text", "image"] }, |
| 115 | + } as unknown as OcxProviderConfig; |
| 116 | + |
| 117 | + // Ground truth first: the resolver this evidence claims to describe says text-only. |
| 118 | + expect(isModelTextOnly(provider, "gpt-oss:120b")).toBe(true); |
| 119 | + |
| 120 | + const evidence = candidateCapabilityEvidence(configFor(provider), "custom", "gpt-oss:120b"); |
| 121 | + expect(evidence.image).toBe(false); |
| 122 | + }); |
| 123 | + |
| 124 | + test("a model outside noVisionModels keeps its declared image modality", () => { |
| 125 | + // The negative half: the no-vision check must not spread to models the list does |
| 126 | + // not cover, or the fix would trade a false positive for a false negative. |
| 127 | + const provider = { |
| 128 | + ...providerWithFamilyEntries(), |
| 129 | + models: ["gpt-oss:120b", "llava:13b"], |
| 130 | + noVisionModels: ["gpt-oss"], |
| 131 | + modelInputModalities: { "llava:13b": ["text", "image"] }, |
| 132 | + } as unknown as OcxProviderConfig; |
| 133 | + |
| 134 | + expect(isModelTextOnly(provider, "llava:13b")).toBe(false); |
| 135 | + expect(candidateCapabilityEvidence(configFor(provider), "custom", "llava:13b").image).toBe(true); |
| 136 | + }); |
| 137 | + |
| 138 | + test("a prototype-shaped model id resolves nothing", () => { |
| 139 | + // modelRecordValue uses hasOwnProperty; a bare lookup would return Object.prototype |
| 140 | + // members here and hand routing a function as evidence. |
| 141 | + for (const modelId of ["constructor", "toString", "valueOf", "hasOwnProperty"]) { |
| 142 | + const evidence = candidateCapabilityEvidence( |
| 143 | + configFor(providerWithFamilyEntries()), |
| 144 | + "custom", |
| 145 | + modelId, |
| 146 | + ); |
| 147 | + expect(evidence.contextWindow).toBe(8_000); |
| 148 | + expect(evidence.reasoningEfforts).toBeUndefined(); |
| 149 | + expect(evidence.image).toBeUndefined(); |
| 150 | + } |
| 151 | + }); |
| 152 | +}); |
0 commit comments