|
| 1 | +/** |
| 2 | + * Devin live-discovery collapse and advertised-catalog propagation for |
| 3 | + * ClientModelConfig field #5 (supportsImages). |
| 4 | + * |
| 5 | + * Catalogs are hand-encoded protobuf run through the real parser |
| 6 | + * (parseCatalogBuffer) and installed through setCachedCatalogForTests, so the |
| 7 | + * tests cover the collapse in fetchDevinUsableModels and the Devin branch of |
| 8 | + * fetchProviderModels without touching the network. KEY is unique to this |
| 9 | + * file and HOST is the stripped default host: getCachedCatalog hits only on |
| 10 | + * an exact (apiKey, host) match with a fresh fetchedAt. |
| 11 | + */ |
| 12 | +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; |
| 13 | +import * as oauth from "../../src/oauth"; |
| 14 | +import { fetchDevinUsableModels } from "../../src/adapters/devin/live-models"; |
| 15 | +import { parseCatalogBuffer, setCachedCatalogForTests } from "../../src/adapters/devin/cloud-direct/catalog"; |
| 16 | +import { encodeMessage, encodeString, encodeVarintField } from "../../src/adapters/devin/cloud-direct/wire"; |
| 17 | +import { fetchProviderModels } from "../../src/codex/catalog/provider-fetch"; |
| 18 | +import { clearModelCache, providerCacheGenerations } from "../../src/codex/model-cache"; |
| 19 | +import type { OcxProviderConfig } from "../../src/types"; |
| 20 | + |
| 21 | +const HOST = "https://server.codeium.com"; |
| 22 | +const KEY = "devin-live-models-test-key"; |
| 23 | + |
| 24 | +/** One ClientModelConfig body; field #5 stays absent unless opts asserts it. */ |
| 25 | +function catalogEntry( |
| 26 | + uid: string, |
| 27 | + opts: { disabled?: boolean; supportsImages?: boolean; contextWindow?: number } = {}, |
| 28 | +): Buffer { |
| 29 | + return Buffer.concat([ |
| 30 | + encodeString(1, uid), |
| 31 | + ...(opts.disabled === true ? [encodeVarintField(4, 1)] : []), |
| 32 | + // encodeVarintField(5, 0) is a measured text-only vote — real bytes, not |
| 33 | + // an omission — while leaving field #5 out keeps the row unknown. |
| 34 | + ...(opts.supportsImages !== undefined ? [encodeVarintField(5, opts.supportsImages ? 1 : 0)] : []), |
| 35 | + ...(opts.contextWindow !== undefined ? [encodeVarintField(18, opts.contextWindow)] : []), |
| 36 | + encodeString(22, uid), |
| 37 | + ]); |
| 38 | +} |
| 39 | + |
| 40 | +function seedCatalog(...entries: Buffer[]): void { |
| 41 | + setCachedCatalogForTests(parseCatalogBuffer( |
| 42 | + Buffer.concat(entries.map((entry) => encodeMessage(1, entry))), |
| 43 | + KEY, |
| 44 | + HOST, |
| 45 | + )); |
| 46 | +} |
| 47 | + |
| 48 | +beforeEach(() => { |
| 49 | + setCachedCatalogForTests(null); |
| 50 | + clearModelCache("devin-test"); |
| 51 | + providerCacheGenerations.delete("devin-test"); |
| 52 | +}); |
| 53 | +afterEach(() => { |
| 54 | + setCachedCatalogForTests(null); |
| 55 | + clearModelCache("devin-test"); |
| 56 | + providerCacheGenerations.delete("devin-test"); |
| 57 | +}); |
| 58 | + |
| 59 | +describe("devin live model discovery", () => { |
| 60 | + test("collapses per-variant supportsImages votes into per-base input modalities", async () => { |
| 61 | + seedCatalog( |
| 62 | + // Unanimous measured rows advertise. |
| 63 | + catalogEntry("vision-model", { supportsImages: true, contextWindow: 262_000 }), |
| 64 | + catalogEntry("vision-model-high", { supportsImages: true, contextWindow: 1_000_000 }), |
| 65 | + catalogEntry("text-model-low", { supportsImages: false }), |
| 66 | + catalogEntry("text-model-high", { supportsImages: false }), |
| 67 | + // An unsuffixed row that never asserted field #5 abstains instead of |
| 68 | + // poisoning a measured image base. |
| 69 | + catalogEntry("abstain-model"), |
| 70 | + catalogEntry("abstain-model-high", { supportsImages: true }), |
| 71 | + // Measured disagreement stays unadvertised — a single false is not |
| 72 | + // outvoted by its siblings. |
| 73 | + catalogEntry("split-model", { supportsImages: true }), |
| 74 | + catalogEntry("split-model-low", { supportsImages: true }), |
| 75 | + catalogEntry("split-model-high", { supportsImages: false }), |
| 76 | + catalogEntry("mixed-model-low", { supportsImages: true }), |
| 77 | + catalogEntry("mixed-model-high", { supportsImages: false }), |
| 78 | + // Zero measured rows advertise nothing. |
| 79 | + catalogEntry("mystery-model"), |
| 80 | + catalogEntry("mystery-model-high"), |
| 81 | + // Disabled and MODEL_* rows are skipped before they can vote: if the |
| 82 | + // disabled true voted, text-off-model would read as disagreement. |
| 83 | + catalogEntry("text-off-model", { supportsImages: false }), |
| 84 | + catalogEntry("text-off-model-high", { disabled: true, supportsImages: true }), |
| 85 | + catalogEntry("ghost-model-high", { disabled: true, supportsImages: true }), |
| 86 | + catalogEntry("MODEL_INTERNAL_VISION", { supportsImages: true }), |
| 87 | + ); |
| 88 | + const result = await fetchDevinUsableModels({ apiKey: KEY, baseUrl: HOST }); |
| 89 | + if (!result.ok) throw new Error(`expected ok, got ${result.error}`); |
| 90 | + expect(result.models).toEqual([ |
| 91 | + "abstain-model", |
| 92 | + "mixed-model", |
| 93 | + "mystery-model", |
| 94 | + "split-model", |
| 95 | + "text-model", |
| 96 | + "text-off-model", |
| 97 | + "vision-model", |
| 98 | + ]); |
| 99 | + expect(result.inputModalities).toEqual({ |
| 100 | + "vision-model": ["text", "image"], |
| 101 | + "text-model": ["text"], |
| 102 | + "abstain-model": ["text", "image"], |
| 103 | + "text-off-model": ["text"], |
| 104 | + }); |
| 105 | + // The collapse adds a field; the existing projections are unchanged. |
| 106 | + expect(result.contextWindows["vision-model"]).toBe(262_000); |
| 107 | + expect(result.efforts["text-model"]).toEqual(["low", "high"]); |
| 108 | + }); |
| 109 | + |
| 110 | + test("a catalog with no measured rows still carries an empty record", async () => { |
| 111 | + seedCatalog(catalogEntry("plain-model"), catalogEntry("plain-model-high")); |
| 112 | + const result = await fetchDevinUsableModels({ apiKey: KEY, baseUrl: HOST }); |
| 113 | + if (!result.ok) throw new Error(`expected ok, got ${result.error}`); |
| 114 | + expect(result.inputModalities).toEqual({}); |
| 115 | + }); |
| 116 | +}); |
| 117 | + |
| 118 | +describe("devin advertised catalog input modalities", () => { |
| 119 | + // Devin is an oauth provider, so discovery resolves its bearer through |
| 120 | + // resolveModelsAuthToken; the tests lend it a token rather than an account |
| 121 | + // store (the same seam the Copilot oauth cases use). |
| 122 | + let authSpy: ReturnType<typeof spyOn> | undefined; |
| 123 | + beforeEach(() => { |
| 124 | + authSpy = spyOn(oauth, "resolveModelsAuthToken").mockResolvedValue(KEY); |
| 125 | + }); |
| 126 | + afterEach(() => { |
| 127 | + authSpy?.mockRestore(); |
| 128 | + authSpy = undefined; |
| 129 | + }); |
| 130 | + |
| 131 | + const devinProvider = (extra: Partial<OcxProviderConfig> = {}): OcxProviderConfig => ({ |
| 132 | + adapter: "devin", |
| 133 | + baseUrl: HOST, |
| 134 | + apiKey: KEY, |
| 135 | + authMode: "oauth", |
| 136 | + liveModels: true, |
| 137 | + ...extra, |
| 138 | + } as OcxProviderConfig); |
| 139 | + |
| 140 | + test("a measured image base advertises text and image", async () => { |
| 141 | + seedCatalog(catalogEntry("img-model", { supportsImages: true })); |
| 142 | + const models = await fetchProviderModels("devin-test", devinProvider(), 60_000); |
| 143 | + expect(models.map((model) => model.id)).toEqual(["img-model"]); |
| 144 | + expect(models[0]?.inputModalities).toEqual(["text", "image"]); |
| 145 | + }); |
| 146 | + |
| 147 | + test("an exact modelCapabilities declaration overwrites the live value", async () => { |
| 148 | + seedCatalog(catalogEntry("img-model", { supportsImages: true })); |
| 149 | + const models = await fetchProviderModels("devin-test", devinProvider({ |
| 150 | + modelCapabilities: { "img-model": { inputModalities: ["audio"] } }, |
| 151 | + }), 60_000); |
| 152 | + expect(models[0]?.inputModalities).toEqual(["audio"]); |
| 153 | + }); |
| 154 | + |
| 155 | + test("an exact text-only declaration still takes the sidecar path", async () => { |
| 156 | + // A text-only modelCapabilities entry makes the row a vision-sidecar |
| 157 | + // consumer (src/vision/eligibility.ts): the declaration governs runtime |
| 158 | + // eligibility while the catalog keeps attachments unblocked. |
| 159 | + seedCatalog(catalogEntry("img-model", { supportsImages: true })); |
| 160 | + const models = await fetchProviderModels("devin-test", devinProvider({ |
| 161 | + modelCapabilities: { "img-model": { inputModalities: ["text"] } }, |
| 162 | + }), 60_000); |
| 163 | + expect(models[0]?.inputModalities).toEqual(["text", "image"]); |
| 164 | + }); |
| 165 | + |
| 166 | + test("a noVisionModels entry upgrades a live text-only row through the sidecar", async () => { |
| 167 | + seedCatalog(catalogEntry("side-model", { supportsImages: false })); |
| 168 | + const models = await fetchProviderModels("devin-test", devinProvider({ |
| 169 | + noVisionModels: ["side-model"], |
| 170 | + }), 60_000); |
| 171 | + expect(models[0]?.inputModalities).toEqual(["text", "image"]); |
| 172 | + }); |
| 173 | + |
| 174 | + test("a measured text-only base is not upgraded without a sidecar consumer", async () => { |
| 175 | + seedCatalog(catalogEntry("plain-model", { supportsImages: false })); |
| 176 | + const models = await fetchProviderModels("devin-test", devinProvider(), 60_000); |
| 177 | + expect(models.map((model) => model.id)).toEqual(["plain-model"]); |
| 178 | + expect(models[0]?.inputModalities).toEqual(["text"]); |
| 179 | + }); |
| 180 | +}); |
0 commit comments