Skip to content

Commit 5d95dbf

Browse files
committed
feat(devin): propagate catalog supportsImages to the advertised catalog
Round 1 made the Devin catalog parser preserve ClientModelConfig field #5 as a tri-state on ModelCatalogEntry; the flag stopped at the cache. Carry it through: fetchDevinUsableModels now votes per base across the rows that collapse into it and returns inputModalities, and the Devin branch of provider-fetch spreads that value before catalogHintsFromProviderConfig, so exact modelCapabilities declarations, the legacy modelInputModalities record and the vision-sidecar rewrite keep precedence and the live value survives only when none of them applies. Collapse policy, pinned with the round-1 #1796 precedent: rows that never asserted field #5 abstain, so one unsuffixed unknown row cannot poison a base whose effort variants were measured image-capable; unanimous measured rows advertise ["text"] or ["text","image"]; measured disagreement advertises nothing, because a single measured false is not outvoted by its siblings. The accepted mismatch is documented in code: resolveWireModelUid prefers the plain UID when the catalog lists it, so a variant-measured image base can route a no-effort request to an unasserted plain row. Tests: a new devin-live-models suite seeds the cache through the real parser via a setCachedCatalogForTests seam and covers the collapse matrix (disabled and MODEL_* rows proven non-voters by behavior) plus five fetchProviderModels advertised-catalog cases locking the hint precedence, including the existing sidecar path for exact text-only declarations. Layout registries name the new file; structure/catalog.md and structure/adapters/registry.md record the contract.
1 parent f7e4af0 commit 5d95dbf

8 files changed

Lines changed: 241 additions & 3 deletions

File tree

‎scripts/test-layout/layout.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -648,6 +648,7 @@
648648
"devin-adapter.test.ts": "providers",
649649
"devin-cli-authmode-migration.test.ts": "providers",
650650
"devin-effort-ladder.test.ts": "providers",
651+
"devin-live-models.test.ts": "providers",
651652
"devin-login.test.ts": "providers",
652653
"devin-provider-merge-migration.test.ts": "providers",
653654
"devin-hardening.test.ts": "providers",

‎src/adapters/devin/cloud-direct/catalog.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -297,6 +297,19 @@ export function clearCachedCatalog(): void {
297297
cacheEpoch++;
298298
}
299299

300+
/**
301+
* Test seam: install a catalog as the live cache entry. Mirrors
302+
* clearCachedCatalog's invalidation — the in-flight slot is dropped and the
303+
* epoch bumped — so a fetch racing the seed cannot overwrite it, and a null
304+
* entry resets the cache between tests.
305+
*/
306+
export function setCachedCatalogForTests(entry: CacheEntry | null): void {
307+
cached = entry;
308+
inFlight = null;
309+
inFlightKey = null;
310+
cacheEpoch++;
311+
}
312+
300313
/**
301314
* Tier-disabled error — thrown by the chat pre-flight when the catalog lists
302315
* a model as `disabled: true` for this account. The message names the model

‎src/adapters/devin/live-models.ts‎

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,13 @@ export const DEVIN_MODEL_EFFORTS: Record<string, string[]> = {
139139
export const DEVIN_DEFAULT_EFFORTS = ["low", "medium", "high", "xhigh", "max"];
140140

141141
export type DevinUsableModelsResult =
142-
| { ok: true; models: string[]; contextWindows: Record<string, number>; efforts: Record<string, string[]> }
142+
| {
143+
ok: true;
144+
models: string[];
145+
contextWindows: Record<string, number>;
146+
efforts: Record<string, string[]>;
147+
inputModalities: Record<string, string[]>;
148+
}
143149
| { ok: false; error: "auth" | "http" | "empty" | "unknown"; detail?: string };
144150

145151
/**
@@ -160,6 +166,8 @@ export async function fetchDevinUsableModels(opts: {
160166
const contextWindows: Record<string, number> = {};
161167
// Effort rungs per base, recovered from the suffixes the collapse strips.
162168
const rungs = new Map<string, Set<string>>();
169+
// supportsImages votes per base; only rows that asserted field #5 vote.
170+
const imageVotes = new Map<string, { sawTrue: boolean; sawFalse: boolean }>();
163171
for (const entry of catalog.byUid.values()) {
164172
if (entry.disabled) continue;
165173
// Skip internal enum constants (e.g. MODEL_GPT_5_2_LOW, MODEL_PRIVATE_*).
@@ -183,6 +191,15 @@ export async function fetchDevinUsableModels(opts: {
183191
const seen = contextWindows[base];
184192
contextWindows[base] = seen === undefined ? entry.contextWindow : Math.min(seen, entry.contextWindow);
185193
}
194+
if (entry.supportsImages !== undefined) {
195+
let votes = imageVotes.get(base);
196+
if (!votes) {
197+
votes = { sawTrue: false, sawFalse: false };
198+
imageVotes.set(base, votes);
199+
}
200+
if (entry.supportsImages) votes.sawTrue = true;
201+
else votes.sawFalse = true;
202+
}
186203
}
187204
if (bases.size === 0) return { ok: false, error: "empty" };
188205
const efforts: Record<string, string[]> = {};
@@ -191,7 +208,21 @@ export async function fetchDevinUsableModels(opts: {
191208
// would draw a picker whose only option is the value already in effect.
192209
if (set.size > 1) efforts[base] = sortDevinRungs(set);
193210
}
194-
return { ok: true, models: [...bases].sort(), contextWindows, efforts };
211+
// supportsImages arrives tri-state per catalog row, so the collapse votes:
212+
// a row that never asserted field #5 abstains, which keeps an unsuffixed
213+
// unknown row from poisoning a base whose effort variants were measured
214+
// image-capable. Unanimous measured rows advertise; measured disagreement
215+
// advertises nothing, because a single measured false is not outvoted by
216+
// its siblings. One accepted mismatch: resolveWireModelUid prefers the
217+
// plain UID when the catalog lists it, so a base advertised
218+
// ["text","image"] on variant evidence can still route a no-effort request
219+
// to a plain row that never asserted the field.
220+
const inputModalities: Record<string, string[]> = {};
221+
for (const [base, votes] of imageVotes) {
222+
if (votes.sawTrue && votes.sawFalse) continue;
223+
inputModalities[base] = votes.sawTrue ? ["text", "image"] : ["text"];
224+
}
225+
return { ok: true, models: [...bases].sort(), contextWindows, efforts, inputModalities };
195226
} catch (error) {
196227
const message = error instanceof Error ? error.message : String(error);
197228
if (/unauth|401|invalid token|login/i.test(message)) return { ok: false, error: "auth", detail: message };

‎src/codex/catalog/provider-fetch.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1745,6 +1745,12 @@ async function fetchProviderModelsWithAuth(
17451745
// away, and every client that keys an effort control off this field —
17461746
// the Pi-shaped exports — renders no control at all.
17471747
...(liveResult.efforts[id]?.length ? { reasoningEfforts: liveResult.efforts[id] } : {}),
1748+
// The account catalog's per-base supportsImages vote collapses to one
1749+
// modalities value. It spreads before the hints so exact
1750+
// modelCapabilities declarations, the legacy modelInputModalities
1751+
// record and the vision-sidecar rewrite keep winning — the live
1752+
// value survives only when none of them applies.
1753+
...(liveResult.inputModalities[id]?.length ? { inputModalities: liveResult.inputModalities[id] } : {}),
17481754
...catalogHintsFromProviderConfig(name, prov, id, contextCap, metadataModelIdCaseFold, captured.effectiveAlias),
17491755
} as CatalogModel;
17501756
});

‎structure/adapters/registry.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,9 @@ Some adapters share another adapter's routed-tool semantics while retaining inde
5656
`ClientModelConfig` field #4 as the per-account disabled gate, field #18 as the per-account
5757
context window, and field #5 as an optional `supportsImages` tri-state — a present value
5858
asserts image support or its absence, while an omitted field stays unknown (the #1796
59-
precedent).
59+
precedent). `src/adapters/devin/live-models.ts` collapses that tri-state across each base
60+
model's effort variants: unmeasured rows abstain, unanimous measured rows advertise
61+
`["text"]` or `["text", "image"]`, and measured disagreement stays unadvertised.
6062

6163
The registry records those relationships with `contractParent`. A parent relationship does **not** mean the registry recursively constructs a parent adapter and injects it into the child. Azure and MiMo keep owning their existing internal composition. This avoids making production constructors depend on test/conformance needs and keeps this authority refactor behavior-neutral.
6264

‎structure/catalog.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,10 @@ Provider live-model lists are cached with a configured TTL (`src/codex/model-cac
9595
deleting, or editing a provider's shape clears that per-provider cache; a disabled-only change
9696
deliberately does not, because a disabled provider is already excluded from the catalog gather
9797
instead. Codex's own `models_cache.json` is a different cache, invalidated by catalog refresh.
98+
A Devin live row spreads its measured `contextWindow`, `reasoningEfforts` and
99+
`inputModalities` before `catalogHintsFromProviderConfig`, so exact `modelCapabilities`
100+
declarations, the legacy `modelInputModalities` record and the vision-sidecar rewrite keep
101+
precedence and a live value survives only when none of them applies.
98102

99103
For `liveModels: false`, a static provider publishes the ordered union of `models` and
100104
`retainModels`. When `models` is absent or empty, its configured `defaultModel` seeds that

‎tests/fixtures/test-layout-expected.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -482,6 +482,7 @@
482482
"devin-effort-ladder.test.ts": "providers",
483483
"devin-hardening.test.ts": "providers",
484484
"devin-image-passthrough.test.ts": "providers",
485+
"devin-live-models.test.ts": "providers",
485486
"devin-prompt-cache.test.ts": "providers",
486487
"devin-stream-deadline.test.ts": "providers",
487488
"digitalocean-scaleway-provider.test.ts": "providers",
Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
/**
2+
* Devin live-discovery collapse and advertised-catalog propagation for
3+
* ClientModelConfig field #5 (supportsImages).
4+
*
5+
* Catalogs are hand-encoded protobuf run through the real parser
6+
* (parseCatalogBuffer) and installed through setCachedCatalogForTests, so the
7+
* tests cover the collapse in fetchDevinUsableModels and the Devin branch of
8+
* fetchProviderModels without touching the network. KEY is unique to this
9+
* file and HOST is the stripped default host: getCachedCatalog hits only on
10+
* an exact (apiKey, host) match with a fresh fetchedAt.
11+
*/
12+
import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test";
13+
import * as oauth from "../../src/oauth";
14+
import { fetchDevinUsableModels } from "../../src/adapters/devin/live-models";
15+
import { parseCatalogBuffer, setCachedCatalogForTests } from "../../src/adapters/devin/cloud-direct/catalog";
16+
import { encodeMessage, encodeString, encodeVarintField } from "../../src/adapters/devin/cloud-direct/wire";
17+
import { fetchProviderModels } from "../../src/codex/catalog/provider-fetch";
18+
import { clearModelCache, providerCacheGenerations } from "../../src/codex/model-cache";
19+
import type { OcxProviderConfig } from "../../src/types";
20+
21+
const HOST = "https://server.codeium.com";
22+
const KEY = "devin-live-models-test-key";
23+
24+
/** One ClientModelConfig body; field #5 stays absent unless opts asserts it. */
25+
function catalogEntry(
26+
uid: string,
27+
opts: { disabled?: boolean; supportsImages?: boolean; contextWindow?: number } = {},
28+
): Buffer {
29+
return Buffer.concat([
30+
encodeString(1, uid),
31+
...(opts.disabled === true ? [encodeVarintField(4, 1)] : []),
32+
// encodeVarintField(5, 0) is a measured text-only vote — real bytes, not
33+
// an omission — while leaving field #5 out keeps the row unknown.
34+
...(opts.supportsImages !== undefined ? [encodeVarintField(5, opts.supportsImages ? 1 : 0)] : []),
35+
...(opts.contextWindow !== undefined ? [encodeVarintField(18, opts.contextWindow)] : []),
36+
encodeString(22, uid),
37+
]);
38+
}
39+
40+
function seedCatalog(...entries: Buffer[]): void {
41+
setCachedCatalogForTests(parseCatalogBuffer(
42+
Buffer.concat(entries.map((entry) => encodeMessage(1, entry))),
43+
KEY,
44+
HOST,
45+
));
46+
}
47+
48+
beforeEach(() => {
49+
setCachedCatalogForTests(null);
50+
clearModelCache("devin-test");
51+
providerCacheGenerations.delete("devin-test");
52+
});
53+
afterEach(() => {
54+
setCachedCatalogForTests(null);
55+
clearModelCache("devin-test");
56+
providerCacheGenerations.delete("devin-test");
57+
});
58+
59+
describe("devin live model discovery", () => {
60+
test("collapses per-variant supportsImages votes into per-base input modalities", async () => {
61+
seedCatalog(
62+
// Unanimous measured rows advertise.
63+
catalogEntry("vision-model", { supportsImages: true, contextWindow: 262_000 }),
64+
catalogEntry("vision-model-high", { supportsImages: true, contextWindow: 1_000_000 }),
65+
catalogEntry("text-model-low", { supportsImages: false }),
66+
catalogEntry("text-model-high", { supportsImages: false }),
67+
// An unsuffixed row that never asserted field #5 abstains instead of
68+
// poisoning a measured image base.
69+
catalogEntry("abstain-model"),
70+
catalogEntry("abstain-model-high", { supportsImages: true }),
71+
// Measured disagreement stays unadvertised — a single false is not
72+
// outvoted by its siblings.
73+
catalogEntry("split-model", { supportsImages: true }),
74+
catalogEntry("split-model-low", { supportsImages: true }),
75+
catalogEntry("split-model-high", { supportsImages: false }),
76+
catalogEntry("mixed-model-low", { supportsImages: true }),
77+
catalogEntry("mixed-model-high", { supportsImages: false }),
78+
// Zero measured rows advertise nothing.
79+
catalogEntry("mystery-model"),
80+
catalogEntry("mystery-model-high"),
81+
// Disabled and MODEL_* rows are skipped before they can vote: if the
82+
// disabled true voted, text-off-model would read as disagreement.
83+
catalogEntry("text-off-model", { supportsImages: false }),
84+
catalogEntry("text-off-model-high", { disabled: true, supportsImages: true }),
85+
catalogEntry("ghost-model-high", { disabled: true, supportsImages: true }),
86+
catalogEntry("MODEL_INTERNAL_VISION", { supportsImages: true }),
87+
);
88+
const result = await fetchDevinUsableModels({ apiKey: KEY, baseUrl: HOST });
89+
if (!result.ok) throw new Error(`expected ok, got ${result.error}`);
90+
expect(result.models).toEqual([
91+
"abstain-model",
92+
"mixed-model",
93+
"mystery-model",
94+
"split-model",
95+
"text-model",
96+
"text-off-model",
97+
"vision-model",
98+
]);
99+
expect(result.inputModalities).toEqual({
100+
"vision-model": ["text", "image"],
101+
"text-model": ["text"],
102+
"abstain-model": ["text", "image"],
103+
"text-off-model": ["text"],
104+
});
105+
// The collapse adds a field; the existing projections are unchanged.
106+
expect(result.contextWindows["vision-model"]).toBe(262_000);
107+
expect(result.efforts["text-model"]).toEqual(["low", "high"]);
108+
});
109+
110+
test("a catalog with no measured rows still carries an empty record", async () => {
111+
seedCatalog(catalogEntry("plain-model"), catalogEntry("plain-model-high"));
112+
const result = await fetchDevinUsableModels({ apiKey: KEY, baseUrl: HOST });
113+
if (!result.ok) throw new Error(`expected ok, got ${result.error}`);
114+
expect(result.inputModalities).toEqual({});
115+
});
116+
});
117+
118+
describe("devin advertised catalog input modalities", () => {
119+
// Devin is an oauth provider, so discovery resolves its bearer through
120+
// resolveModelsAuthToken; the tests lend it a token rather than an account
121+
// store (the same seam the Copilot oauth cases use).
122+
let authSpy: ReturnType<typeof spyOn> | undefined;
123+
beforeEach(() => {
124+
authSpy = spyOn(oauth, "resolveModelsAuthToken").mockResolvedValue(KEY);
125+
});
126+
afterEach(() => {
127+
authSpy?.mockRestore();
128+
authSpy = undefined;
129+
});
130+
131+
const devinProvider = (extra: Partial<OcxProviderConfig> = {}): OcxProviderConfig => ({
132+
adapter: "devin",
133+
baseUrl: HOST,
134+
apiKey: KEY,
135+
authMode: "oauth",
136+
liveModels: true,
137+
...extra,
138+
} as OcxProviderConfig);
139+
140+
test("a measured image base advertises text and image", async () => {
141+
seedCatalog(catalogEntry("img-model", { supportsImages: true }));
142+
const models = await fetchProviderModels("devin-test", devinProvider(), 60_000);
143+
expect(models.map((model) => model.id)).toEqual(["img-model"]);
144+
expect(models[0]?.inputModalities).toEqual(["text", "image"]);
145+
});
146+
147+
test("an exact modelCapabilities declaration overwrites the live value", async () => {
148+
seedCatalog(catalogEntry("img-model", { supportsImages: true }));
149+
const models = await fetchProviderModels("devin-test", devinProvider({
150+
modelCapabilities: { "img-model": { inputModalities: ["audio"] } },
151+
}), 60_000);
152+
expect(models[0]?.inputModalities).toEqual(["audio"]);
153+
});
154+
155+
test("an exact text-only declaration still takes the sidecar path", async () => {
156+
// A text-only modelCapabilities entry makes the row a vision-sidecar
157+
// consumer (src/vision/eligibility.ts): the declaration governs runtime
158+
// eligibility while the catalog keeps attachments unblocked.
159+
seedCatalog(catalogEntry("img-model", { supportsImages: true }));
160+
const models = await fetchProviderModels("devin-test", devinProvider({
161+
modelCapabilities: { "img-model": { inputModalities: ["text"] } },
162+
}), 60_000);
163+
expect(models[0]?.inputModalities).toEqual(["text", "image"]);
164+
});
165+
166+
test("a noVisionModels entry upgrades a live text-only row through the sidecar", async () => {
167+
seedCatalog(catalogEntry("side-model", { supportsImages: false }));
168+
const models = await fetchProviderModels("devin-test", devinProvider({
169+
noVisionModels: ["side-model"],
170+
}), 60_000);
171+
expect(models[0]?.inputModalities).toEqual(["text", "image"]);
172+
});
173+
174+
test("a measured text-only base is not upgraded without a sidecar consumer", async () => {
175+
seedCatalog(catalogEntry("plain-model", { supportsImages: false }));
176+
const models = await fetchProviderModels("devin-test", devinProvider(), 60_000);
177+
expect(models.map((model) => model.id)).toEqual(["plain-model"]);
178+
expect(models[0]?.inputModalities).toEqual(["text"]);
179+
});
180+
});

0 commit comments

Comments
 (0)