Skip to content

Commit 8326838

Browse files
authored
fix(devin): score family axes sparsely with own-key targets (#6324)
1 parent 57fc9cc commit 8326838

3 files changed

Lines changed: 110 additions & 9 deletions

File tree

‎src/adapters/devin/live-models.ts‎

Lines changed: 28 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,22 @@ function effortIndex(rung: string | undefined): number {
207207
return rung === undefined ? -1 : FAMILY_EFFORT_LADDER.indexOf(rung);
208208
}
209209

210+
function axisMismatchCount(
211+
axes: ModelCatalogEntry["familyAxes"],
212+
targets: Record<string, number>,
213+
baseline: number,
214+
): number {
215+
let mismatches = baseline;
216+
for (const [axis, value] of Object.entries(axes ?? {})) {
217+
if (isEffortAxis(axis)) continue;
218+
const target = Object.hasOwn(targets, axis) ? targets[axis]! : 0;
219+
if (target !== 0) {
220+
if (value.order === target) mismatches--;
221+
} else if (value.order !== 0) mismatches++;
222+
}
223+
return mismatches;
224+
}
225+
210226
/**
211227
* The member closest to `targets` on the non-effort axes, then the lowest rung
212228
* at or above `effort`, falling back to the highest rung below it. A missing
@@ -227,11 +243,15 @@ function closestMember(
227243
effort: string | undefined,
228244
): ModelCatalogEntry | undefined {
229245
const want = effortIndex(effort);
246+
// The baseline counts only `targets`, so scan it once: recomputing it per member
247+
// made a wide anchor (many exposed axes) quadratic in members x axes.
248+
let mismatchBaseline = 0;
249+
for (const order of Object.values(targets)) if (order !== 0) mismatchBaseline++;
230250
let best: ModelCatalogEntry | undefined;
231251
let bestScore: number[] | undefined;
232252
for (const member of members) {
233253
const axes = member.familyAxes ?? {};
234-
const mismatches = Object.entries(targets).filter(([axis, order]) => (axes[axis]?.order ?? 0) !== order).length;
254+
const mismatches = axisMismatchCount(axes, targets, mismatchBaseline);
235255
const have = effortIndex(devinFamilyEffortOf(member));
236256
const lowered = want > 0 && have < want ? 1 : 0;
237257
const distance = want < 0 ? 0
@@ -263,18 +283,18 @@ export function selectDevinFamilyMember(
263283
anchor?: ModelCatalogEntry,
264284
options: { includeDisabled?: boolean } = {},
265285
): ModelCatalogEntry | undefined {
266-
const axisNames = new Set(members.flatMap((m) => Object.keys(m.familyAxes ?? {})).filter((a) => !isEffortAxis(a)));
267286
const base = anchor
268287
?? members.find((m) => m.isFamilyDefault)
269-
?? closestMember(members, Object.fromEntries([...axisNames].map((a) => [a, 0])), "medium");
288+
?? closestMember(members, {}, "medium");
270289
if (!base) return undefined;
271-
const targets: Record<string, number> = {};
272-
for (const axis of axisNames) targets[axis] = base.familyAxes?.[axis]?.order ?? 0;
273-
if (request.fast && axisNames.has(FAST_AXIS)) targets[FAST_AXIS] = 1;
274-
if (request.longContext && axisNames.has(LONG_CONTEXT_AXIS)) targets[LONG_CONTEXT_AXIS] = 1;
290+
const targets = Object.fromEntries(Object.entries(base.familyAxes ?? {})
291+
.filter(([axis]) => !isEffortAxis(axis))
292+
.map(([axis, value]) => [axis, value.order]));
293+
if (request.fast) targets[FAST_AXIS] = 1;
294+
if (request.longContext) targets[LONG_CONTEXT_AXIS] = 1;
275295
// `none` means no reasoning. Families like Claude Sonnet 4.6 express that as
276296
// Thinking off rather than as an effort rung.
277-
if (request.effort === "none" && axisNames.has(THINKING_AXIS)) targets[THINKING_AXIS] = 0;
297+
if (request.effort === "none") targets[THINKING_AXIS] = 0;
278298
const enabled = options.includeDisabled ? members : members.filter((m) => !m.disabled);
279299
return closestMember(enabled.length > 0 ? enabled : members, targets, request.effort ?? devinFamilyEffortOf(base));
280300
}

‎structure/providers-and-adapters.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,8 @@ rewrite rules and the routed-id settlement.
192192

193193
Devin pairs a late signature only with immediately preceding unsigned thinking in one assistant message; a call between them breaks the pair. While a signed attempt is held for an optional unsigned retry, a timer emits plain heartbeats even when the upstream stalls. The held queue is capped at 1,024 events or approximately 1 MiB of UTF-16 reasoning, signature, and signature-type payload; signature types are individually capped at 4 KiB before UTF-8 decoding, and crossing either queue cap releases the events and disables that retry. Held usage frames merge per cumulative field, and the refused attempt's usage is added to the retry. A signed `invalid_argument` refusal is offered the unsigned retry before the history-overflow classifier sees any final refusal. If the send budget withholds that retry, the original refusal reaches the classifier and the recovery is recorded as withheld.
194194

195+
Devin family selection in `src/adapters/devin/live-models.ts` counts the anchor's nonzero targets once and visits each candidate's own axes. Missing axes have order zero; extra nonzero axes, including prototype names such as `toString`, count as mismatches. Effort precedence and family confinement remain unchanged; `tests/providers/devin-family-resolution.test.ts` covers the sparse-scoring contract.
196+
195197
Inline document admission shares one encoding predicate between its scanner and parser in
196198
`src/responses/inline-document.ts`: malformed base64 quantum/padding lengths are refused,
197199
and valid padded or unpadded payloads pass unchanged without a decoding allocation.

‎tests/providers/devin-family-resolution.test.ts‎

Lines changed: 80 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
*/
1111
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
1212
import { resolveWireModelUidForTests } from "../../src/adapters/devin";
13-
import { fetchDevinUsableModels } from "../../src/adapters/devin/live-models";
13+
import { fetchDevinUsableModels, selectDevinFamilyMember } from "../../src/adapters/devin/live-models";
1414
import { parseCatalogBuffer, setCachedCatalogForTests, type CacheEntry } from "../../src/adapters/devin/cloud-direct/catalog";
1515
import { encodeMessage, encodeString, encodeVarintField } from "../../src/adapters/devin/cloud-direct/wire";
1616

@@ -259,6 +259,85 @@ describe("family-based wire model resolution", () => {
259259
expect(await resolve(catalogOf(rows), "devin-test-think", "none")).toBe("think-off");
260260
expect(await resolve(catalogOf(rows), "devin-test-think", "high")).toBe("think-high");
261261
});
262+
263+
test("scores distinct family axes without a members-by-axes cross-product", () => {
264+
// The anchor row carries the wide axis set: the mismatch baseline scans `targets`,
265+
// which the anchor fills. With per-member baseline recomputation this case is
266+
// members x axes (~144M iterations); hoisted, it is anchor axes + member axes.
267+
const wideAxes = Object.fromEntries(
268+
Array.from({ length: 12_000 }, (_, index) => [`Anchor ${index}`, { order: 1 }]),
269+
);
270+
const members = [
271+
{
272+
modelUid: "wide-anchor",
273+
displayName: "wide-anchor",
274+
familyUid: "wide",
275+
familyAxes: wideAxes,
276+
isFamilyDefault: true,
277+
},
278+
...Array.from({ length: 12_000 }, (_, index) => ({
279+
modelUid: `wide-${index}`,
280+
displayName: `wide-${index}`,
281+
familyUid: "wide",
282+
familyAxes: { [`Axis ${index}`]: { order: 1 } },
283+
})),
284+
];
285+
const started = performance.now();
286+
expect(selectDevinFamilyMember(members, {})?.modelUid).toBe("wide-anchor");
287+
// Wall-clock bound, generous on purpose: the hoist makes this ~36k axis touches
288+
// (ms even on a contended runner), while the per-member rescan needs ~144M — the
289+
// bound only has to distinguish those two orders, not measure fast hardware.
290+
expect(performance.now() - started).toBeLessThan(5_000);
291+
});
292+
293+
test.each(["toString", "constructor", "__proto__"])("counts an extra own %s axis instead of inheriting a target", axis => {
294+
const member = (modelUid: string, familyAxes: Record<string, { order: number }>) => ({
295+
modelUid, displayName: modelUid, familyUid: "own-axes", familyAxes,
296+
});
297+
const anchor = member("anchor", { A: { order: 1 } });
298+
const exact = member("exact", { A: { order: 1 } });
299+
const extra = member("extra", Object.fromEntries([["A", { order: 1 }], [axis, { order: 1 }]]));
300+
expect(selectDevinFamilyMember([extra, exact], {}, anchor)?.modelUid).toBe("exact");
301+
const namedAnchor = member("named-anchor", Object.fromEntries([[axis, { order: 2 }]]));
302+
const namedExact = member("named-exact", Object.fromEntries([[axis, { order: 2 }]]));
303+
expect(selectDevinFamilyMember([member("missing", {}), namedExact], {}, namedAnchor)?.modelUid).toBe("named-exact");
304+
});
305+
306+
test("ranks an exact axis match ahead of missing and extra axes", () => {
307+
const anchor = {
308+
modelUid: "axis-anchor",
309+
displayName: "axis-anchor",
310+
familyUid: "wide",
311+
familyAxes: { A: { order: 1 }, B: { order: 2 } },
312+
isFamilyDefault: true,
313+
};
314+
const exact = {
315+
modelUid: "axis-exact",
316+
displayName: "axis-exact",
317+
familyUid: "wide",
318+
familyAxes: { A: { order: 1 }, B: { order: 2 } },
319+
};
320+
const missing = {
321+
modelUid: "axis-missing",
322+
displayName: "axis-missing",
323+
familyUid: "wide",
324+
familyAxes: { A: { order: 1 } },
325+
};
326+
const extra = {
327+
modelUid: "axis-extra",
328+
displayName: "axis-extra",
329+
familyUid: "wide",
330+
familyAxes: { A: { order: 1 }, B: { order: 2 }, C: { order: 1 }, D: { order: 3 } },
331+
};
332+
// The anchor itself always wins while listed: the ranking claim is about the
333+
// members it cannot see, so exclude it and pass it explicitly.
334+
const members = [missing, extra, exact];
335+
expect(selectDevinFamilyMember(members, {}, anchor)?.modelUid).toBe("axis-exact");
336+
// Without the exact row, a member missing one target axis scores baseline - 1,
337+
// while a member matching every target but exposing two extra axes scores
338+
// 0 + 2 — unequal mismatch counts, so the ordering is proven rather than a tie.
339+
expect(selectDevinFamilyMember([missing, extra], {}, anchor)?.modelUid).toBe("axis-missing");
340+
});
262341
});
263342

264343
describe("suffix fallback without family metadata never crosses families", () => {

0 commit comments

Comments
 (0)