|
10 | 10 | */ |
11 | 11 | import { afterEach, beforeEach, describe, expect, test } from "bun:test"; |
12 | 12 | import { resolveWireModelUidForTests } from "../../src/adapters/devin"; |
13 | | -import { fetchDevinUsableModels } from "../../src/adapters/devin/live-models"; |
| 13 | +import { fetchDevinUsableModels, selectDevinFamilyMember } from "../../src/adapters/devin/live-models"; |
14 | 14 | import { parseCatalogBuffer, setCachedCatalogForTests, type CacheEntry } from "../../src/adapters/devin/cloud-direct/catalog"; |
15 | 15 | import { encodeMessage, encodeString, encodeVarintField } from "../../src/adapters/devin/cloud-direct/wire"; |
16 | 16 |
|
@@ -259,6 +259,85 @@ describe("family-based wire model resolution", () => { |
259 | 259 | expect(await resolve(catalogOf(rows), "devin-test-think", "none")).toBe("think-off"); |
260 | 260 | expect(await resolve(catalogOf(rows), "devin-test-think", "high")).toBe("think-high"); |
261 | 261 | }); |
| 262 | + |
| 263 | + test("scores distinct family axes without a members-by-axes cross-product", () => { |
| 264 | + // The anchor row carries the wide axis set: the mismatch baseline scans `targets`, |
| 265 | + // which the anchor fills. With per-member baseline recomputation this case is |
| 266 | + // members x axes (~144M iterations); hoisted, it is anchor axes + member axes. |
| 267 | + const wideAxes = Object.fromEntries( |
| 268 | + Array.from({ length: 12_000 }, (_, index) => [`Anchor ${index}`, { order: 1 }]), |
| 269 | + ); |
| 270 | + const members = [ |
| 271 | + { |
| 272 | + modelUid: "wide-anchor", |
| 273 | + displayName: "wide-anchor", |
| 274 | + familyUid: "wide", |
| 275 | + familyAxes: wideAxes, |
| 276 | + isFamilyDefault: true, |
| 277 | + }, |
| 278 | + ...Array.from({ length: 12_000 }, (_, index) => ({ |
| 279 | + modelUid: `wide-${index}`, |
| 280 | + displayName: `wide-${index}`, |
| 281 | + familyUid: "wide", |
| 282 | + familyAxes: { [`Axis ${index}`]: { order: 1 } }, |
| 283 | + })), |
| 284 | + ]; |
| 285 | + const started = performance.now(); |
| 286 | + expect(selectDevinFamilyMember(members, {})?.modelUid).toBe("wide-anchor"); |
| 287 | + // Wall-clock bound, generous on purpose: the hoist makes this ~36k axis touches |
| 288 | + // (ms even on a contended runner), while the per-member rescan needs ~144M — the |
| 289 | + // bound only has to distinguish those two orders, not measure fast hardware. |
| 290 | + expect(performance.now() - started).toBeLessThan(5_000); |
| 291 | + }); |
| 292 | + |
| 293 | + test.each(["toString", "constructor", "__proto__"])("counts an extra own %s axis instead of inheriting a target", axis => { |
| 294 | + const member = (modelUid: string, familyAxes: Record<string, { order: number }>) => ({ |
| 295 | + modelUid, displayName: modelUid, familyUid: "own-axes", familyAxes, |
| 296 | + }); |
| 297 | + const anchor = member("anchor", { A: { order: 1 } }); |
| 298 | + const exact = member("exact", { A: { order: 1 } }); |
| 299 | + const extra = member("extra", Object.fromEntries([["A", { order: 1 }], [axis, { order: 1 }]])); |
| 300 | + expect(selectDevinFamilyMember([extra, exact], {}, anchor)?.modelUid).toBe("exact"); |
| 301 | + const namedAnchor = member("named-anchor", Object.fromEntries([[axis, { order: 2 }]])); |
| 302 | + const namedExact = member("named-exact", Object.fromEntries([[axis, { order: 2 }]])); |
| 303 | + expect(selectDevinFamilyMember([member("missing", {}), namedExact], {}, namedAnchor)?.modelUid).toBe("named-exact"); |
| 304 | + }); |
| 305 | + |
| 306 | + test("ranks an exact axis match ahead of missing and extra axes", () => { |
| 307 | + const anchor = { |
| 308 | + modelUid: "axis-anchor", |
| 309 | + displayName: "axis-anchor", |
| 310 | + familyUid: "wide", |
| 311 | + familyAxes: { A: { order: 1 }, B: { order: 2 } }, |
| 312 | + isFamilyDefault: true, |
| 313 | + }; |
| 314 | + const exact = { |
| 315 | + modelUid: "axis-exact", |
| 316 | + displayName: "axis-exact", |
| 317 | + familyUid: "wide", |
| 318 | + familyAxes: { A: { order: 1 }, B: { order: 2 } }, |
| 319 | + }; |
| 320 | + const missing = { |
| 321 | + modelUid: "axis-missing", |
| 322 | + displayName: "axis-missing", |
| 323 | + familyUid: "wide", |
| 324 | + familyAxes: { A: { order: 1 } }, |
| 325 | + }; |
| 326 | + const extra = { |
| 327 | + modelUid: "axis-extra", |
| 328 | + displayName: "axis-extra", |
| 329 | + familyUid: "wide", |
| 330 | + familyAxes: { A: { order: 1 }, B: { order: 2 }, C: { order: 1 }, D: { order: 3 } }, |
| 331 | + }; |
| 332 | + // The anchor itself always wins while listed: the ranking claim is about the |
| 333 | + // members it cannot see, so exclude it and pass it explicitly. |
| 334 | + const members = [missing, extra, exact]; |
| 335 | + expect(selectDevinFamilyMember(members, {}, anchor)?.modelUid).toBe("axis-exact"); |
| 336 | + // Without the exact row, a member missing one target axis scores baseline - 1, |
| 337 | + // while a member matching every target but exposing two extra axes scores |
| 338 | + // 0 + 2 — unequal mismatch counts, so the ordering is proven rather than a tie. |
| 339 | + expect(selectDevinFamilyMember([missing, extra], {}, anchor)?.modelUid).toBe("axis-missing"); |
| 340 | + }); |
262 | 341 | }); |
263 | 342 |
|
264 | 343 | describe("suffix fallback without family metadata never crosses families", () => { |
|
0 commit comments