Skip to content

Commit 8c8dfa7

Browse files
steipetesteipete
committed
refactor(models): share catalog capability lookup
1 parent defdded commit 8c8dfa7

7 files changed

Lines changed: 201 additions & 45 deletions

File tree

‎src/agents/model-catalog-lookup.ts‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
import {
2+
normalizeLowercaseStringOrEmpty,
3+
normalizeOptionalString,
4+
} from "../shared/string-coerce.js";
5+
import type { ModelCatalogEntry, ModelInputType } from "./model-catalog.types.js";
6+
import { normalizeProviderId } from "./provider-id.js";
7+
8+
export function modelSupportsInput(
9+
entry: ModelCatalogEntry | undefined,
10+
input: ModelInputType,
11+
): boolean {
12+
return entry?.input?.includes(input) ?? false;
13+
}
14+
15+
export function findModelInCatalog(
16+
catalog: ModelCatalogEntry[],
17+
provider: string,
18+
modelId: string,
19+
): ModelCatalogEntry | undefined {
20+
const normalizedProvider = normalizeProviderId(provider);
21+
const normalizedModelId = normalizeLowercaseStringOrEmpty(modelId);
22+
return catalog.find(
23+
(entry) =>
24+
normalizeProviderId(entry.provider) === normalizedProvider &&
25+
normalizeLowercaseStringOrEmpty(entry.id) === normalizedModelId,
26+
);
27+
}
28+
29+
export function findModelCatalogEntry(
30+
catalog: ModelCatalogEntry[],
31+
params: { provider?: string; modelId: string },
32+
): ModelCatalogEntry | undefined {
33+
const modelId = normalizeOptionalString(params.modelId) ?? "";
34+
if (!modelId) {
35+
return undefined;
36+
}
37+
38+
const provider = normalizeOptionalString(params.provider);
39+
if (provider) {
40+
return findModelInCatalog(catalog, provider, modelId);
41+
}
42+
43+
const normalizedModelId = normalizeLowercaseStringOrEmpty(modelId);
44+
const matches = catalog.filter(
45+
(entry) => normalizeLowercaseStringOrEmpty(entry.id) === normalizedModelId,
46+
);
47+
return matches.length === 1 ? matches[0] : undefined;
48+
}

‎src/agents/model-catalog.test.ts‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,10 @@ import { resetLogger, setLoggerOverride } from "../logging/logger.js";
55
type PiSdkModule = typeof import("./pi-model-discovery.js");
66

77
let __setModelCatalogImportForTest: typeof import("./model-catalog.js").__setModelCatalogImportForTest;
8+
let findModelCatalogEntry: typeof import("./model-catalog.js").findModelCatalogEntry;
89
let findModelInCatalog: typeof import("./model-catalog.js").findModelInCatalog;
910
let loadModelCatalog: typeof import("./model-catalog.js").loadModelCatalog;
11+
let modelSupportsInput: typeof import("./model-catalog.js").modelSupportsInput;
1012
let resetModelCatalogCacheForTest: typeof import("./model-catalog.js").resetModelCatalogCacheForTest;
1113
let augmentCatalogMock: ReturnType<typeof vi.fn>;
1214
let ensureOpenClawModelsJsonMock: ReturnType<typeof vi.fn>;
@@ -73,8 +75,10 @@ describe("loadModelCatalog", () => {
7375

7476
({
7577
__setModelCatalogImportForTest,
78+
findModelCatalogEntry,
7679
findModelInCatalog,
7780
loadModelCatalog,
81+
modelSupportsInput,
7882
resetModelCatalogCacheForTest,
7983
} = await import("./model-catalog.js"));
8084
const providerRuntime = await import("../plugins/provider-runtime.runtime.js");
@@ -482,4 +486,23 @@ describe("loadModelCatalog", () => {
482486
name: "GLM-5",
483487
});
484488
});
489+
490+
it("resolves catalog entries with explicit providers and unique providerless matches", () => {
491+
const catalog = [
492+
{ provider: "first", id: "shared", name: "First", input: ["text"] },
493+
{ provider: "second", id: "shared", name: "Second", input: ["text", "image"] },
494+
{ provider: "modelscope", id: "qwen/qwen3.5-35b-a3b", name: "Qwen", input: ["text"] },
495+
] satisfies Awaited<ReturnType<typeof loadModelCatalog>>;
496+
497+
expect(findModelCatalogEntry(catalog, { provider: "second", modelId: "SHARED" })).toEqual(
498+
catalog[1],
499+
);
500+
expect(
501+
findModelCatalogEntry(catalog, { provider: "modelscope", modelId: "Qwen/Qwen3.5-35B-A3B" }),
502+
).toEqual(catalog[2]);
503+
expect(findModelCatalogEntry(catalog, { modelId: "shared" })).toBeUndefined();
504+
expect(findModelCatalogEntry(catalog, { modelId: "Qwen/Qwen3.5-35B-A3B" })).toEqual(catalog[2]);
505+
expect(modelSupportsInput(catalog[1], "image")).toBe(true);
506+
expect(modelSupportsInput(catalog[2], "image")).toBe(false);
507+
});
485508
});

‎src/agents/model-catalog.ts‎

Lines changed: 8 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import {
88
normalizeOptionalString,
99
} from "../shared/string-coerce.js";
1010
import { resolveOpenClawAgentDir } from "./agent-paths.js";
11+
import { modelSupportsInput as modelCatalogEntrySupportsInput } from "./model-catalog-lookup.js";
1112
import type { ModelCatalogEntry, ModelInputType } from "./model-catalog.types.js";
1213
import { buildConfiguredModelCatalog } from "./model-selection-shared.js";
1314
import { ensureOpenClawModelsJson } from "./models-config.js";
@@ -16,6 +17,11 @@ import { normalizeProviderId } from "./provider-id.js";
1617
const log = createSubsystemLogger("model-catalog");
1718

1819
export type { ModelCatalogEntry, ModelInputType } from "./model-catalog.types.js";
20+
export {
21+
findModelCatalogEntry,
22+
findModelInCatalog,
23+
modelSupportsInput,
24+
} from "./model-catalog-lookup.js";
1925

2026
type DiscoveredModel = {
2127
id: string;
@@ -238,29 +244,12 @@ export async function loadModelCatalog(params?: {
238244
* Check if a model supports image input based on its catalog entry.
239245
*/
240246
export function modelSupportsVision(entry: ModelCatalogEntry | undefined): boolean {
241-
return entry?.input?.includes("image") ?? false;
247+
return modelCatalogEntrySupportsInput(entry, "image");
242248
}
243249

244250
/**
245251
* Check if a model supports native document/PDF input based on its catalog entry.
246252
*/
247253
export function modelSupportsDocument(entry: ModelCatalogEntry | undefined): boolean {
248-
return entry?.input?.includes("document") ?? false;
249-
}
250-
251-
/**
252-
* Find a model in the catalog by provider and model ID.
253-
*/
254-
export function findModelInCatalog(
255-
catalog: ModelCatalogEntry[],
256-
provider: string,
257-
modelId: string,
258-
): ModelCatalogEntry | undefined {
259-
const normalizedProvider = normalizeProviderId(provider);
260-
const normalizedModelId = normalizeLowercaseStringOrEmpty(modelId);
261-
return catalog.find(
262-
(entry) =>
263-
normalizeProviderId(entry.provider) === normalizedProvider &&
264-
normalizeLowercaseStringOrEmpty(entry.id) === normalizedModelId,
265-
);
254+
return modelCatalogEntrySupportsInput(entry, "document");
266255
}

‎src/agents/model-selection-shared.ts‎

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import {
88
import { sanitizeForLog, stripAnsi } from "../terminal/ansi.js";
99
import { resolveConfiguredProviderFallback } from "./configured-provider-fallback.js";
1010
import { DEFAULT_PROVIDER } from "./defaults.js";
11+
import { findModelCatalogEntry } from "./model-catalog-lookup.js";
1112
import type { ModelCatalogEntry } from "./model-catalog.types.js";
1213
import { splitTrailingAuthProfile } from "./model-ref-profile.js";
1314
import { normalizeStaticProviderModelId } from "./model-ref-shared.js";
@@ -573,7 +574,18 @@ export function buildAllowedModelSetWithFallbacks(params: {
573574
}
574575

575576
const allowedKeys = new Set<string>();
577+
const allowedRefs: ModelRef[] = [];
576578
const syntheticCatalogEntries = new Map<string, ModelCatalogEntry>();
579+
const addAllowedCatalogRef = (ref: ModelRef) => {
580+
if (
581+
!allowedRefs.some(
582+
(existing) =>
583+
modelKey(existing.provider, existing.model) === modelKey(ref.provider, ref.model),
584+
)
585+
) {
586+
allowedRefs.push(ref);
587+
}
588+
};
577589
const addAllowedModelRef = (raw: string) => {
578590
const trimmed = raw.trim();
579591
const defaultProvider = !trimmed.includes("/")
@@ -594,8 +606,12 @@ export function buildAllowedModelSetWithFallbacks(params: {
594606
}
595607
const key = modelKey(parsed.provider, parsed.model);
596608
allowedKeys.add(key);
609+
addAllowedCatalogRef(parsed);
597610

598-
if (!catalogKeys.has(key) && !syntheticCatalogEntries.has(key)) {
611+
if (
612+
!findModelCatalogEntry(catalog, { provider: parsed.provider, modelId: parsed.model }) &&
613+
!syntheticCatalogEntries.has(key)
614+
) {
599615
syntheticCatalogEntries.set(key, buildSyntheticAllowedCatalogEntry({ parsed, metadata }));
600616
}
601617
};
@@ -610,10 +626,18 @@ export function buildAllowedModelSetWithFallbacks(params: {
610626

611627
if (defaultKey) {
612628
allowedKeys.add(defaultKey);
629+
if (defaultRef) {
630+
addAllowedCatalogRef(defaultRef);
631+
}
613632
}
614633

615634
const allowedCatalog = [
616-
...catalog.filter((entry) => allowedKeys.has(modelKey(entry.provider, entry.id))),
635+
...catalog.filter((entry) =>
636+
allowedRefs.some(
637+
(ref) =>
638+
findModelCatalogEntry([entry], { provider: ref.provider, modelId: ref.model }) === entry,
639+
),
640+
),
617641
...syntheticCatalogEntries.values(),
618642
];
619643

@@ -655,7 +679,12 @@ export function getModelRefStatusFromAllowedSet(params: {
655679
const key = modelKey(params.ref.provider, params.ref.model);
656680
return {
657681
key,
658-
inCatalog: params.catalog.some((entry) => modelKey(entry.provider, entry.id) === key),
682+
inCatalog: Boolean(
683+
findModelCatalogEntry(params.catalog, {
684+
provider: params.ref.provider,
685+
modelId: params.ref.model,
686+
}),
687+
),
659688
allowAny: params.allowed.allowAny,
660689
allowed: params.allowed.allowAny || params.allowed.allowedKeys.has(key),
661690
};

‎src/agents/model-selection.test.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -654,6 +654,39 @@ describe("model-selection", () => {
654654
]);
655655
});
656656

657+
it("matches allowlisted catalog entries with normalized provider and model ids", () => {
658+
const cfg: OpenClawConfig = {
659+
agents: {
660+
defaults: {
661+
models: {
662+
"modelscope/Qwen/Qwen3.5-35B-A3B": {},
663+
},
664+
},
665+
},
666+
} as unknown as OpenClawConfig;
667+
668+
const result = buildAllowedModelSet({
669+
cfg,
670+
catalog: [
671+
{
672+
provider: "modelscope",
673+
id: "qwen/qwen3.5-35b-a3b",
674+
name: "Qwen3.5 35B",
675+
input: ["text", "image"],
676+
},
677+
],
678+
defaultProvider: "anthropic",
679+
});
680+
681+
expect(result.allowedCatalog).toEqual([
682+
expect.objectContaining({
683+
provider: "modelscope",
684+
id: "qwen/qwen3.5-35b-a3b",
685+
input: ["text", "image"],
686+
}),
687+
]);
688+
});
689+
657690
it("applies configured provider metadata and alias to synthetic allowlist entries", () => {
658691
const cfg: OpenClawConfig = {
659692
agents: {

‎src/gateway/server-methods/chat.directive-tags.test.ts‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2403,6 +2403,54 @@ describe("chat directive tag stripping for non-streaming final payloads", () =>
24032403
]);
24042404
});
24052405

2406+
it("keeps image attachments inline for configured custom vision models", async () => {
2407+
createTranscriptFixture("openclaw-chat-send-configured-custom-vision-");
2408+
mockState.finalText = "ok";
2409+
mockState.sessionEntry = {
2410+
modelProvider: "modelscope",
2411+
model: "Qwen/Qwen3.5-35B-A3B",
2412+
};
2413+
mockState.modelCatalog = [
2414+
{
2415+
provider: "modelscope",
2416+
id: "qwen/qwen3.5-35b-a3b",
2417+
name: "Qwen3.5 35B",
2418+
input: ["text", "image"],
2419+
},
2420+
];
2421+
const respond = vi.fn();
2422+
const context = createChatContext();
2423+
2424+
await runNonStreamingChatSend({
2425+
context,
2426+
respond,
2427+
idempotencyKey: "idem-configured-custom-vision",
2428+
message: "describe image",
2429+
requestParams: {
2430+
attachments: [
2431+
{
2432+
mimeType: "image/png",
2433+
content:
2434+
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/woAAn8B9FD5fHAAAAAASUVORK5CYII=",
2435+
},
2436+
],
2437+
},
2438+
expectBroadcast: false,
2439+
});
2440+
2441+
expect(mockState.lastDispatchImages).toEqual([
2442+
expect.objectContaining({
2443+
mimeType: "image/png",
2444+
data: expect.any(String),
2445+
}),
2446+
]);
2447+
expect(mockState.lastDispatchImageOrder).toEqual(["inline"]);
2448+
expect(mockState.lastDispatchCtx?.Body).toBe("describe image");
2449+
expect(mockState.savedMediaCalls).toEqual([
2450+
expect.objectContaining({ contentType: "image/png", subdir: "inbound" }),
2451+
]);
2452+
});
2453+
24062454
it("keeps image attachments for text-only sessions bound to ACP", async () => {
24072455
createTranscriptFixture("openclaw-chat-send-text-only-acp-bound-attachments-");
24082456
mockState.finalText = "ok";

‎src/gateway/session-utils.ts‎

Lines changed: 9 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,11 @@ import {
1010
} from "../agents/agent-scope.js";
1111
import { lookupContextTokens, resolveContextTokensForModel } from "../agents/context.js";
1212
import { DEFAULT_CONTEXT_TOKENS, DEFAULT_MODEL, DEFAULT_PROVIDER } from "../agents/defaults.js";
13-
import { findModelInCatalog, type ModelCatalogEntry } from "../agents/model-catalog.js";
13+
import {
14+
findModelCatalogEntry,
15+
modelSupportsInput,
16+
type ModelCatalogEntry,
17+
} from "../agents/model-catalog.js";
1418
import {
1519
inferUniqueProviderFromConfiguredModels,
1620
normalizeStoredOverrideModel,
@@ -68,8 +72,8 @@ import {
6872
} from "../shared/avatar-policy.js";
6973
import {
7074
normalizeLowercaseStringOrEmpty,
71-
normalizeOptionalLowercaseString,
7275
normalizeOptionalString,
76+
normalizeOptionalLowercaseString,
7377
} from "../shared/string-coerce.js";
7478
import { normalizeSessionDeliveryFields } from "../utils/delivery-context.shared.js";
7579
import { estimateUsageCost, resolveModelCostConfig } from "../utils/usage-format.js";
@@ -1115,23 +1119,6 @@ export function resolveSessionModelRef(
11151119
return resolved;
11161120
}
11171121

1118-
function findGatewayImageSupportCatalogEntry(params: {
1119-
catalog: ModelCatalogEntry[];
1120-
provider?: string;
1121-
model: string;
1122-
}): ModelCatalogEntry | undefined {
1123-
const provider = normalizeOptionalString(params.provider);
1124-
if (provider) {
1125-
return findModelInCatalog(params.catalog, provider, params.model);
1126-
}
1127-
1128-
const normalizedModel = normalizeLowercaseStringOrEmpty(params.model);
1129-
const matches = params.catalog.filter(
1130-
(entry) => normalizeLowercaseStringOrEmpty(entry.id) === normalizedModel,
1131-
);
1132-
return matches.length === 1 ? matches[0] : undefined;
1133-
}
1134-
11351122
export async function resolveGatewayModelSupportsImages(params: {
11361123
loadGatewayModelCatalog: () => Promise<ModelCatalogEntry[]>;
11371124
provider?: string;
@@ -1143,10 +1130,9 @@ export async function resolveGatewayModelSupportsImages(params: {
11431130

11441131
try {
11451132
const catalog = await params.loadGatewayModelCatalog();
1146-
const modelEntry = findGatewayImageSupportCatalogEntry({
1147-
catalog,
1133+
const modelEntry = findModelCatalogEntry(catalog, {
11481134
provider: params.provider,
1149-
model: params.model,
1135+
modelId: params.model,
11501136
});
11511137
const normalizedProvider = normalizeOptionalLowercaseString(
11521138
params.provider ?? modelEntry?.provider,
@@ -1156,7 +1142,7 @@ export async function resolveGatewayModelSupportsImages(params: {
11561142
normalizeLowercaseStringOrEmpty(modelEntry?.name),
11571143
].filter(Boolean);
11581144
if (modelEntry) {
1159-
if (modelEntry.input?.includes("image")) {
1145+
if (modelSupportsInput(modelEntry, "image")) {
11601146
return true;
11611147
}
11621148
// Legacy safety shim for stale persisted Foundry rows that predate

0 commit comments

Comments
 (0)