Skip to content

Commit 4cb00f1

Browse files
zhangguiping-xydtzhangguiping-xydt
committed
fix(qwen): keep capability facts in internal routing
1 parent 5842bd8 commit 4cb00f1

9 files changed

Lines changed: 147 additions & 42 deletions

‎src/agents/tools/image-tool.helpers.ts‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,11 @@ import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/st
88
import type { OpenClawConfig } from "../../config/types.openclaw.js";
99
import type { AssistantMessage } from "../../llm/types.js";
1010
import { configuredModelInputSupportsImage } from "../../media-understanding/known-model-capabilities.js";
11-
import type { MediaUnderstandingProviderModelCapabilities } from "../../media-understanding/model-capability-overrides.js";
12-
import { providerModelCapabilities } from "../../media-understanding/model-capability-overrides.js";
11+
import {
12+
knownProviderModelCapabilities,
13+
providerModelCapabilities,
14+
type MediaUnderstandingProviderModelCapabilities,
15+
} from "../../media-understanding/model-capability-overrides.js";
1316
import { extractAssistantText } from "../embedded-agent-utils.js";
1417
import { isMinimaxVlmProvider } from "../minimax-vlm.js";
1518
import { findNormalizedProviderValue, normalizeProviderId } from "../model-selection.js";
@@ -207,7 +210,9 @@ function findConfiguredImageModelMatches(params: {
207210
!configuredModelInputSupportsImage({
208211
modelId,
209212
input: entry?.input,
210-
provider: providerModelCapabilities(params.providerRegistry?.get(provider)),
213+
provider:
214+
providerModelCapabilities(params.providerRegistry?.get(provider)) ??
215+
knownProviderModelCapabilities(provider),
211216
})
212217
) {
213218
continue;
@@ -299,9 +304,10 @@ export function resolveProviderVisionModelFromConfig(params: {
299304
params.provider,
300305
) as unknown as { models?: Array<{ id?: string; input?: string[] }> } | undefined;
301306
const models = providerCfg?.models ?? [];
302-
const providerMetadata = providerModelCapabilities(
303-
params.providerRegistry?.get(normalizeProviderId(params.provider)),
304-
);
307+
const providerId = normalizeProviderId(params.provider);
308+
const providerMetadata =
309+
providerModelCapabilities(params.providerRegistry?.get(providerId)) ??
310+
knownProviderModelCapabilities(providerId);
305311
const picked = models.find((m) => {
306312
const id = (m?.id ?? "").trim();
307313
return Boolean(

‎src/agents/tools/image-tool.test.ts‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import type { SandboxFsBridge } from "../sandbox/fs-bridge.js";
2424
import { createHostSandboxFsBridge } from "../test-helpers/host-sandbox-fs-bridge.js";
2525
import { createUnsafeMountedSandbox } from "../test-helpers/unsafe-mounted-sandbox.js";
2626
import { makeZeroUsageSnapshot } from "../usage.js";
27+
import { resolveProviderVisionModelFromConfig } from "./image-tool.helpers.js";
2728
import { testing, createImageTool, resolveImageModelConfigForTool } from "./image-tool.js";
2829
import { resolveMediaToolInboundRoots } from "./media-tool-shared.js";
2930

@@ -1596,6 +1597,52 @@ describe("image tool implicit imageModel config", () => {
15961597
});
15971598
});
15981599

1600+
it("passes provider capability metadata into implicit default image routing", async () => {
1601+
await withTempAgentDir(async (agentDir) => {
1602+
vi.stubEnv("DASHSCOPE_API_KEY", "qwen-test");
1603+
const qwenProvider: ImageToolTestProvider = {
1604+
id: "qwen",
1605+
capabilities: ["image"],
1606+
defaultModels: { image: "qwen-vl-max-latest" },
1607+
modelCapabilityOverrides: { nonImageModels: ["qwen3.7-max"] },
1608+
};
1609+
const providerRegistry = new Map<string, ImageToolTestProvider>([["qwen", qwenProvider]]);
1610+
testing.setProviderDepsForTest({
1611+
buildProviderRegistry: () => providerRegistry,
1612+
getMediaUnderstandingProvider: (id: string, registry: Map<string, ImageToolTestProvider>) =>
1613+
registry.get(id.toLowerCase()),
1614+
describeImageWithModel: describeGenericImageWithModel,
1615+
describeImagesWithModel: describeGenericImagesWithModel,
1616+
resolveAutoMediaKeyProviders: ({ capability }) => (capability === "image" ? ["qwen"] : []),
1617+
resolveDefaultMediaModel: ({ providerId, capability, providerRegistry }) => {
1618+
if (providerId !== "qwen" || capability !== "image") {
1619+
return undefined;
1620+
}
1621+
const provider = providerRegistry?.get("qwen") as ImageToolTestProvider | undefined;
1622+
return provider?.modelCapabilityOverrides ? "qwen-vl-max-latest" : "qwen3.7-max";
1623+
},
1624+
});
1625+
const cfg: OpenClawConfig = {
1626+
agents: { defaults: { model: { primary: "qwen/qwen3.7-max" } } },
1627+
models: {
1628+
mode: "merge",
1629+
providers: {
1630+
qwen: {
1631+
baseUrl: "https://dashscope.aliyuncs.com/compatible-mode/v1",
1632+
apiKey: "${DASHSCOPE_API_KEY}",
1633+
api: "openai-completions",
1634+
models: [makeModelDefinition("qwen3.7-max", ["text", "image"])],
1635+
},
1636+
},
1637+
},
1638+
};
1639+
1640+
expect(resolveImageModelConfigForTool({ cfg, agentDir })).toEqual({
1641+
primary: "qwen/qwen-vl-max-latest",
1642+
});
1643+
});
1644+
});
1645+
15991646
it("pairs opencode primary with the plugin-owned image model when auth exists", async () => {
16001647
await withTempAgentDir(async (agentDir) => {
16011648
vi.stubEnv("OPENCODE_API_KEY", "opencode-test");
@@ -1797,6 +1844,23 @@ describe("image tool implicit imageModel config", () => {
17971844
});
17981845
});
17991846

1847+
it("ignores known non-image Qwen models when resolving configured provider vision", () => {
1848+
const cfg: OpenClawConfig = {
1849+
models: {
1850+
providers: {
1851+
qwen: {
1852+
baseUrl: "https://dashscope.aliyuncs.com/compatible-mode/v1",
1853+
apiKey: "${DASHSCOPE_API_KEY}",
1854+
api: "openai-completions",
1855+
models: [makeModelDefinition("qwen3.7-max", ["text", "image"])],
1856+
},
1857+
},
1858+
},
1859+
};
1860+
1861+
expect(resolveProviderVisionModelFromConfig({ cfg, provider: "qwen" })).toBeNull();
1862+
});
1863+
18001864
it("runs providerless explicit image models on the inferred provider", async () => {
18011865
await withTempAgentDir(async (agentDir) => {
18021866
const describeImage = vi.fn(async (params: ImageDescriptionRequest) => ({

‎src/agents/tools/image-tool.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -232,7 +232,7 @@ function resolveImageToolMaxTokens(modelMaxTokens: number | undefined, requested
232232
*/
233233
export function resolveImageModelConfigForTool(params: {
234234
cfg?: OpenClawConfig;
235-
providerRegistry?: ReadonlyMap<string, MediaUnderstandingProviderModelCapabilities>;
235+
providerRegistry?: Map<string, MediaUnderstandingProvider>;
236236
agentDir: string;
237237
workspaceDir?: string;
238238
authStore?: AuthProfileStore;
@@ -302,6 +302,7 @@ export function resolveImageModelConfigForTool(params: {
302302
cfg: params.cfg,
303303
workspaceDir: params.workspaceDir,
304304
providerId: primary.provider,
305+
providerRegistry,
305306
capability: "image",
306307
includeConfiguredImageModels: !isMinimaxVlmProvider(primary.provider),
307308
});
@@ -328,6 +329,7 @@ export function resolveImageModelConfigForTool(params: {
328329
cfg: params.cfg,
329330
workspaceDir: params.workspaceDir,
330331
providerId,
332+
providerRegistry,
331333
capability: "image",
332334
includeConfiguredImageModels: !isMinimaxVlmProvider(providerId),
333335
});
@@ -795,9 +797,7 @@ export function createImageTool(options?: {
795797
}
796798
return null;
797799
}
798-
let providerRegistry:
799-
| ReadonlyMap<string, MediaUnderstandingProviderModelCapabilities>
800-
| undefined;
800+
let providerRegistry: Map<string, MediaUnderstandingProvider> | undefined;
801801
const getProviderRegistry = () => {
802802
providerRegistry ??= imageToolProviderDeps.buildProviderRegistry(undefined, options?.config);
803803
return providerRegistry;

‎src/agents/tools/pdf-tool.model-config.ts‎

Lines changed: 3 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,7 @@ import {
1111
resolveDocumentMediaModel,
1212
} from "../../media-understanding/defaults.js";
1313
import { configuredModelInputSupportsImage } from "../../media-understanding/known-model-capabilities.js";
14-
import {
15-
QWEN_MODEL_CAPABILITY_OVERRIDES,
16-
QWEN_MODEL_CAPABILITY_PROVIDER_IDS,
17-
} from "../../media-understanding/model-capability-overrides.js";
14+
import { knownProviderModelCapabilities } from "../../media-understanding/model-capability-overrides.js";
1815
import { normalizeMediaProviderId } from "../../media-understanding/provider-id.js";
1916
import { buildMediaUnderstandingRegistry } from "../../media-understanding/provider-registry.js";
2017
import type { AuthProfileStore } from "../auth-profiles/types.js";
@@ -64,19 +61,8 @@ function configuredProviderRuntimeCapabilityRegistry(params: {
6461
return undefined;
6562
}
6663
const providerId = normalizeMediaProviderId(params.providerId);
67-
const qwenProviderIds: readonly string[] = QWEN_MODEL_CAPABILITY_PROVIDER_IDS;
68-
if (!qwenProviderIds.includes(providerId)) {
69-
return undefined;
70-
}
71-
return new Map([
72-
[
73-
providerId,
74-
{
75-
id: providerId,
76-
modelCapabilityOverrides: QWEN_MODEL_CAPABILITY_OVERRIDES,
77-
},
78-
],
79-
]);
64+
const capabilities = knownProviderModelCapabilities(providerId);
65+
return capabilities ? new Map([[providerId, capabilities]]) : undefined;
8066
}
8167

8268
function resolveConfiguredImageRefsForPdf(params: {

‎src/media-understanding/config-provider-models.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,10 @@
22
// media-understanding auto-registration.
33
import type { OpenClawConfig } from "../config/types.js";
44
import { configuredModelInputSupportsImage } from "./known-model-capabilities.js";
5-
import type { MediaUnderstandingProviderModelCapabilities } from "./model-capability-overrides.js";
5+
import {
6+
knownProviderModelCapabilities,
7+
type MediaUnderstandingProviderModelCapabilities,
8+
} from "./model-capability-overrides.js";
69
import { normalizeMediaProviderId } from "./provider-id.js";
710

811
type ConfigProvider = NonNullable<
@@ -24,7 +27,9 @@ function hasImageCapableModel(params: {
2427
configuredModelInputSupportsImage({
2528
modelId,
2629
input: model?.input,
27-
provider: params.providerRegistry?.get(normalizeMediaProviderId(params.providerId)),
30+
provider:
31+
params.providerRegistry?.get(normalizeMediaProviderId(params.providerId)) ??
32+
knownProviderModelCapabilities(params.providerId),
2833
}),
2934
);
3035
});

‎src/media-understanding/defaults.test.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,27 @@ describe("resolveDefaultMediaModel", () => {
187187
}),
188188
).toBe("auto");
189189
});
190+
191+
it("ignores known non-image Qwen configured models when a runtime registry is provided", () => {
192+
const cfg = {
193+
models: {
194+
providers: {
195+
qwen: {
196+
models: [{ id: "qwen3.7-max", input: ["text", "image"] }],
197+
},
198+
},
199+
},
200+
} as never;
201+
202+
expect(
203+
resolveDefaultMediaModel({
204+
providerId: "qwen",
205+
capability: "image",
206+
cfg,
207+
providerRegistry: new Map(),
208+
}),
209+
).toBeUndefined();
210+
});
190211
});
191212

192213
describe("resolveAutoMediaKeyProviders", () => {

‎src/media-understanding/defaults.ts‎

Lines changed: 18 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { resolveRuntimeConfigCacheKey } from "../config/runtime-snapshot.js";
66
import type { OpenClawConfig } from "../config/types.js";
77
import { configuredModelInputSupportsImage } from "./known-model-capabilities.js";
88
import { buildMediaUnderstandingManifestMetadataRegistry } from "./manifest-metadata.js";
9+
import { knownProviderModelCapabilities } from "./model-capability-overrides.js";
910
import {
1011
normalizeMediaExecutionProviderId,
1112
normalizeMediaProviderId,
@@ -24,14 +25,14 @@ export {
2425
MIN_AUDIO_FILE_BYTES,
2526
} from "./defaults.constants.js";
2627

27-
let defaultRegistryCache: Map<string, MediaUnderstandingProvider> | null = null;
28-
const configRegistryCache = new Map<string, Map<string, MediaUnderstandingProvider>>();
28+
let defaultRegistryCache: ReadonlyMap<string, MediaUnderstandingProvider> | null = null;
29+
const configRegistryCache = new Map<string, ReadonlyMap<string, MediaUnderstandingProvider>>();
2930
const MAX_CONFIG_REGISTRY_CACHE_ENTRIES = 32;
3031

3132
function cacheConfigRegistry(
3233
key: string,
33-
registry: Map<string, MediaUnderstandingProvider>,
34-
): Map<string, MediaUnderstandingProvider> {
34+
registry: ReadonlyMap<string, MediaUnderstandingProvider>,
35+
): ReadonlyMap<string, MediaUnderstandingProvider> {
3536
// Config snapshots are process-stable enough for bounded reuse; cap entries so
3637
// tests and multi-workspace runs cannot grow this cache without limit.
3738
if (
@@ -73,7 +74,7 @@ function providerHasDeclaredCapability(
7374
function resolveConfiguredImageProviderModel(params: {
7475
cfg?: OpenClawConfig;
7576
providerId: string;
76-
registry: Map<string, MediaUnderstandingProvider>;
77+
registry: ReadonlyMap<string, MediaUnderstandingProvider>;
7778
}): string | undefined {
7879
const normalizedProviderId = normalizeMediaProviderId(params.providerId);
7980
const providers = params.cfg?.models?.providers;
@@ -92,7 +93,9 @@ function resolveConfiguredImageProviderModel(params: {
9293
configuredModelInputSupportsImage({
9394
modelId: id,
9495
input: model?.input,
95-
provider: params.registry.get(normalizeMediaProviderId(providerKey)),
96+
provider:
97+
params.registry.get(normalizeMediaProviderId(providerKey)) ??
98+
knownProviderModelCapabilities(providerKey),
9699
}),
97100
);
98101
});
@@ -103,7 +106,7 @@ function resolveConfiguredImageProviderModel(params: {
103106

104107
function resolveConfiguredImageProviderIds(params: {
105108
cfg?: OpenClawConfig;
106-
registry: Map<string, MediaUnderstandingProvider>;
109+
registry: ReadonlyMap<string, MediaUnderstandingProvider>;
107110
}): string[] {
108111
const providers = params.cfg?.models?.providers;
109112
if (!providers || typeof providers !== "object") {
@@ -123,7 +126,9 @@ function resolveConfiguredImageProviderIds(params: {
123126
configuredModelInputSupportsImage({
124127
modelId: id,
125128
input: model?.input,
126-
provider: params.registry.get(normalizeMediaProviderId(providerKey)),
129+
provider:
130+
params.registry.get(normalizeMediaProviderId(providerKey)) ??
131+
knownProviderModelCapabilities(providerKey),
127132
}),
128133
);
129134
});
@@ -164,12 +169,12 @@ export function resolveDefaultMediaModel(params: {
164169
capability: MediaUnderstandingCapability;
165170
cfg?: OpenClawConfig;
166171
workspaceDir?: string;
167-
providerRegistry?: Map<string, MediaUnderstandingProvider>;
172+
providerRegistry?: ReadonlyMap<string, MediaUnderstandingProvider>;
168173
includeConfiguredImageModels?: boolean;
169174
}): string | undefined {
170175
const registry =
171176
params.providerRegistry ?? resolveDefaultRegistry(params.cfg, params.workspaceDir);
172-
if (!params.providerRegistry && params.includeConfiguredImageModels !== false) {
177+
if (params.includeConfiguredImageModels !== false) {
173178
const configuredImageModel =
174179
params.capability === "image"
175180
? resolveConfiguredImageProviderModel({
@@ -197,7 +202,7 @@ export function resolveAutoMediaKeyProviders(params: {
197202
capability: MediaUnderstandingCapability;
198203
cfg?: OpenClawConfig;
199204
workspaceDir?: string;
200-
providerRegistry?: Map<string, MediaUnderstandingProvider>;
205+
providerRegistry?: ReadonlyMap<string, MediaUnderstandingProvider>;
201206
}): string[] {
202207
const registry =
203208
params.providerRegistry ?? resolveDefaultRegistry(params.cfg, params.workspaceDir);
@@ -236,7 +241,7 @@ export function providerSupportsNativePdfDocument(params: {
236241
providerId: string;
237242
cfg?: OpenClawConfig;
238243
workspaceDir?: string;
239-
providerRegistry?: Map<string, MediaUnderstandingProvider>;
244+
providerRegistry?: ReadonlyMap<string, MediaUnderstandingProvider>;
240245
}): boolean {
241246
const registry =
242247
params.providerRegistry ?? resolveDefaultRegistry(params.cfg, params.workspaceDir);
@@ -251,7 +256,7 @@ export function resolveDocumentMediaModel(params: {
251256
mode: "textExtraction" | "image";
252257
cfg?: OpenClawConfig;
253258
workspaceDir?: string;
254-
providerRegistry?: Map<string, MediaUnderstandingProvider>;
259+
providerRegistry?: ReadonlyMap<string, MediaUnderstandingProvider>;
255260
}): string | false | undefined {
256261
const registry =
257262
params.providerRegistry ?? resolveDefaultRegistry(params.cfg, params.workspaceDir);

‎src/media-understanding/manifest-metadata.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// loading plugin runtime code.
33
import type { OpenClawConfig } from "../config/types.js";
44
import { loadManifestMetadataSnapshot } from "../plugins/manifest-contract-eligibility.js";
5+
import { knownProviderModelCapabilities } from "./model-capability-overrides.js";
56
import { normalizeMediaProviderId } from "./provider-id.js";
67
import type { MediaUnderstandingProvider } from "./types.js";
78

@@ -39,6 +40,7 @@ export function buildMediaUnderstandingManifestMetadataRegistry(
3940
autoPriority: metadata.autoPriority,
4041
nativeDocumentInputs: metadata.nativeDocumentInputs,
4142
documentModels: metadata.documentModels,
43+
...knownProviderModelCapabilities(normalizedProviderId),
4244
});
4345
}
4446
}

‎src/media-understanding/model-capability-overrides.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import { normalizeMediaProviderId } from "./provider-id.js";
2+
13
export const QWEN_MODEL_CAPABILITY_PROVIDER_IDS = [
24
"qwen",
35
"qwencloud",
@@ -27,3 +29,17 @@ export function providerModelCapabilities(
2729
}
2830
return provider as MediaUnderstandingProviderModelCapabilities;
2931
}
32+
33+
export function knownProviderModelCapabilities(
34+
providerId: string,
35+
): MediaUnderstandingProviderModelCapabilities | undefined {
36+
const normalizedProviderId = normalizeMediaProviderId(providerId);
37+
const qwenProviderIds: readonly string[] = QWEN_MODEL_CAPABILITY_PROVIDER_IDS;
38+
if (!qwenProviderIds.includes(normalizedProviderId)) {
39+
return undefined;
40+
}
41+
return {
42+
id: normalizedProviderId,
43+
modelCapabilityOverrides: QWEN_MODEL_CAPABILITY_OVERRIDES,
44+
};
45+
}

0 commit comments

Comments
 (0)