Skip to content

Commit 220b4f7

Browse files
committed
fix(providers): activate vision sidecar for NVIDIA NIM text-only models
Add noVisionModels to the nvidia registry entry (issue #956). The NIM /v1/models catalog carries no input-modality metadata, so the registry is the only source of truth: without the list, the vision sidecar never runs and text-only NIM models either 400 on raw image parts upstream or are advertised text-only so the app blocks attachments client-side. Covers the text-only chat/code families from the live catalog (2026-08-04) plus the documented kimi family. Vision-capable NIM models (llama-3.2-*- vision-instruct, phi-3-vision, neva-22b, vila, nemotron-*-vl, ...) and non-chat endpoints stay out. Regression tests: registry list contents/exclusions, bare-config router inheritance, planVisionSidecar activation, and catalog inputModalities advertisement.
1 parent cb8fcf3 commit 220b4f7

2 files changed

Lines changed: 167 additions & 0 deletions

File tree

‎src/providers/registry.ts‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -520,6 +520,79 @@ const NVIDIA_NIM_KIMI_MODELS = [
520520
...NVIDIA_NIM_KIMI_THINKING_MODELS,
521521
"moonshotai/kimi-k2-instruct", "moonshotai/kimi-k2-instruct-0905",
522522
];
523+
// 260804 NVIDIA NIM text-only chat/code families (issue #956): the NIM /v1/models catalog
524+
// carries no input-modality metadata, so the registry is the only source of truth. Snapshot
525+
// from the live catalog (2026-08-04) plus the documented kimi family above. Vision-capable
526+
// NIM models (llama-3.2-*-vision-instruct, phi-3-vision, kosmos-2, fuyu-8b, deplot, neva-22b,
527+
// vila, nemotron-*-vl, nemotron-nano-12b-v2-vl, nemotron-3-nano-omni) and non-chat endpoints
528+
// (embeddings, guards, translators, detectors, image/video) stay out. New text-only ids must
529+
// be appended individually.
530+
const NVIDIA_NIM_NO_VISION_MODELS = [
531+
"01-ai/yi-large",
532+
"ai21labs/jamba-1.5-large-instruct",
533+
"aisingapore/sea-lion-7b-instruct",
534+
"bigcode/starcoder2-15b",
535+
"databricks/dbrx-instruct",
536+
"deepseek-ai/deepseek-coder-6.7b-instruct",
537+
"deepseek-ai/deepseek-v4-flash",
538+
"deepseek-ai/deepseek-v4-pro",
539+
"google/codegemma-1.1-7b",
540+
"google/codegemma-7b",
541+
"google/gemma-2b",
542+
"google/recurrentgemma-2b",
543+
"ibm/granite-3.0-3b-a800m-instruct",
544+
"ibm/granite-3.0-8b-instruct",
545+
"ibm/granite-34b-code-instruct",
546+
"ibm/granite-8b-code-instruct",
547+
"meta/codellama-70b",
548+
"meta/llama-3.1-70b-instruct",
549+
"meta/llama-3.1-8b-instruct",
550+
"meta/llama-3.2-1b-instruct",
551+
"meta/llama-3.2-3b-instruct",
552+
"meta/llama-3.3-70b-instruct",
553+
"meta/llama2-70b",
554+
"microsoft/phi-3.5-moe-instruct",
555+
"minimaxai/minimax-m3",
556+
"mistralai/codestral-22b-instruct-v0.1",
557+
"mistralai/mistral-7b-instruct-v0.3",
558+
"mistralai/mistral-large",
559+
"mistralai/mistral-large-2-instruct",
560+
"mistralai/mistral-medium-3.5-128b",
561+
"mistralai/mistral-nemotron",
562+
"mistralai/mixtral-8x22b-v0.1",
563+
"moonshotai/kimi-k2.6",
564+
"moonshotai/kimi-k2.5",
565+
"moonshotai/kimi-k2-thinking",
566+
"moonshotai/kimi-k2-instruct",
567+
"moonshotai/kimi-k2-instruct-0905",
568+
"nv-mistralai/mistral-nemo-12b-instruct",
569+
"nvidia/llama-3.1-nemotron-51b-instruct",
570+
"nvidia/llama-3.1-nemotron-70b-instruct",
571+
"nvidia/llama-3.1-nemotron-nano-8b-v1",
572+
"nvidia/llama-3.1-nemotron-ultra-253b-v1",
573+
"nvidia/llama-3.3-nemotron-super-49b-v1",
574+
"nvidia/llama-3.3-nemotron-super-49b-v1.5",
575+
"nvidia/llama3-chatqa-1.5-70b",
576+
"nvidia/mistral-nemo-minitron-8b-8k-instruct",
577+
"nvidia/nemotron-3-nano-30b-a3b",
578+
"nvidia/nemotron-3-super-120b-a12b",
579+
"nvidia/nemotron-3-ultra-550b-a55b",
580+
"nvidia/nemotron-4-340b-instruct",
581+
"nvidia/nemotron-mini-4b-instruct",
582+
"nvidia/nemotron-nano-3-30b-a3b",
583+
"nvidia/nvidia-nemotron-nano-9b-v2",
584+
"openai/gpt-oss-120b",
585+
"openai/gpt-oss-20b",
586+
"poolside/laguna-xs-2.1",
587+
"stepfun-ai/step-3.7-flash",
588+
"thinkingmachines/inkling",
589+
"writer/palmyra-creative-122b",
590+
"writer/palmyra-fin-70b-32k",
591+
"writer/palmyra-med-70b",
592+
"writer/palmyra-med-70b-32k",
593+
"z-ai/glm-5.2",
594+
"zyphra/zamba2-7b-instruct",
595+
];
523596
const KIMI_CODING_MODEL_CONTEXT_WINDOWS: Record<string, number> = Object.fromEntries(
524597
KIMI_CODING_MODELS.map(id => [id, id === "k3[1m]" ? KIMI_K3_1M_CONTEXT_WINDOW : KIMI_K3_STANDARD_CONTEXT_WINDOW]),
525598
);
@@ -1230,11 +1303,14 @@ export const PROVIDER_REGISTRY: readonly ProviderRegistryEntry[] = [
12301303
// family is live-discovered with no capability metadata, so Codex would otherwise send
12311304
// reasoning_effort=medium. Exact-id lists per modelInList semantics; gpt-oss on NIM keeps
12321305
// its working reasoning_effort. Future kimi ids must be appended individually.
1306+
// - NIM exposes no input modalities, so text-only chat/code families are listed in
1307+
// noVisionModels (issue #956) to activate the vision sidecar and advertise image input.
12331308
{
12341309
id: "nvidia", label: "NVIDIA NIM", baseUrl: "https://integrate.api.nvidia.com/v1", adapter: "openai-chat", authKind: "key", dashboardUrl: "https://build.nvidia.com",
12351310
// Free pricing, but an API key is still required (free key from build.nvidia.com).
12361311
freeTier: true,
12371312
parallelToolCalls: false,
1313+
noVisionModels: NVIDIA_NIM_NO_VISION_MODELS,
12381314
noReasoningModels: NVIDIA_NIM_KIMI_MODELS,
12391315
modelReasoningEfforts: Object.fromEntries(NVIDIA_NIM_KIMI_MODELS.map(id => [id, []])),
12401316
preserveReasoningContentModels: NVIDIA_NIM_KIMI_THINKING_MODELS,

‎tests/nvidia-nim-hardening.test.ts‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,15 @@
11
// 260715 issue #126: NVIDIA NIM hardening — parallel_tool_calls opt-out, kimi
22
// reasoning_effort suppression, and openai-chat formatErrorBody detail surfacing.
3+
// 260804 issue #956: NIM text-only families get noVisionModels so the vision sidecar runs.
34
// Plan/evidence: devlog/_plan/260715_issue126_nim_kimi.
45
import { describe, expect, test } from "bun:test";
56
import { createOpenAIChatAdapter, formatOpenAIChatErrorBody } from "../src/adapters/openai-chat";
67
import { applyProviderConfigHints, normalizeRoutedCatalogEntry } from "../src/codex/catalog";
8+
import { PROVIDER_REGISTRY } from "../src/providers/registry";
9+
import { parseRequest } from "../src/responses/parser";
710
import { routeModel } from "../src/router";
811
import type { OcxConfig, OcxParsedRequest, OcxTool } from "../src/types";
12+
import { planVisionSidecar } from "../src/vision";
913

1014
const tools: OcxTool[] = [{ name: "shell", description: "run", parameters: { type: "object" } }];
1115

@@ -86,6 +90,93 @@ describe("nvidia NIM registry hardening (issue #126)", () => {
8690
const entry = normalizeRoutedCatalogEntry({ slug: "nvidia/moonshotai/kimi-k2.6" }, hinted.parallelToolCalls);
8791
expect(entry.supports_parallel_tool_calls).toBe(false);
8892
});
93+
94+
test("registry nvidia entry declares text-only families and excludes vision-capable models", () => {
95+
const nvidia = PROVIDER_REGISTRY.find(entry => entry.id === "nvidia")!;
96+
for (const id of [
97+
"deepseek-ai/deepseek-v4-flash",
98+
"deepseek-ai/deepseek-v4-pro",
99+
"z-ai/glm-5.2",
100+
"minimaxai/minimax-m3",
101+
"moonshotai/kimi-k2.6",
102+
"nvidia/nemotron-3-ultra-550b-a55b",
103+
"nvidia/nemotron-3-super-120b-a12b",
104+
"openai/gpt-oss-120b",
105+
]) {
106+
expect(nvidia.noVisionModels).toContain(id);
107+
}
108+
for (const id of [
109+
"meta/llama-3.2-11b-vision-instruct",
110+
"meta/llama-3.2-90b-vision-instruct",
111+
"microsoft/phi-3-vision-128k-instruct",
112+
"adept/fuyu-8b",
113+
"nvidia/llama-3.1-nemotron-nano-vl-8b-v1",
114+
"nvidia/nemotron-nano-12b-v2-vl",
115+
"nvidia/neva-22b",
116+
"nvidia/vila",
117+
]) {
118+
expect(nvidia.noVisionModels).not.toContain(id);
119+
}
120+
});
121+
122+
test("bare persisted nvidia config inherits noVisionModels from the registry", () => {
123+
const route = routeModel(nvidiaConfig(), "nvidia/deepseek-ai/deepseek-v4-flash");
124+
expect(route.provider.noVisionModels).toContain("deepseek-ai/deepseek-v4-flash");
125+
expect(route.modelId).toBe("deepseek-ai/deepseek-v4-flash");
126+
});
127+
128+
test("vision sidecar plans for text-only NIM models but not vision-capable ones", () => {
129+
const config = nvidiaConfig();
130+
const openAiSidecar = {
131+
providerName: "openai" as const,
132+
provider: { adapter: "openai-responses", baseUrl: "https://chatgpt.test/v1", authMode: "forward" as const },
133+
accountMode: "direct" as const,
134+
authContext: { kind: "main" as const, accountId: null },
135+
headers: new Headers({ authorization: "Bearer chatgpt" }),
136+
};
137+
const withImage = parseRequest({
138+
model: "nvidia/deepseek-ai/deepseek-v4-flash",
139+
input: [{
140+
type: "message",
141+
role: "user",
142+
content: [
143+
{ type: "input_text", text: "What is in this screenshot?" },
144+
{ type: "input_image", image_url: "data:image/png;base64,aGVsbG8=" },
145+
],
146+
}],
147+
});
148+
const noImage = parseRequest({
149+
model: "nvidia/deepseek-ai/deepseek-v4-flash",
150+
input: [{ type: "message", role: "user", content: [{ type: "input_text", text: "hi" }] }],
151+
});
152+
153+
const textRoute = routeModel(config, "nvidia/deepseek-ai/deepseek-v4-flash");
154+
expect(planVisionSidecar(config, textRoute.provider, textRoute.modelId, withImage, openAiSidecar))
155+
.toMatchObject({ backend: "openai" });
156+
expect(planVisionSidecar(config, textRoute.provider, textRoute.modelId, noImage, openAiSidecar))
157+
.toBeUndefined();
158+
159+
const visionRoute = routeModel(config, "nvidia/meta/llama-3.2-11b-vision-instruct");
160+
expect(planVisionSidecar(config, visionRoute.provider, visionRoute.modelId, withImage, openAiSidecar))
161+
.toBeUndefined();
162+
});
163+
164+
test("catalog advertises image input for text-only NIM models, not for vision-capable ones", () => {
165+
const config = nvidiaConfig();
166+
const textRoute = routeModel(config, "nvidia/deepseek-ai/deepseek-v4-flash");
167+
const hinted = applyProviderConfigHints("nvidia", textRoute.provider, {
168+
id: "deepseek-ai/deepseek-v4-flash",
169+
provider: "nvidia",
170+
});
171+
expect(hinted.inputModalities).toContain("image");
172+
173+
const visionRoute = routeModel(config, "nvidia/meta/llama-3.2-11b-vision-instruct");
174+
const visionHinted = applyProviderConfigHints("nvidia", visionRoute.provider, {
175+
id: "meta/llama-3.2-11b-vision-instruct",
176+
provider: "nvidia",
177+
});
178+
expect(visionHinted.inputModalities).toBeUndefined();
179+
});
89180
});
90181

91182
describe("formatOpenAIChatErrorBody (web-search sidecar detail surfacing)", () => {

0 commit comments

Comments
 (0)