Skip to content

Commit 3161904

Browse files
committed
feat(vision): routed describe executor via loopback self-fetch (lidge-jun#2188 roadmap 180)
The routed backend describes images by POSTing the proxy's own /v1/chat/completions with the namespaced describer model, so every provider wire the router speaks is a valid describer. Recursion fence: the request carries x-opencodex-vision-describe, detected at the chat surface (bridge rebuilds headers) and honored at the Responses plan site — marked requests strip instead of describing (depth cap 1). Native chat fast path now defers image-bearing text-only-model requests to the Responses pipeline so vision coverage is symmetric. Admission ladder: env token, service token file, first apiKeys entry, sent as x-opencodex-api-key.
1 parent 21aec54 commit 3161904

6 files changed

Lines changed: 571 additions & 5 deletions

File tree

‎src/server/chat-completions.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -258,6 +258,10 @@ async function handleChatCompletionsWithBudget(
258258
abortSignal: req.signal,
259259
// Body is Responses-shaped by now, but the client spoke Chat Completions.
260260
inboundWire: "chat",
261+
// Terminal vision-describe marker (roadmap 180): the bridge rebuilds
262+
// headers from the FORWARD_HEADERS allowlist, which would drop the raw
263+
// header — so the fact is detected here and carried as an option flag.
264+
...(req.headers.get("x-opencodex-vision-describe") === "1" ? { visionDescribeTerminal: true } : {}),
261265
translatorBudget,
262266
...(logIds ? { onFirstOutput: () => recordFirstOutput(logCtx, logIds.start) } : {}),
263267
onNativePassthroughTerminal: status => finalizeNativeLog(httpStatusForRequestLogTerminal(status, logCtx), { terminalStatus: status, closeReason: "terminal" }),

‎src/server/chat-native.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import type { AdmissionLease } from "../lib/admission";
1111
import { readBoundedResponseBody } from "../lib/bounded-body";
1212
import { redactSecretString } from "../lib/redact";
1313
import { resolveClientRetryAfter } from "../lib/retry-after";
14+
import { isModelTextOnly } from "../vision";
1415
import {
1516
applyUpstreamRecoveryInit,
1617
fetchWithResetRetry,
@@ -61,6 +62,12 @@ export function isNativeChatRouteEligible(route: RouteResult, rawBody: Rec): boo
6162
if (rawBody.store === true || rawBody.background === true) return false;
6263
if (typeof rawBody.previous_response_id === "string" && rawBody.previous_response_id.length > 0) return false;
6364
if (rawBody.compaction_trigger !== undefined) return false;
65+
// Vision sidecar coverage (roadmap 180): a text-only routed model with an
66+
// image-bearing body must go through the Responses pipeline, whose plan
67+
// site describes or strips the image. The native fast path has no vision
68+
// handling, so letting it keep such a request forwards raw pixels to a
69+
// model the operator declared blind.
70+
if (isModelTextOnly(provider, route.modelId) && chatBodyCarriesImage(rawBody)) return false;
6471
if (Array.isArray(rawBody.tools)) {
6572
for (const tool of rawBody.tools) {
6673
if (!isRec(tool)) continue;
@@ -72,6 +79,19 @@ export function isNativeChatRouteEligible(route: RouteResult, rawBody: Rec): boo
7279
return true;
7380
}
7481

82+
/** Any messages[].content[] part of type image_url. */
83+
function chatBodyCarriesImage(rawBody: Rec): boolean {
84+
const messages = rawBody.messages;
85+
if (!Array.isArray(messages)) return false;
86+
for (const message of messages) {
87+
if (!isRec(message) || !Array.isArray(message.content)) continue;
88+
for (const part of message.content) {
89+
if (isRec(part) && part.type === "image_url") return true;
90+
}
91+
}
92+
return false;
93+
}
94+
7595
function chatCompletionJson(value: unknown): Rec | null {
7696
if (!isRec(value) || !Array.isArray(value.choices) || value.choices.length === 0) return null;
7797
return value;

‎src/server/responses/core.ts‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1227,6 +1227,15 @@ export interface HandleResponsesOptions {
12271227
onConsumedComboFailure?: (failure: ConsumedComboFailure) => void;
12281228
/** Caller-owned for Chat/Claude replay; omitted only at genuine Responses ingress. */
12291229
translatorBudget?: TranslatorBudget;
1230+
/**
1231+
* Terminal vision-describe marker (roadmap 180): true when the inbound
1232+
* request IS the vision sidecar's own loopback describe call. The plan site
1233+
* then STRIPS images instead of planning another describe — a depth cap of 1
1234+
* that holds under predicate drift and combo re-resolution. The Chat surface
1235+
* detects the raw `x-opencodex-vision-describe` header before its bridge
1236+
* rebuilds headers and carries the fact through this flag.
1237+
*/
1238+
visionDescribeTerminal?: boolean;
12301239
}
12311240

12321241

@@ -2725,7 +2734,15 @@ async function handleResponsesInner(
27252734
// Vision sidecar: the routed model can't see images (provider.noVisionModels). Describe each
27262735
// attached image through the selected sidecar backend and replace it with text BEFORE the main
27272736
// call, so the text-only model can reason about it.
2728-
const visionPlan = planVisionSidecar(config, route.provider, route.modelId, parsed, openAiSidecar);
2737+
// Terminal describe fence (roadmap 180): the sidecar's OWN loopback describe
2738+
// call must never plan another describe. The flag arrives from the Chat
2739+
// surface (whose bridge rebuilds headers) or as the raw header for native
2740+
// Responses callers. Marked + text-only routed model → strip, depth cap 1.
2741+
const visionDescribeTerminal = options.visionDescribeTerminal === true
2742+
|| req.headers.get("x-opencodex-vision-describe") === "1";
2743+
const visionPlan = visionDescribeTerminal
2744+
? undefined
2745+
: planVisionSidecar(config, route.provider, route.modelId, parsed, openAiSidecar);
27292746
const recordSidecarOutcome = openAiSidecar?.recordOutcome;
27302747
if (visionPlan) {
27312748
await describeImagesInPlace(

‎src/vision/index.ts‎

Lines changed: 68 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import { modelRecordValue } from "../reasoning-effort";
55
import type { VisionReasoningEffort } from "../reasoning-effort";
66
import { describeImage, type DescribeOutcome, type VisionSettings } from "./describe";
77
import { describeImageAnthropic } from "./anthropic-describe";
8+
import { describeImageRouted } from "./routed-describe";
9+
import { modelAcceptsImageInput } from "./eligibility";
810
import { normalizeVisionReasoningForModel } from "./reasoning";
911
import type { CodexAuthContext } from "../codex/auth-context";
1012
import { resolveSidecarAuth } from "../sidecar/auth";
@@ -239,17 +241,26 @@ export function resolveVisionBackend(
239241

240242
/** Native model used by the OpenAI vision helper, including its bounded default. */
241243
export function resolveOpenAiVisionModel(config: Pick<OcxConfig, "visionSidecar">): string {
242-
return config.visionSidecar?.model || DEFAULT_VISION_MODEL;
244+
const configured = config.visionSidecar?.model;
245+
// Namespaced routed ids never reach the forward executor (see
246+
// resolveEffectiveVisionModel).
247+
return configured && !configured.includes("/") ? configured : DEFAULT_VISION_MODEL;
243248
}
244249

245250
/** Effective describer model for the backend `planVisionSidecar` selected. */
246251
export function resolveEffectiveVisionModel(
247252
config: Pick<OcxConfig, "visionSidecar">,
248253
backend: "openai" | "anthropic",
249254
): string {
255+
const configured = config.visionSidecar?.model;
256+
// A namespaced "provider/model" id belongs to the routed backend only; the
257+
// forward/OAuth executors POST the model string verbatim, so it falls back
258+
// to the side's default here (PUT coherence rejects new writes of this
259+
// shape, but a legacy or hand-edited config must not break the executor).
260+
const usable = configured && !configured.includes("/") ? configured : undefined;
250261
return backend === "anthropic"
251-
? config.visionSidecar?.model || DEFAULT_ANTHROPIC_VISION_MODEL
252-
: resolveOpenAiVisionModel(config);
262+
? usable || DEFAULT_ANTHROPIC_VISION_MODEL
263+
: usable || DEFAULT_VISION_MODEL;
253264
}
254265

255266
/** A user/developer/toolResult message can carry images (toolResult: e.g. Codex view_image output). */
@@ -275,9 +286,13 @@ export function shouldResolveOpenAiVisionSidecar(
275286
}
276287

277288
export interface VisionPlan {
278-
backend: "openai" | "anthropic";
289+
backend: "openai" | "anthropic" | "routed";
279290
forwardSidecar?: ResolvedOpenAiForwardSidecar;
280291
anthropicSidecar?: AnthropicVisionProvider;
292+
/** Namespaced "provider/model" describer for the routed backend (roadmap 180). */
293+
routedModel?: string;
294+
/** Loopback dispatch inputs for the routed backend. */
295+
routedConfig?: Pick<OcxConfig, "port" | "apiKeys">;
281296
settings: VisionSettings;
282297
maxDescriptionsPerTurn: number;
283298
}
@@ -299,8 +314,45 @@ export function planVisionSidecar(
299314
if (!messagesHaveImage(parsed)) return undefined;
300315
const cfg = config.visionSidecar ?? {};
301316
if (cfg.enabled === false) return undefined;
317+
318+
// Routed arm (roadmap 180 revised): explicit backend + NAMESPACED explicit
319+
// model only — never inferred from credential availability. Plan-time
320+
// fence: the target must not be provably blind, and must not itself be a
321+
// model this planner would re-enter for (belt; the terminal marker on the
322+
// loopback request is the braces).
323+
if (cfg.backend === "routed") {
324+
const routedModel = cfg.model;
325+
const sep = routedModel ? routedModel.indexOf("/") : -1;
326+
if (routedModel && sep > 0) {
327+
const targetProvider = routedModel.slice(0, sep);
328+
const targetId = routedModel.slice(sep + 1);
329+
const targetProviderConfig = config.providers?.[targetProvider];
330+
const targetVisible = modelAcceptsImageInput(config, { provider: targetProvider, id: targetId }) !== false
331+
&& !(targetProviderConfig && isModelTextOnly(targetProviderConfig, targetId));
332+
if (targetVisible) {
333+
return {
334+
backend: "routed",
335+
routedModel,
336+
routedConfig: { port: config.port, ...(config.apiKeys ? { apiKeys: config.apiKeys } : {}) },
337+
settings: {
338+
model: routedModel,
339+
reasoning: DEFAULT_REASONING,
340+
timeoutMs: resolveVisionTimeoutMs(cfg.timeoutMs),
341+
},
342+
maxDescriptionsPerTurn: resolveMaxDescriptionsPerTurn(cfg.maxDescriptionsPerTurn),
343+
};
344+
}
345+
}
346+
// Misconfigured routed backend (bare id, unknown provider, or provably
347+
// blind target): fall through to the legacy default order below rather
348+
// than dispatching a describe that cannot work.
349+
}
350+
302351
const anthropicSidecar = findAnthropicVisionProvider(config);
303352
const backend = resolveVisionBackend(cfg.backend, anthropicSidecar);
353+
// A namespaced routed model must never reach the forward/OAuth executors
354+
// (they POST the string verbatim); the effective-model resolver falls back
355+
// to each side's default in that case.
304356
const model = resolveEffectiveVisionModel(config, backend);
305357
const maxDescriptionsPerTurn = resolveMaxDescriptionsPerTurn(cfg.maxDescriptionsPerTurn);
306358

@@ -451,6 +503,18 @@ async function executeDescription(
451503
abortSignal?: AbortSignal,
452504
recordSidecarOutcome?: SidecarOutcomeRecorder,
453505
): Promise<DescribeOutcome> {
506+
if (plan.backend === "routed") {
507+
if (!plan.routedModel || !plan.routedConfig) return { text: "", error: "routed vision sidecar is unavailable" };
508+
return describeImageRouted(
509+
job.imageUrl,
510+
job.detail,
511+
job.contextText,
512+
plan.routedModel,
513+
plan.routedConfig,
514+
plan.settings,
515+
abortSignal,
516+
);
517+
}
454518
if (plan.backend === "anthropic") {
455519
const sidecar = plan.anthropicSidecar;
456520
if (!sidecar) return { text: "", error: "anthropic vision sidecar is unavailable" };

‎src/vision/routed-describe.ts‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
/**
2+
* Describe ONE image via a ROUTED model through the proxy's own
3+
* /v1/chat/completions on loopback (#2188 roadmap 180 revised).
4+
*
5+
* One executor for every provider the router can reach: the chat inbound
6+
* translates image_url parts and each adapter compiles its own wire
7+
* (Anthropic blocks, Antigravity inlineData, xai Responses input_image, plain
8+
* openai-chat), so provider coverage is the router's job, not this file's.
9+
*
10+
* Recursion fence: the request carries `x-opencodex-vision-describe: 1`.
11+
* The Chat surface detects the raw header before its bridge rebuilds headers
12+
* and carries it into handleResponses as `visionDescribeTerminal`; a marked
13+
* request STRIPS images instead of planning another describe (depth cap 1,
14+
* holds under predicate drift and combo re-resolution — audit rounds 2-4).
15+
*
16+
* Admission ladder (audit round 3): configuredApiAuthToken() (env token) ||
17+
* service token file || first config.apiKeys entry, sent as
18+
* `x-opencodex-api-key` — never Authorization (gateway-cache.ts rule: an
19+
* admission secret in a forwardable header is a forwarding hazard). Loopback
20+
* binds require no token at all (resolveApiAuth admits loopback).
21+
*
22+
* Known limitation (recorded in roadmap 170): a bindHost where 127.0.0.1
23+
* does not answer cannot reach its own loopback — same latent limitation
24+
* gateway-cache has.
25+
*/
26+
import type { OcxConfig } from "../types";
27+
import { signalWithTimeout, cancelBodyOnAbort } from "../lib/abort";
28+
import { redactSecretString } from "../lib/redact";
29+
import { sidecarEnter } from "../lib/sidecar-tracker";
30+
import { configuredApiAuthToken, configuredPort } from "../server/auth-cors";
31+
import { loadServiceTokenFromFile } from "../lib/service-secrets";
32+
import type { DescribeOutcome, VisionSettings } from "./describe";
33+
34+
export const VISION_DESCRIBE_TERMINAL_HEADER = "x-opencodex-vision-describe";
35+
36+
const ALLOWED_IMAGE_MIME = new Set(["image/png", "image/jpeg", "image/jpg", "image/webp", "image/gif"]);
37+
const MAX_IMAGE_BYTES = 20 * 1024 * 1024;
38+
/** Bound the loopback JSON response; descriptions are clamped to ~2k chars by the caller anyway. */
39+
const MAX_ROUTED_RESPONSE_BYTES = 4 * 1024 * 1024;
40+
41+
const DESCRIBE_INSTRUCTION =
42+
"You are a vision describer for a text-only model that cannot see the image. Describe the image "
43+
+ "thoroughly and factually so that model can fully reason about it: transcribe any visible text "
44+
+ "verbatim, and note UI/layout, colors, branding/logos, charts, and notable details. Focus on "
45+
+ "what's relevant to the user's request. Output only the description.";
46+
47+
function validateImageUrl(url: string): string | null {
48+
if (url.startsWith("data:")) {
49+
const match = /^data:([^;,]+?)(;base64)?,(.*)$/s.exec(url);
50+
if (!match) return "malformed data URL";
51+
const mime = match[1].toLowerCase();
52+
if (!ALLOWED_IMAGE_MIME.has(mime)) return `unsupported image type "${mime}"`;
53+
if (match[2]) {
54+
const bytes = Math.floor((match[3].length * 3) / 4);
55+
if (bytes > MAX_IMAGE_BYTES) return `image too large (~${Math.round(bytes / 1024 / 1024)}MB)`;
56+
}
57+
return null;
58+
}
59+
if (url.startsWith("https://")) return null;
60+
return "unsupported image URL scheme (expected data: or https:)";
61+
}
62+
63+
/** The admission ladder: env token, service token file, first configured API key. */
64+
export function routedDescribeAdmissionToken(config: Pick<OcxConfig, "apiKeys">): string | undefined {
65+
const envToken = configuredApiAuthToken();
66+
if (envToken) return envToken;
67+
const fileToken = loadServiceTokenFromFile(process.env);
68+
if (fileToken) return fileToken;
69+
const first = config.apiKeys?.[0]?.key?.trim();
70+
return first || undefined;
71+
}
72+
73+
/** Base URL seam for tests; production always self-fetches loopback. */
74+
export function routedDescribeBaseUrl(config: Pick<OcxConfig, "port">): string {
75+
// config.port can be 0 (ephemeral bind, tests) or stale after a live port
76+
// override; the server records its ACTUAL bound port via setCorsOrigin at
77+
// startup, so prefer that when config carries no positive port.
78+
const port = config.port && config.port > 0 ? String(config.port) : configuredPort();
79+
return `http://127.0.0.1:${port}`;
80+
}
81+
82+
export async function describeImageRouted(
83+
imageUrl: string,
84+
_detail: string | undefined,
85+
contextText: string,
86+
routedModel: string,
87+
config: Pick<OcxConfig, "port" | "apiKeys">,
88+
settings: VisionSettings,
89+
abortSignal?: AbortSignal,
90+
baseUrlOverride?: string,
91+
): Promise<DescribeOutcome> {
92+
const invalid = validateImageUrl(imageUrl);
93+
if (invalid) return { text: "", error: invalid };
94+
95+
const headers: Record<string, string> = {
96+
"Content-Type": "application/json",
97+
[VISION_DESCRIBE_TERMINAL_HEADER]: "1",
98+
};
99+
const admission = routedDescribeAdmissionToken(config);
100+
if (admission) headers["x-opencodex-api-key"] = admission;
101+
102+
const requestBody = {
103+
model: routedModel,
104+
stream: false,
105+
messages: [
106+
{ role: "system", content: DESCRIBE_INSTRUCTION },
107+
{
108+
role: "user",
109+
content: [
110+
...(contextText ? [{ type: "text", text: `User's request context: ${contextText}` }] : []),
111+
{ type: "image_url", image_url: { url: imageUrl } },
112+
],
113+
},
114+
],
115+
};
116+
117+
const linkedSignal = signalWithTimeout(settings.timeoutMs, abortSignal);
118+
const sidecarExit = sidecarEnter("vision");
119+
const t0 = Date.now();
120+
try {
121+
const res = await fetch(`${baseUrlOverride ?? routedDescribeBaseUrl(config)}/v1/chat/completions`, {
122+
method: "POST",
123+
headers,
124+
body: JSON.stringify(requestBody),
125+
signal: linkedSignal.signal,
126+
redirect: "manual",
127+
});
128+
const detachBodyGuard = cancelBodyOnAbort(res.body, linkedSignal.signal);
129+
try {
130+
const raw = await res.text();
131+
if (raw.length > MAX_ROUTED_RESPONSE_BYTES) {
132+
return { text: "", error: "routed describe response exceeded byte bound" };
133+
}
134+
if (!res.ok) {
135+
return { text: "", error: `routed describe HTTP ${res.status}: ${redactSecretString(raw.slice(0, 200))}` };
136+
}
137+
let payload: unknown;
138+
try { payload = JSON.parse(raw); } catch {
139+
return { text: "", error: "routed describe returned non-JSON" };
140+
}
141+
const content = extractChatContent(payload);
142+
if (!content) return { text: "", error: "routed describe returned no text" };
143+
return { text: content };
144+
} finally {
145+
detachBodyGuard();
146+
}
147+
} catch (e) {
148+
const kind = e instanceof Error && e.name === "TimeoutError" ? "timeout" : "connect_error";
149+
console.warn(`[vision] routed describe ${kind} (${Date.now() - t0}ms)`);
150+
return { text: "", error: redactSecretString(e instanceof Error ? e.message : String(e)) };
151+
} finally {
152+
sidecarExit();
153+
linkedSignal.cleanup();
154+
}
155+
}
156+
157+
function extractChatContent(payload: unknown): string | undefined {
158+
if (!payload || typeof payload !== "object") return undefined;
159+
const choices = (payload as { choices?: unknown }).choices;
160+
if (!Array.isArray(choices) || choices.length === 0) return undefined;
161+
const message = (choices[0] as { message?: unknown })?.message;
162+
if (!message || typeof message !== "object") return undefined;
163+
const content = (message as { content?: unknown }).content;
164+
if (typeof content === "string" && content.trim().length > 0) return content;
165+
// Some adapters emit content parts; join text parts.
166+
if (Array.isArray(content)) {
167+
const joined = content
168+
.map(part => (part && typeof part === "object" && typeof (part as { text?: unknown }).text === "string"
169+
? (part as { text: string }).text
170+
: ""))
171+
.join("");
172+
if (joined.trim().length > 0) return joined;
173+
}
174+
return undefined;
175+
}

0 commit comments

Comments
 (0)