|
| 1 | +/** |
| 2 | + * Describe ONE image via a ROUTED model through the proxy's own |
| 3 | + * /v1/chat/completions on loopback (#2188 roadmap 180 revised). |
| 4 | + * |
| 5 | + * One executor for every provider the router can reach: the chat inbound |
| 6 | + * translates image_url parts and each adapter compiles its own wire |
| 7 | + * (Anthropic blocks, Antigravity inlineData, xai Responses input_image, plain |
| 8 | + * openai-chat), so provider coverage is the router's job, not this file's. |
| 9 | + * |
| 10 | + * Recursion fence: the request carries `x-opencodex-vision-describe: 1`. |
| 11 | + * The Chat surface detects the raw header before its bridge rebuilds headers |
| 12 | + * and carries it into handleResponses as `visionDescribeTerminal`; a marked |
| 13 | + * request STRIPS images instead of planning another describe (depth cap 1, |
| 14 | + * holds under predicate drift and combo re-resolution — audit rounds 2-4). |
| 15 | + * |
| 16 | + * Admission ladder (audit round 3): configuredApiAuthToken() (env token) || |
| 17 | + * service token file || first config.apiKeys entry, sent as |
| 18 | + * `x-opencodex-api-key` — never Authorization (gateway-cache.ts rule: an |
| 19 | + * admission secret in a forwardable header is a forwarding hazard). Loopback |
| 20 | + * binds require no token at all (resolveApiAuth admits loopback). |
| 21 | + * |
| 22 | + * Known limitation (recorded in roadmap 170): a bindHost where 127.0.0.1 |
| 23 | + * does not answer cannot reach its own loopback — same latent limitation |
| 24 | + * gateway-cache has. |
| 25 | + */ |
| 26 | +import type { OcxConfig } from "../types"; |
| 27 | +import { signalWithTimeout, cancelBodyOnAbort } from "../lib/abort"; |
| 28 | +import { redactSecretString } from "../lib/redact"; |
| 29 | +import { sidecarEnter } from "../lib/sidecar-tracker"; |
| 30 | +import { configuredApiAuthToken, configuredPort } from "../server/auth-cors"; |
| 31 | +import { loadServiceTokenFromFile } from "../lib/service-secrets"; |
| 32 | +import type { DescribeOutcome, VisionSettings } from "./describe"; |
| 33 | + |
| 34 | +export const VISION_DESCRIBE_TERMINAL_HEADER = "x-opencodex-vision-describe"; |
| 35 | + |
| 36 | +const ALLOWED_IMAGE_MIME = new Set(["image/png", "image/jpeg", "image/jpg", "image/webp", "image/gif"]); |
| 37 | +const MAX_IMAGE_BYTES = 20 * 1024 * 1024; |
| 38 | +/** Bound the loopback JSON response; descriptions are clamped to ~2k chars by the caller anyway. */ |
| 39 | +const MAX_ROUTED_RESPONSE_BYTES = 4 * 1024 * 1024; |
| 40 | + |
| 41 | +const DESCRIBE_INSTRUCTION = |
| 42 | + "You are a vision describer for a text-only model that cannot see the image. Describe the image " |
| 43 | + + "thoroughly and factually so that model can fully reason about it: transcribe any visible text " |
| 44 | + + "verbatim, and note UI/layout, colors, branding/logos, charts, and notable details. Focus on " |
| 45 | + + "what's relevant to the user's request. Output only the description."; |
| 46 | + |
| 47 | +function validateImageUrl(url: string): string | null { |
| 48 | + if (url.startsWith("data:")) { |
| 49 | + const match = /^data:([^;,]+?)(;base64)?,(.*)$/s.exec(url); |
| 50 | + if (!match) return "malformed data URL"; |
| 51 | + const mime = match[1].toLowerCase(); |
| 52 | + if (!ALLOWED_IMAGE_MIME.has(mime)) return `unsupported image type "${mime}"`; |
| 53 | + if (match[2]) { |
| 54 | + const bytes = Math.floor((match[3].length * 3) / 4); |
| 55 | + if (bytes > MAX_IMAGE_BYTES) return `image too large (~${Math.round(bytes / 1024 / 1024)}MB)`; |
| 56 | + } |
| 57 | + return null; |
| 58 | + } |
| 59 | + if (url.startsWith("https://")) return null; |
| 60 | + return "unsupported image URL scheme (expected data: or https:)"; |
| 61 | +} |
| 62 | + |
| 63 | +/** The admission ladder: env token, service token file, first configured API key. */ |
| 64 | +export function routedDescribeAdmissionToken(config: Pick<OcxConfig, "apiKeys">): string | undefined { |
| 65 | + const envToken = configuredApiAuthToken(); |
| 66 | + if (envToken) return envToken; |
| 67 | + const fileToken = loadServiceTokenFromFile(process.env); |
| 68 | + if (fileToken) return fileToken; |
| 69 | + const first = config.apiKeys?.[0]?.key?.trim(); |
| 70 | + return first || undefined; |
| 71 | +} |
| 72 | + |
| 73 | +/** Base URL seam for tests; production always self-fetches loopback. */ |
| 74 | +export function routedDescribeBaseUrl(config: Pick<OcxConfig, "port">): string { |
| 75 | + // config.port can be 0 (ephemeral bind, tests) or stale after a live port |
| 76 | + // override; the server records its ACTUAL bound port via setCorsOrigin at |
| 77 | + // startup, so prefer that when config carries no positive port. |
| 78 | + const port = config.port && config.port > 0 ? String(config.port) : configuredPort(); |
| 79 | + return `http://127.0.0.1:${port}`; |
| 80 | +} |
| 81 | + |
| 82 | +export async function describeImageRouted( |
| 83 | + imageUrl: string, |
| 84 | + _detail: string | undefined, |
| 85 | + contextText: string, |
| 86 | + routedModel: string, |
| 87 | + config: Pick<OcxConfig, "port" | "apiKeys">, |
| 88 | + settings: VisionSettings, |
| 89 | + abortSignal?: AbortSignal, |
| 90 | + baseUrlOverride?: string, |
| 91 | +): Promise<DescribeOutcome> { |
| 92 | + const invalid = validateImageUrl(imageUrl); |
| 93 | + if (invalid) return { text: "", error: invalid }; |
| 94 | + |
| 95 | + const headers: Record<string, string> = { |
| 96 | + "Content-Type": "application/json", |
| 97 | + [VISION_DESCRIBE_TERMINAL_HEADER]: "1", |
| 98 | + }; |
| 99 | + const admission = routedDescribeAdmissionToken(config); |
| 100 | + if (admission) headers["x-opencodex-api-key"] = admission; |
| 101 | + |
| 102 | + const requestBody = { |
| 103 | + model: routedModel, |
| 104 | + stream: false, |
| 105 | + messages: [ |
| 106 | + { role: "system", content: DESCRIBE_INSTRUCTION }, |
| 107 | + { |
| 108 | + role: "user", |
| 109 | + content: [ |
| 110 | + ...(contextText ? [{ type: "text", text: `User's request context: ${contextText}` }] : []), |
| 111 | + { type: "image_url", image_url: { url: imageUrl } }, |
| 112 | + ], |
| 113 | + }, |
| 114 | + ], |
| 115 | + }; |
| 116 | + |
| 117 | + const linkedSignal = signalWithTimeout(settings.timeoutMs, abortSignal); |
| 118 | + const sidecarExit = sidecarEnter("vision"); |
| 119 | + const t0 = Date.now(); |
| 120 | + try { |
| 121 | + const res = await fetch(`${baseUrlOverride ?? routedDescribeBaseUrl(config)}/v1/chat/completions`, { |
| 122 | + method: "POST", |
| 123 | + headers, |
| 124 | + body: JSON.stringify(requestBody), |
| 125 | + signal: linkedSignal.signal, |
| 126 | + redirect: "manual", |
| 127 | + }); |
| 128 | + const detachBodyGuard = cancelBodyOnAbort(res.body, linkedSignal.signal); |
| 129 | + try { |
| 130 | + const raw = await res.text(); |
| 131 | + if (raw.length > MAX_ROUTED_RESPONSE_BYTES) { |
| 132 | + return { text: "", error: "routed describe response exceeded byte bound" }; |
| 133 | + } |
| 134 | + if (!res.ok) { |
| 135 | + return { text: "", error: `routed describe HTTP ${res.status}: ${redactSecretString(raw.slice(0, 200))}` }; |
| 136 | + } |
| 137 | + let payload: unknown; |
| 138 | + try { payload = JSON.parse(raw); } catch { |
| 139 | + return { text: "", error: "routed describe returned non-JSON" }; |
| 140 | + } |
| 141 | + const content = extractChatContent(payload); |
| 142 | + if (!content) return { text: "", error: "routed describe returned no text" }; |
| 143 | + return { text: content }; |
| 144 | + } finally { |
| 145 | + detachBodyGuard(); |
| 146 | + } |
| 147 | + } catch (e) { |
| 148 | + const kind = e instanceof Error && e.name === "TimeoutError" ? "timeout" : "connect_error"; |
| 149 | + console.warn(`[vision] routed describe ${kind} (${Date.now() - t0}ms)`); |
| 150 | + return { text: "", error: redactSecretString(e instanceof Error ? e.message : String(e)) }; |
| 151 | + } finally { |
| 152 | + sidecarExit(); |
| 153 | + linkedSignal.cleanup(); |
| 154 | + } |
| 155 | +} |
| 156 | + |
| 157 | +function extractChatContent(payload: unknown): string | undefined { |
| 158 | + if (!payload || typeof payload !== "object") return undefined; |
| 159 | + const choices = (payload as { choices?: unknown }).choices; |
| 160 | + if (!Array.isArray(choices) || choices.length === 0) return undefined; |
| 161 | + const message = (choices[0] as { message?: unknown })?.message; |
| 162 | + if (!message || typeof message !== "object") return undefined; |
| 163 | + const content = (message as { content?: unknown }).content; |
| 164 | + if (typeof content === "string" && content.trim().length > 0) return content; |
| 165 | + // Some adapters emit content parts; join text parts. |
| 166 | + if (Array.isArray(content)) { |
| 167 | + const joined = content |
| 168 | + .map(part => (part && typeof part === "object" && typeof (part as { text?: unknown }).text === "string" |
| 169 | + ? (part as { text: string }).text |
| 170 | + : "")) |
| 171 | + .join(""); |
| 172 | + if (joined.trim().length > 0) return joined; |
| 173 | + } |
| 174 | + return undefined; |
| 175 | +} |
0 commit comments