Skip to content

Commit dfc6079

Browse files
committed
fix(cli): report bounded vision failures
1 parent 22e3cca commit dfc6079

4 files changed

Lines changed: 218 additions & 52 deletions

File tree

‎packages/cli/src/capture/contentExtractor.test.ts‎

Lines changed: 148 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -86,17 +86,35 @@ describe("captionImagesWithGemini — OpenRouter provider", () => {
8686
vi.stubGlobal(
8787
"fetch",
8888
vi.fn(
89-
async () => new Response("invalid api key", { status: 401, statusText: "Unauthorized" }),
89+
async () =>
90+
new Response("provider detail containing or-bad-key", {
91+
status: 401,
92+
statusText: "Unauthorized",
93+
}),
9094
),
9195
);
9296

9397
const warnings: string[] = [];
94-
// captionOne throws on !res.ok, but the throw is per-image inside
95-
// Promise.allSettled, so it's filtered out as a rejected result rather than
96-
// bubbling up — same silent degradation as the existing Gemini path.
97-
const captions = await captionImagesWithGemini(dir, () => {}, warnings);
98+
let outcome: VisionCaptionOutcome | undefined;
99+
const captions = await captionImagesWithGemini(dir, () => {}, warnings, {
100+
onOutcome: (value) => {
101+
outcome = value;
102+
},
103+
});
98104

99105
expect(captions).toEqual({});
106+
expect(warnings).toEqual(["OpenRouter vision failed for 1 asset(s); captions omitted."]);
107+
expect(warnings.join(" ")).not.toContain("or-bad-key");
108+
expect(outcome).toEqual({
109+
timedOutRequests: 0,
110+
failedRequests: 1,
111+
budgetExhausted: false,
112+
});
113+
if (!outcome) throw new Error("Expected vision caption outcome");
114+
expect(resolveVisionPhaseCompletion(outcome, 10_000)).toEqual({
115+
status: "degraded",
116+
reason: "provider-error",
117+
});
100118
});
101119

102120
it("terminates captioning when the vision provider never responds", async () => {
@@ -125,6 +143,94 @@ describe("captionImagesWithGemini — OpenRouter provider", () => {
125143
expect(result).toEqual({});
126144
});
127145

146+
it.each([
147+
{ label: "success JSON", status: 200, partialBody: '{"choices":[' },
148+
{ label: "failure text", status: 500, partialBody: "upstream failed: " },
149+
])(
150+
"terminates captioning when OpenRouter $label body never ends",
151+
async ({ status, partialBody }) => {
152+
const dir = makeProjectWithImages();
153+
dirs.push(dir);
154+
vi.stubEnv("OPENROUTER_API_KEY", "or-stalled-body-key");
155+
vi.stubEnv("HYPERFRAMES_VISION_TIMEOUT_MS", "20");
156+
157+
vi.stubGlobal(
158+
"fetch",
159+
vi.fn(async () => {
160+
const body = new ReadableStream<Uint8Array>({
161+
start(controller) {
162+
controller.enqueue(new TextEncoder().encode(partialBody));
163+
},
164+
});
165+
return new Response(body, {
166+
status,
167+
headers: { "content-type": "application/json" },
168+
});
169+
}),
170+
);
171+
172+
const warnings: string[] = [];
173+
let outcome: VisionCaptionOutcome | undefined;
174+
const result = await Promise.race([
175+
captionImagesWithGemini(dir, () => {}, warnings, {
176+
onOutcome: (value) => {
177+
outcome = value;
178+
},
179+
}),
180+
new Promise<"hung">((resolve) => setTimeout(() => resolve("hung"), 250)),
181+
]);
182+
183+
expect(result).toEqual({});
184+
expect(warnings).toEqual(["OpenRouter vision timed out for 1 asset(s); captions omitted."]);
185+
expect(outcome).toEqual({
186+
timedOutRequests: 1,
187+
failedRequests: 0,
188+
budgetExhausted: false,
189+
});
190+
},
191+
);
192+
193+
it("keeps successful captions while reporting a failed sibling request", async () => {
194+
const dir = makeProjectWithImages(["failed.png", "success.png"]);
195+
dirs.push(dir);
196+
vi.stubEnv("OPENROUTER_API_KEY", "or-mixed-failure-key");
197+
198+
let requestCount = 0;
199+
vi.stubGlobal(
200+
"fetch",
201+
vi.fn(async () => {
202+
requestCount++;
203+
if (requestCount === 1) {
204+
return new Response("upstream exploded with or-mixed-failure-key", {
205+
status: 500,
206+
statusText: "Internal Server Error",
207+
});
208+
}
209+
return new Response(
210+
JSON.stringify({ choices: [{ message: { content: "Successful image." } }] }),
211+
{ status: 200, headers: { "content-type": "application/json" } },
212+
);
213+
}),
214+
);
215+
216+
const warnings: string[] = [];
217+
let outcome: VisionCaptionOutcome | undefined;
218+
const captions = await captionImagesWithGemini(dir, () => {}, warnings, {
219+
onOutcome: (value) => {
220+
outcome = value;
221+
},
222+
});
223+
224+
expect(captions).toEqual({ "success.png": "Successful image." });
225+
expect(warnings).toEqual(["OpenRouter vision failed for 1 asset(s); captions omitted."]);
226+
expect(warnings.join(" ")).not.toContain("or-mixed-failure-key");
227+
expect(outcome).toEqual({
228+
timedOutRequests: 0,
229+
failedRequests: 1,
230+
budgetExhausted: false,
231+
});
232+
});
233+
128234
it("keeps a successful sibling caption when another request times out", async () => {
129235
const dir = makeProjectWithImages(["hang.png", "success.png"]);
130236
dirs.push(dir);
@@ -163,7 +269,11 @@ describe("captionImagesWithGemini — OpenRouter provider", () => {
163269
});
164270

165271
expect(captions).toEqual({ "success.png": "Successful image." });
166-
expect(outcome).toEqual({ timedOutRequests: 1, budgetExhausted: false });
272+
expect(outcome).toEqual({
273+
timedOutRequests: 1,
274+
failedRequests: 0,
275+
budgetExhausted: false,
276+
});
167277
if (!outcome) throw new Error("Expected vision caption outcome");
168278
expect(resolveVisionPhaseCompletion(outcome, 10_000)).toEqual({
169279
status: "degraded",
@@ -242,4 +352,36 @@ describe("captionImagesWithGemini — Gemini provider", () => {
242352

243353
expect(result).toEqual({});
244354
});
355+
356+
it("reports a rejected Gemini request without leaking its error detail", async () => {
357+
const dir = makeProjectWithImages();
358+
dirs.push(dir);
359+
vi.stubEnv("OPENROUTER_API_KEY", "");
360+
vi.stubEnv("GEMINI_API_KEY", "gemini-secret-key");
361+
generateContentMock.mockRejectedValue(
362+
new Error("provider echoed gemini-secret-key in its response"),
363+
);
364+
365+
const warnings: string[] = [];
366+
let outcome: VisionCaptionOutcome | undefined;
367+
const captions = await captionImagesWithGemini(dir, () => {}, warnings, {
368+
onOutcome: (value) => {
369+
outcome = value;
370+
},
371+
});
372+
373+
expect(captions).toEqual({});
374+
expect(warnings).toEqual(["Gemini vision failed for 1 asset(s); captions omitted."]);
375+
expect(warnings.join(" ")).not.toContain("gemini-secret-key");
376+
expect(outcome).toEqual({
377+
timedOutRequests: 0,
378+
failedRequests: 1,
379+
budgetExhausted: false,
380+
});
381+
if (!outcome) throw new Error("Expected vision caption outcome");
382+
expect(resolveVisionPhaseCompletion(outcome, 10_000)).toEqual({
383+
status: "degraded",
384+
reason: "provider-error",
385+
});
386+
});
245387
});

‎packages/cli/src/capture/contentExtractor.ts‎

Lines changed: 68 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ const DEFAULT_VISION_REQUEST_TIMEOUT_MS = 30_000;
1919

2020
export interface VisionCaptionOutcome {
2121
timedOutRequests: number;
22+
failedRequests: number;
2223
budgetExhausted: boolean;
2324
}
2425

@@ -33,13 +34,19 @@ export function resolveVisionPhaseCompletion(
3334
remainingMs: number,
3435
):
3536
| { status: "completed" }
36-
| { status: "degraded"; reason: "budget-exhausted" | "request-timeout" } {
37+
| {
38+
status: "degraded";
39+
reason: "budget-exhausted" | "request-timeout" | "provider-error";
40+
} {
3741
if (outcome.budgetExhausted || remainingMs <= 0) {
3842
return { status: "degraded", reason: "budget-exhausted" };
3943
}
4044
if (outcome.timedOutRequests > 0) {
4145
return { status: "degraded", reason: "request-timeout" };
4246
}
47+
if (outcome.failedRequests > 0) {
48+
return { status: "degraded", reason: "provider-error" };
49+
}
4350
return { status: "completed" };
4451
}
4552

@@ -236,9 +243,14 @@ export async function captionImagesWithGemini(
236243
): Promise<Record<string, string>> {
237244
const geminiCaptions: Record<string, string> = {};
238245
let timedOutCount = 0;
246+
let failedRequestCount = 0;
239247
let budgetExhausted = false;
240248
const reportOutcome = (): void => {
241-
options.onOutcome?.({ timedOutRequests: timedOutCount, budgetExhausted });
249+
options.onOutcome?.({
250+
timedOutRequests: timedOutCount,
251+
failedRequests: failedRequestCount,
252+
budgetExhausted,
253+
});
242254
};
243255
if (options.skipVision) {
244256
reportOutcome();
@@ -279,42 +291,40 @@ export async function captionImagesWithGemini(
279291
let captionOne: CaptionOne;
280292
if (openRouterKey) {
281293
captionOne = async ({ mimeType, base64, prompt, maxTokens, timeoutMs }) => {
282-
const res = await runBoundedVisionRequest(
283-
(signal) =>
284-
fetch("https://openrouter.ai/api/v1/chat/completions", {
285-
method: "POST",
286-
headers: {
287-
Authorization: `Bearer ${openRouterKey}`,
288-
"Content-Type": "application/json",
289-
},
290-
signal,
291-
body: JSON.stringify({
292-
model,
293-
messages: [
294-
{
295-
role: "user",
296-
content: [
297-
{ type: "text", text: prompt },
298-
{
299-
type: "image_url",
300-
image_url: { url: `data:${mimeType};base64,${base64}` },
301-
},
302-
],
303-
},
304-
],
305-
max_tokens: maxTokens,
306-
}),
294+
return runBoundedVisionRequest(async (signal) => {
295+
const res = await fetch("https://openrouter.ai/api/v1/chat/completions", {
296+
method: "POST",
297+
headers: {
298+
Authorization: `Bearer ${openRouterKey}`,
299+
"Content-Type": "application/json",
300+
},
301+
signal,
302+
body: JSON.stringify({
303+
model,
304+
messages: [
305+
{
306+
role: "user",
307+
content: [
308+
{ type: "text", text: prompt },
309+
{
310+
type: "image_url",
311+
image_url: { url: `data:${mimeType};base64,${base64}` },
312+
},
313+
],
314+
},
315+
],
316+
max_tokens: maxTokens,
307317
}),
308-
timeoutMs,
309-
);
310-
if (!res.ok) {
311-
const detail = await res.text().catch(() => "");
312-
throw new Error(`OpenRouter ${res.status} ${res.statusText}: ${detail.slice(0, 200)}`);
313-
}
314-
const data = (await res.json()) as {
315-
choices?: Array<{ message?: { content?: string } }>;
316-
};
317-
return data.choices?.[0]?.message?.content?.trim() || "";
318+
});
319+
if (!res.ok) {
320+
await res.text();
321+
throw new Error(`OpenRouter request failed with HTTP ${res.status}`);
322+
}
323+
const data = (await res.json()) as {
324+
choices?: Array<{ message?: { content?: string } }>;
325+
};
326+
return data.choices?.[0]?.message?.content?.trim() || "";
327+
}, timeoutMs);
318328
};
319329
} else {
320330
// Unreachable when geminiKey is unset (guarded above); re-narrow for TS.
@@ -350,13 +360,14 @@ export async function captionImagesWithGemini(
350360

351361
// Caption in parallel batches. Gemini free tier is ~5 RPM (slow but $0),
352362
// paid/OpenRouter ~2000 RPM. We batch 20 with a 2s inter-batch pause and rely
353-
// on Promise.allSettled so a rate-limited image degrades to "" rather than
354-
// failing the batch.
363+
// on Promise.allSettled so one rate-limited image does not fail its siblings;
364+
// rejected requests are aggregated into a sanitized warning below.
355365
const BATCH_SIZE = 20;
356366
const collectCaptionResults = (
357367
results: PromiseSettledResult<{ file: string; caption: string }>[],
358-
): number => {
368+
): { timeouts: number; failures: number } => {
359369
let timeouts = 0;
370+
let failures = 0;
360371
for (const result of results) {
361372
if (result.status === "fulfilled" && result.value.caption) {
362373
geminiCaptions[result.value.file] = result.value.caption;
@@ -365,9 +376,11 @@ export async function captionImagesWithGemini(
365376
result.reason instanceof VisionRequestTimeoutError
366377
) {
367378
timeouts++;
379+
} else if (result.status === "rejected") {
380+
failures++;
368381
}
369382
}
370-
return timeouts;
383+
return { timeouts, failures };
371384
};
372385
for (let i = 0; i < imageFiles.length; i += BATCH_SIZE) {
373386
const remainingMs = options.remainingMs?.() ?? Number.POSITIVE_INFINITY;
@@ -397,7 +410,9 @@ export async function captionImagesWithGemini(
397410
return { file, caption };
398411
}),
399412
);
400-
timedOutCount += collectCaptionResults(results);
413+
const counts = collectCaptionResults(results);
414+
timedOutCount += counts.timeouts;
415+
failedRequestCount += counts.failures;
401416
// Pace requests between batches (paid tier: 2000+ RPM, free tier: rate-limited)
402417
if (i + BATCH_SIZE < imageFiles.length) {
403418
await new Promise((r) => setTimeout(r, 2000)); // 2s pause between batches — paid tier handles 2000 RPM, free tier retries via Promise.allSettled
@@ -501,7 +516,9 @@ export async function captionImagesWithGemini(
501516
return { file: relPath, caption };
502517
}),
503518
);
504-
timedOutCount += collectCaptionResults(results);
519+
const counts = collectCaptionResults(results);
520+
timedOutCount += counts.timeouts;
521+
failedRequestCount += counts.failures;
505522
if (i + SVG_BATCH < svgFiles.length) {
506523
await new Promise((r) => setTimeout(r, 2000));
507524
}
@@ -523,8 +540,14 @@ export async function captionImagesWithGemini(
523540
`${providerName} vision timed out for ${timedOutCount} asset(s); captions omitted.`,
524541
);
525542
}
526-
} catch (err) {
527-
warnings.push(`${providerName} captioning failed: ${err}`);
543+
if (failedRequestCount > 0) {
544+
warnings.push(
545+
`${providerName} vision failed for ${failedRequestCount} asset(s); captions omitted.`,
546+
);
547+
}
548+
} catch {
549+
failedRequestCount = Math.max(1, failedRequestCount);
550+
warnings.push(`${providerName} captioning failed; captions omitted.`);
528551
}
529552

530553
reportOutcome();

‎packages/cli/src/capture/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -684,6 +684,7 @@ export async function captureWebsite(
684684
phase("vision", "started");
685685
let visionOutcome: VisionCaptionOutcome = {
686686
timedOutRequests: 0,
687+
failedRequests: 0,
687688
budgetExhausted: false,
688689
};
689690
geminiCaptions = await captionImagesWithGemini(outputDir, progress, warnings, {

‎packages/cli/src/capture/types.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ export interface CapturePhaseProgress {
2626
status: "started" | "completed" | "degraded";
2727
/** Null before the post-navigation budget begins. */
2828
remainingMs: number | null;
29-
reason?: "budget-exhausted" | "disabled" | "request-timeout";
29+
reason?: "budget-exhausted" | "disabled" | "request-timeout" | "provider-error";
3030
}
3131

3232
export interface CaptureOptions {

0 commit comments

Comments
 (0)