Skip to content

Commit f44619a

Browse files
yansigitcursoragent
andcommitted
fix(cursor): harden vision prep for Wibias round-2 review
Require JPEG SOF before soft-cap skip, sniff WebP dims before Bun decode, scope prepare/fetch to the active vision window, and honor the image-phase AbortSignal across local encode work. Co-authored-by: Cursor <[email protected]>
1 parent 2dcd4d1 commit f44619a

2 files changed

Lines changed: 231 additions & 17 deletions

File tree

‎src/adapters/cursor/images.ts‎

Lines changed: 121 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -281,13 +281,51 @@ async function fetchHttpsImageBytes(url: string, signal?: AbortSignal): Promise<
281281
return { data, mimeType };
282282
} catch (error) {
283283
if (error instanceof CursorImageError) throw error;
284+
// Preserve AbortError so the image-phase deadline is not soft-omitted as a fetch failure.
285+
if (signal?.aborted || (error instanceof Error && error.name === "AbortError")) throw error;
284286
throw new CursorImageError("Could not fetch the image URL.");
285287
} finally {
286288
clearTimeout(timer);
287289
signal?.removeEventListener("abort", onAbort);
288290
}
289291
}
290292

293+
function throwIfImagePhaseAborted(signal?: AbortSignal): void {
294+
if (!signal?.aborted) return;
295+
if (signal.reason instanceof Error) throw signal.reason;
296+
const err = new Error("Cursor image phase aborted");
297+
err.name = "AbortError";
298+
throw err;
299+
}
300+
301+
/** Magic-byte format sniff (independent of declared MIME). */
302+
export function sniffCursorImageFormat(
303+
data: Uint8Array,
304+
): "png" | "jpeg" | "gif" | "webp" | undefined {
305+
if (
306+
data.byteLength >= 8
307+
&& data[0] === 0x89 && data[1] === 0x50 && data[2] === 0x4e && data[3] === 0x47
308+
&& data[4] === 0x0d && data[5] === 0x0a && data[6] === 0x1a && data[7] === 0x0a
309+
) {
310+
return "png";
311+
}
312+
if (
313+
data.byteLength >= 6
314+
&& data[0] === 0x47 && data[1] === 0x49 && data[2] === 0x46 && data[3] === 0x38
315+
) {
316+
return "gif";
317+
}
318+
if (data.byteLength >= 4 && data[0] === 0xff && data[1] === 0xd8) return "jpeg";
319+
if (
320+
data.byteLength >= 12
321+
&& data[0] === 0x52 && data[1] === 0x49 && data[2] === 0x46 && data[3] === 0x46
322+
&& data[8] === 0x57 && data[9] === 0x45 && data[10] === 0x42 && data[11] === 0x50
323+
) {
324+
return "webp";
325+
}
326+
return undefined;
327+
}
328+
291329
/** Collect image URLs from one message's content parts, preserving order. */
292330
export function extractCursorImageUrls(content: string | readonly OcxContentPart[]): string[] {
293331
return extractCursorImageParts(content).map(part => part.imageUrl);
@@ -331,6 +369,7 @@ export async function resolveCursorImages(
331369

332370
const out: ResolvedCursorImage[] = [];
333371
for (let i = 0; i < imageUrls.length; i++) {
372+
throwIfImagePhaseAborted(signal);
334373
const url = imageUrls[i];
335374
if (typeof url !== "string" || url.length === 0) {
336375
// Soft-omit missing URLs rather than aborting a mixed turn.
@@ -346,13 +385,13 @@ export async function resolveCursorImages(
346385
mimeType: resolved.mimeType,
347386
uuid: randomUUID(),
348387
...(options?.details?.[i] ? { detail: options.details[i] } : {}),
349-
});
388+
}, signal);
350389
if (outcome.status === "omitted") continue;
351390
if (outcome.image.data.byteLength > MAX_CURSOR_IMAGE_BYTES) continue;
352391
out.push(outcome.image);
353392
} catch (err) {
393+
if (signal?.aborted || (err instanceof Error && err.name === "AbortError")) throw err;
354394
if (err instanceof CursorImageError) continue;
355-
if (signal?.aborted) throw err;
356395
continue;
357396
}
358397
}
@@ -389,7 +428,9 @@ export function cursorImageAttachmentPath(uuid: string, mimeType: string): strin
389428
*/
390429
export async function prepareCursorImageForWire(
391430
image: ResolvedCursorImage,
431+
signal?: AbortSignal,
392432
): Promise<PrepareCursorImageOutcome> {
433+
throwIfImagePhaseAborted(signal);
393434
const mime = image.mimeType.toLowerCase();
394435
const softMax = softMaxBytesForDetail(image.detail);
395436
const qualities = jpegQualitiesForDetail(image.detail);
@@ -399,6 +440,7 @@ export async function prepareCursorImageForWire(
399440
return { status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED };
400441
}
401442

443+
const format = sniffCursorImageFormat(image.data);
402444
// Peek headers before Bun.Image so huge compressed bombs fail closed cheaply.
403445
const sniffed = sniffCursorImageDimensions(image.data);
404446
if (sniffed) {
@@ -409,24 +451,27 @@ export async function prepareCursorImageForWire(
409451
}
410452
}
411453

412-
// Second-pass skip: already soft-capped JPEG from prepareCursorRawMessages.
413-
const alreadySmallJpeg = (mime === "image/jpeg" || mime === "image/jpg")
414-
&& image.data.byteLength <= softMax
415-
&& image.data.byteLength >= 4
416-
&& image.data[0] === 0xff
417-
&& image.data[1] === 0xd8;
454+
// Second-pass skip: already soft-capped JPEG that has a real SOF (not SOI-only junk).
455+
const declaredJpeg = mime === "image/jpeg" || mime === "image/jpg";
456+
const alreadySmallJpeg = declaredJpeg
457+
&& format === "jpeg"
458+
&& sniffed !== undefined
459+
&& image.data.byteLength <= softMax;
418460
if (alreadySmallJpeg) {
419461
return { status: "ready", image };
420462
}
421463

422464
try {
465+
throwIfImagePhaseAborted(signal);
423466
// Force a full decode before accepting passthrough / encode (Anthropic-style validate).
424467
await new Bun.Image(image.data).resize(1, 1).jpeg({ quality: 1 }).toBuffer();
425468

426-
if ((mime === "image/jpeg" || mime === "image/jpg") && image.data.byteLength <= softMax) {
469+
// Passthrough only when declared MIME matches actual JPEG magic (never PNG-as-JPEG).
470+
if (declaredJpeg && format === "jpeg" && image.data.byteLength <= softMax) {
427471
return { status: "ready", image };
428472
}
429473

474+
throwIfImagePhaseAborted(signal);
430475
const meta = await new Bun.Image(image.data).metadata();
431476
const width = typeof meta.width === "number" ? meta.width : 0;
432477
const height = typeof meta.height === "number" ? meta.height : 0;
@@ -445,6 +490,7 @@ export async function prepareCursorImageForWire(
445490
}
446491

447492
const encodeAt = async (w: number, h: number, quality: number): Promise<Uint8Array> => {
493+
throwIfImagePhaseAborted(signal);
448494
let pipeline = new Bun.Image(image.data);
449495
if (w > 0 && h > 0 && (w !== width || h !== height)) {
450496
pipeline = pipeline.resize(w, h);
@@ -472,6 +518,7 @@ export async function prepareCursorImageForWire(
472518
&& targetH > 0
473519
&& Math.max(targetW, targetH) > CURSOR_VISION_SOFT_MIN_EDGE
474520
) {
521+
throwIfImagePhaseAborted(signal);
475522
const nextW = Math.max(1, Math.round(targetW * CURSOR_VISION_SOFT_SHRINK));
476523
const nextH = Math.max(1, Math.round(targetH * CURSOR_VISION_SOFT_SHRINK));
477524
if (Math.max(nextW, nextH) < CURSOR_VISION_SOFT_MIN_EDGE) {
@@ -499,14 +546,19 @@ export async function prepareCursorImageForWire(
499546
image: { ...image, data: best, mimeType: "image/jpeg" },
500547
};
501548
}
549+
// Undeclared/mismatched magic with no encode result — omit rather than lie about MIME.
550+
if (declaredJpeg && format !== "jpeg") {
551+
return { status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED };
552+
}
502553
return { status: "ready", image };
503-
} catch {
554+
} catch (err) {
555+
if (signal?.aborted || (err instanceof Error && err.name === "AbortError")) throw err;
504556
return { status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED };
505557
}
506558
}
507559

508560
/**
509-
* Sniff PNG/JPEG/GIF dimensions from raw bytes when the header is present.
561+
* Sniff PNG/JPEG/GIF/WebP dimensions from raw bytes when the header is present.
510562
* Best-effort only — unknown formats return undefined (dimension is optional).
511563
*/
512564
export function sniffCursorImageDimensions(
@@ -531,6 +583,30 @@ export function sniffCursorImageDimensions(
531583
const height = data[8]! | (data[9]! << 8);
532584
if (width > 0 && height > 0) return { width, height };
533585
}
586+
// WebP: RIFF....WEBP + VP8X / VP8 / VP8L (same layout as anthropic-image-guard).
587+
if (
588+
data.byteLength >= 30
589+
&& data[0] === 0x52 && data[1] === 0x49 && data[2] === 0x46 && data[3] === 0x46
590+
&& data[8] === 0x57 && data[9] === 0x45 && data[10] === 0x42 && data[11] === 0x50
591+
) {
592+
const fourcc = String.fromCharCode(data[12]!, data[13]!, data[14]!, data[15]!);
593+
if (fourcc === "VP8X") {
594+
const width = 1 + (data[24]! | (data[25]! << 8) | (data[26]! << 16));
595+
const height = 1 + (data[27]! | (data[28]! << 8) | (data[29]! << 16));
596+
if (width > 0 && height > 0) return { width, height };
597+
} else if (fourcc === "VP8 ") {
598+
if (data[23] === 0x9d && data[24] === 0x01 && data[25] === 0x2a) {
599+
const width = (data[26]! | (data[27]! << 8)) & 0x3fff;
600+
const height = (data[28]! | (data[29]! << 8)) & 0x3fff;
601+
if (width > 0 && height > 0) return { width, height };
602+
}
603+
} else if (fourcc === "VP8L" && data[20] === 0x2f) {
604+
const raw = data[21]! | (data[22]! << 8) | (data[23]! << 16) | (data[24]! << 24);
605+
const width = (raw & 0x3fff) + 1;
606+
const height = ((raw >> 14) & 0x3fff) + 1;
607+
if (width > 0 && height > 0) return { width, height };
608+
}
609+
}
534610
// JPEG: scan for SOF0/SOF2 marker with dimensions
535611
if (data.byteLength >= 4 && data[0] === 0xff && data[1] === 0xd8) {
536612
let offset = 2;
@@ -755,7 +831,7 @@ export async function prepareCursorImageDataUrl(
755831
mimeType: resolved.mimeType,
756832
uuid: randomUUID(),
757833
...(detail ? { detail } : {}),
758-
});
834+
}, signal);
759835
if (outcome.status === "omitted") return outcome;
760836
if (outcome.image.data.byteLength > MAX_CURSOR_IMAGE_BYTES) {
761837
return { status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED };
@@ -769,7 +845,7 @@ export async function prepareCursorImageDataUrl(
769845
}
770846
return { status: "ready", imageUrl: imageDataUrlFromPrepared(outcome.image) };
771847
} catch (err) {
772-
if (signal?.aborted) throw err;
848+
if (signal?.aborted || (err instanceof Error && err.name === "AbortError")) throw err;
773849
return { status: "omitted", reason: CURSOR_VISION_IMAGE_OMITTED };
774850
}
775851
}
@@ -783,6 +859,7 @@ async function prepareCursorContentParts(
783859
const next: OcxContentPart[] = [];
784860
for (const part of content) {
785861
if (part.type === "image" && typeof part.imageUrl === "string" && part.imageUrl.length > 0) {
862+
throwIfImagePhaseAborted(signal);
786863
const prepared = await prepareCursorImageDataUrl(part.imageUrl, part.detail, signal);
787864
if (prepared.status === "omitted") {
788865
changed = true;
@@ -799,19 +876,46 @@ async function prepareCursorContentParts(
799876
}
800877

801878
/**
802-
* Rewrite image URLs in user/developer/toolResult messages (including `view_image`
803-
* tool outputs) through the JPEG soft-cap path before protobuf encode. HTTPS failures
879+
* First original-message index that still needs image prep for the active vision window.
880+
* Historical messages before this index are left untouched (no HTTPS, no decode).
881+
* Indices map 1:1 into `messages` because transparent developers are only stripped from the end.
882+
*/
883+
export function cursorVisionPrepareStartIndex(messages: readonly OcxMessage[]): number {
884+
const effective = stripTrailingTransparentDeveloperMessages(messages);
885+
if (effective.length === 0) return messages.length;
886+
if (effective.at(-1)?.role === "toolResult") {
887+
let start = effective.length;
888+
while (start > 0 && effective[start - 1]?.role === "toolResult") start -= 1;
889+
return start;
890+
}
891+
for (let i = effective.length - 1; i >= 0; i--) {
892+
const role = effective[i]?.role;
893+
if (role === "user" || role === "developer") return i;
894+
}
895+
return effective.length;
896+
}
897+
898+
/**
899+
* Rewrite image URLs in the active vision window (last user/developer, or trailing
900+
* toolResult block) through the JPEG soft-cap path before protobuf encode. Historical
901+
* messages are left by reference — encode already omits their MCP pixels. HTTPS failures
804902
* become {@link CURSOR_VISION_IMAGE_OMITTED} text so image-only turns stay userMessageAction.
805903
*/
806904
export async function prepareCursorRawMessages(
807905
messages: readonly OcxMessage[] | undefined,
808906
signal?: AbortSignal,
809907
): Promise<readonly OcxMessage[] | undefined> {
810908
if (!messages?.length) return messages;
909+
const prepareFrom = cursorVisionPrepareStartIndex(messages);
811910
let changed = false;
812911
const out: OcxMessage[] = [];
813-
for (const message of messages) {
814-
if (message.role === "user" || message.role === "developer" || message.role === "toolResult") {
912+
for (let i = 0; i < messages.length; i++) {
913+
throwIfImagePhaseAborted(signal);
914+
const message = messages[i]!;
915+
if (
916+
i >= prepareFrom
917+
&& (message.role === "user" || message.role === "developer" || message.role === "toolResult")
918+
) {
815919
const content = await prepareCursorContentParts(message.content, signal);
816920
if (content !== message.content) {
817921
changed = true;

0 commit comments

Comments
 (0)