Skip to content

Commit adb0420

Browse files
t3dotggclaude
andauthored
fix(clients): large pasted images no longer break later Claude image reads (pingdotgg#19364)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 26c48bb commit adb0420

9 files changed

Lines changed: 421 additions & 70 deletions

File tree

‎apps/mobile/src/features/sharing/incoming-share-model.ts‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import * as Schema from "effect/Schema";
1212
import type { ResolvedSharePayload, SharePayload } from "expo-sharing";
1313

1414
import { DraftComposerAttachmentSchema } from "../../lib/composer-image-schema";
15+
import { fitComposerImage } from "../../lib/composerImageFit";
1516
import type { DraftComposerAttachment } from "../../lib/composerImages";
1617
import { estimateBase64ByteSize } from "../../lib/base64";
1718

@@ -406,8 +407,18 @@ export async function buildIncomingShareDraft(input: {
406407
}
407408

408409
try {
409-
const base64 = await input.fileReader.readBase64(uri);
410-
const sizeBytes = resolved?.contentSize ?? estimateBase64ByteSize(base64);
410+
const sharedBase64 = await input.fileReader.readBase64(uri);
411+
// Rendered from the bytes already read, since the provider's file may not be readable
412+
// by the native image loader.
413+
const fitted = await fitComposerImage({
414+
uri: `data:${mimeType};base64,${sharedBase64}`,
415+
mimeType,
416+
base64: sharedBase64,
417+
});
418+
const base64 = fitted?.base64 ?? sharedBase64;
419+
const sizeBytes = fitted
420+
? estimateBase64ByteSize(base64)
421+
: (resolved?.contentSize ?? estimateBase64ByteSize(base64));
411422
if (sizeBytes <= 0 || sizeBytes > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) {
412423
warnings.push(
413424
`'${resolved?.originalName ?? fallbackName(uri, index, mimeType)}' exceeds the 10 MB attachment limit.`,

‎apps/mobile/src/lib/composerFiles.test.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -185,10 +185,10 @@ describe("composer file attachments", () => {
185185
);
186186

187187
it.each([
188-
{ size: { width: 4032, height: 3024 }, resizes: [{ width: 2048 }] },
189-
{ size: { width: 3024, height: 4032 }, resizes: [{ height: 2048 }] },
190-
{ size: { width: 2048, height: 1536 }, resizes: [] },
191-
])("bounds a $size.width x $size.height photo to a 2048 px longest edge", async (input) => {
188+
{ size: { width: 4032, height: 3024 }, resizes: [{ width: 2000, height: 1500 }] },
189+
{ size: { width: 3024, height: 4032 }, resizes: [{ width: 1500, height: 2000 }] },
190+
{ size: { width: 2000, height: 1500 }, resizes: [] },
191+
])("bounds a $size.width x $size.height photo to a 2000 px longest edge", async (input) => {
192192
native.size = input.size;
193193
mocks.pickMedia.mockResolvedValue({ canceled: false, assets: [photo] });
194194

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
import {
2+
downscalableImageMimeType,
3+
fitImageAttachmentSize,
4+
} from "@t3tools/client-runtime/image-attachment-size";
5+
import {
6+
IMAGE_DIMENSIONS_HEADER_BYTES,
7+
readImageDimensions,
8+
} from "@t3tools/shared/imageDimensions";
9+
10+
const PHOTO_JPEG_QUALITY = 0.85;
11+
12+
/**
13+
* Renders an image to a JPEG, or a PNG when `format` asks for one, with its long edge capped at
14+
* `IMAGE_ATTACHMENT_MAX_EDGE`. Decode, downscale,
15+
* and encode run natively; only the bounded result crosses the bridge. Camera photos are
16+
* 12-48 MP HEIC files, so a full-size conversion is both slow to transfer and far more than a
17+
* model can use.
18+
*/
19+
export async function renderPhoto(
20+
uri: string,
21+
format: "jpeg" | "png" = "jpeg",
22+
): Promise<{ base64: string; uri: string }> {
23+
const { ImageManipulator, SaveFormat } = await import("expo-image-manipulator");
24+
let image = await ImageManipulator.manipulate(uri).renderAsync();
25+
try {
26+
const fittedSize = fitImageAttachmentSize(image);
27+
if (fittedSize !== null) {
28+
const resized = await ImageManipulator.manipulate(image).resize(fittedSize).renderAsync();
29+
image.release();
30+
image = resized;
31+
}
32+
const saved = await image.saveAsync({
33+
format: format === "png" ? SaveFormat.PNG : SaveFormat.JPEG,
34+
compress: PHOTO_JPEG_QUALITY,
35+
base64: true,
36+
});
37+
if (!saved.base64) {
38+
throw new Error("The rendered photo has no bytes.");
39+
}
40+
return { base64: saved.base64, uri: saved.uri };
41+
} finally {
42+
image.release();
43+
}
44+
}
45+
46+
/**
47+
* Scales a PNG or JPEG whose long edge is over `IMAGE_ATTACHMENT_MAX_EDGE` down to that edge
48+
* in its own format. Returns null when the image fits, is another format (GIF, WebP), or the
49+
* native renderer fails, so the caller keeps the original bytes. Images inside the limit are
50+
* never re-encoded: only the header is read to decide.
51+
*/
52+
export async function fitComposerImage(image: {
53+
readonly uri: string;
54+
readonly mimeType: string;
55+
readonly base64: string;
56+
}): Promise<{ base64: string; uri: string } | null> {
57+
const mimeType = downscalableImageMimeType(image.mimeType);
58+
if (mimeType === null) return null;
59+
try {
60+
const header = atob(image.base64.slice(0, Math.ceil(IMAGE_DIMENSIONS_HEADER_BYTES / 3) * 4));
61+
const bytes = new Uint8Array(header.length);
62+
for (let index = 0; index < header.length; index += 1) bytes[index] = header.charCodeAt(index);
63+
const dimensions = readImageDimensions(bytes);
64+
if (dimensions === null || fitImageAttachmentSize(dimensions) === null) return null;
65+
return await renderPhoto(image.uri, mimeType === "image/png" ? "png" : "jpeg");
66+
} catch (error) {
67+
console.warn("[composer-images] could not scale an oversized image", error);
68+
return null;
69+
}
70+
}

‎apps/mobile/src/lib/composerImages.test.ts‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,35 @@ const clipboard = vi.hoisted(() => ({
1212

1313
vi.mock("expo-clipboard", () => clipboard);
1414

15+
/** A native image renderer that reports the source size the test gives it. */
16+
const manipulator = vi.hoisted(() => {
17+
const state = { sourceSize: { width: 0, height: 0 } };
18+
const manipulate = vi.fn((source: string | { width: number; height: number }) => {
19+
let size = typeof source === "string" ? state.sourceSize : source;
20+
const context = {
21+
resize: vi.fn((next: { width: number; height: number }) => {
22+
size = next;
23+
return context;
24+
}),
25+
renderAsync: vi.fn(async () => ({
26+
...size,
27+
release: vi.fn(),
28+
saveAsync: vi.fn(async (options: { format: string }) => ({
29+
uri: `file:///cache/rendered.${options.format}`,
30+
base64: `cmVuZGVyZWQ=`,
31+
})),
32+
})),
33+
};
34+
return context;
35+
});
36+
return { state, manipulate };
37+
});
38+
39+
vi.mock("expo-image-manipulator", () => ({
40+
ImageManipulator: { manipulate: manipulator.manipulate },
41+
SaveFormat: { JPEG: "jpeg", PNG: "png" },
42+
}));
43+
1544
vi.mock("expo-file-system", () => ({
1645
File: class {
1746
readonly uri: string;
@@ -128,6 +157,67 @@ describe("composer clipboard paste", () => {
128157
});
129158
});
130159

160+
describe("oversized pasted images", () => {
161+
const u32 = (n: number) => [(n >>> 24) & 0xff, (n >>> 16) & 0xff, (n >>> 8) & 0xff, n & 0xff];
162+
const pngDataUrl = (width: number, height: number) =>
163+
`data:image/png;base64,${Buffer.from([
164+
0x89,
165+
0x50,
166+
0x4e,
167+
0x47,
168+
0x0d,
169+
0x0a,
170+
0x1a,
171+
0x0a,
172+
...u32(13),
173+
0x49,
174+
0x48,
175+
0x44,
176+
0x52,
177+
...u32(width),
178+
...u32(height),
179+
8,
180+
6,
181+
0,
182+
0,
183+
0,
184+
]).toString("base64")}`;
185+
186+
beforeEach(() => {
187+
vi.clearAllMocks();
188+
clipboard.hasImageAsync.mockResolvedValue(true);
189+
});
190+
191+
it("scales a clipboard screenshot over 2000px to 2000px and keeps it a PNG", async () => {
192+
manipulator.state.sourceSize = { width: 3024, height: 1964 };
193+
const pasted = pngDataUrl(3024, 1964);
194+
clipboard.getImageAsync.mockResolvedValue({ data: pasted });
195+
196+
const result = await pasteComposerClipboard({ existingCount: 0 });
197+
198+
expect(manipulator.manipulate).toHaveBeenCalledWith(pasted);
199+
const resize = manipulator.manipulate.mock.results.at(-1)?.value.resize;
200+
expect(resize).toHaveBeenCalledWith({ width: 2000, height: 1299 });
201+
expect(result.images).toEqual([
202+
expect.objectContaining({
203+
mimeType: "image/png",
204+
dataUrl: "data:image/png;base64,cmVuZGVyZWQ=",
205+
previewUri: "data:image/png;base64,cmVuZGVyZWQ=",
206+
}),
207+
]);
208+
});
209+
210+
it("keeps a 2000px clipboard image byte for byte without rendering it", async () => {
211+
const pasted = pngDataUrl(2000, 1200);
212+
clipboard.getImageAsync.mockResolvedValue({ data: pasted });
213+
214+
const result = await pasteComposerClipboard({ existingCount: 0 });
215+
216+
expect(manipulator.manipulate).not.toHaveBeenCalled();
217+
expect(result.images).toEqual([expect.objectContaining({ dataUrl: pasted })]);
218+
});
219+
});
220+
131221
describe("native pasted image cleanup", () => {
132222
beforeEach(() => {
133223
files.clear();

‎apps/mobile/src/lib/composerImages.ts‎

Lines changed: 25 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import {
2020
} from "./composerAttachmentFiles";
2121
import { imageMimeType } from "@t3tools/shared/image";
2222
import { videoMimeType } from "@t3tools/shared/video";
23+
import { fitComposerImage, renderPhoto } from "./composerImageFit";
2324
import { beginForegroundHandoff } from "./foreground-handoff";
2425
import { uuidv4 } from "./uuid";
2526
import { writeFileAtomically } from "./atomic-file";
@@ -341,46 +342,6 @@ export async function pickComposerFiles(input: {
341342
return { files: attachments, error };
342343
}
343344

344-
/**
345-
* Longest edge kept when a photo has to be re-encoded. Matches the web composer's
346-
* MAX_DIMENSION so every client hands providers the same resolution.
347-
*/
348-
const PHOTO_MAX_EDGE = 2048;
349-
const PHOTO_JPEG_QUALITY = 0.85;
350-
351-
/**
352-
* Renders a photo-library pick to a provider-readable JPEG. Decode, downscale, and encode run
353-
* natively; only the bounded result crosses the bridge. Camera photos are 12-48 MP HEIC files,
354-
* so a full-size conversion is both slow to transfer and far more than a model can use.
355-
*/
356-
async function renderPhotoAsJpeg(uri: string): Promise<{ base64: string; uri: string }> {
357-
const { ImageManipulator, SaveFormat } = await import("expo-image-manipulator");
358-
let image = await ImageManipulator.manipulate(uri).renderAsync();
359-
try {
360-
const longestEdge = Math.max(image.width, image.height);
361-
if (longestEdge > PHOTO_MAX_EDGE) {
362-
const resized = await ImageManipulator.manipulate(image)
363-
.resize(
364-
image.width >= image.height ? { width: PHOTO_MAX_EDGE } : { height: PHOTO_MAX_EDGE },
365-
)
366-
.renderAsync();
367-
image.release();
368-
image = resized;
369-
}
370-
const saved = await image.saveAsync({
371-
format: SaveFormat.JPEG,
372-
compress: PHOTO_JPEG_QUALITY,
373-
base64: true,
374-
});
375-
if (!saved.base64) {
376-
throw new Error("The rendered photo has no bytes.");
377-
}
378-
return { base64: saved.base64, uri: saved.uri };
379-
} finally {
380-
image.release();
381-
}
382-
}
383-
384345
async function loadImagePicker() {
385346
try {
386347
return await import("expo-image-picker");
@@ -530,14 +491,20 @@ export async function pickComposerMedia(input: {
530491
try {
531492
if (originalMimeType !== null) {
532493
const { File } = await import("expo-file-system");
494+
const base64 = await new File(asset.uri).base64();
495+
const fitted = await fitComposerImage({
496+
uri: asset.uri,
497+
mimeType: originalMimeType,
498+
base64,
499+
});
533500
image = {
534-
base64: await new File(asset.uri).base64(),
501+
base64: fitted?.base64 ?? base64,
535502
mimeType: originalMimeType,
536503
name,
537-
previewUri: asset.uri,
504+
previewUri: fitted?.uri ?? asset.uri,
538505
};
539506
} else {
540-
const rendered = await renderPhotoAsJpeg(asset.uri);
507+
const rendered = await renderPhoto(asset.uri);
541508
image = {
542509
base64: rendered.base64,
543510
mimeType: "image/jpeg",
@@ -616,7 +583,14 @@ export async function pasteComposerClipboard(input: { readonly existingCount: nu
616583
};
617584
}
618585

619-
const base64 = image.data.split(",")[1] ?? "";
586+
const pastedBase64 = image.data.split(",")[1] ?? "";
587+
const fitted = await fitComposerImage({
588+
uri: image.data,
589+
mimeType: "image/png",
590+
base64: pastedBase64,
591+
});
592+
const base64 = fitted?.base64 ?? pastedBase64;
593+
const dataUrl = fitted ? `data:image/png;base64,${base64}` : image.data;
620594
const sizeBytes = estimateBase64ByteSize(base64);
621595
if (sizeBytes <= 0 || sizeBytes > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) {
622596
return {
@@ -634,8 +608,8 @@ export async function pasteComposerClipboard(input: { readonly existingCount: nu
634608
name: "pasted-image.png",
635609
mimeType: "image/png",
636610
sizeBytes,
637-
dataUrl: image.data,
638-
previewUri: image.data,
611+
dataUrl,
612+
previewUri: dataUrl,
639613
},
640614
],
641615
text: null,
@@ -707,20 +681,22 @@ export async function convertPastedImagesToAttachments(input: {
707681
continue;
708682
}
709683
const file = new File(uri);
710-
const base64 = await file.base64();
684+
const mimeType = mimeTypeFromUri(uri);
685+
const pastedBase64 = await file.base64();
686+
const fitted = await fitComposerImage({ uri, mimeType, base64: pastedBase64 });
687+
const base64 = fitted?.base64 ?? pastedBase64;
711688
const sizeBytes = estimateBase64ByteSize(base64);
712689
if (sizeBytes <= 0 || sizeBytes > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) {
713690
continue;
714691
}
715-
const mimeType = mimeTypeFromUri(uri);
716692
results.push({
717693
id: uuidv4(),
718694
type: "image",
719695
name: `pasted-image.${mimeType.split("/")[1] ?? "png"}`,
720696
mimeType,
721697
sizeBytes,
722698
dataUrl: `data:${mimeType};base64,${base64}`,
723-
previewUri: ownedTemporaryFile ? `data:${mimeType};base64,${base64}` : uri,
699+
previewUri: ownedTemporaryFile ? `data:${mimeType};base64,${base64}` : (fitted?.uri ?? uri),
724700
});
725701
} catch (error) {
726702
console.warn("Failed to read pasted image", uri, error);

0 commit comments

Comments
 (0)