Skip to content

Commit ee0662d

Browse files
Nelglorclaude
andcommitted
fix(mobile): render photo library picks to a bounded JPEG off the JS thread
Picking a camera photo froze the composer. The picker was asked for base64 at quality 1, so iOS decoded and re-encoded every 12-48 MP HEIC at full resolution and handed JS a 10 MB+ string. HEIC previews then used that data URL as the thumbnail, which Fabric re-parsed on every layout pass. The picker now leaves bytes in its file. Originals the provider can read and that fit the 10 MB cap are read as-is so PNG transparency and GIF animation survive. Everything else is rendered natively with expo-image-manipulator to a 2048 px longest edge JPEG at 0.85, matching the web composer, and the thumbnail points at the rendered file instead of the data URL. Fixes #11427 Co-Authored-By: Claude Fable 5.1 <[email protected]>
1 parent b1e223e commit ee0662d

5 files changed

Lines changed: 230 additions & 64 deletions

File tree

‎apps/mobile/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@
9191
"expo-glass-effect": "~57.0.1",
9292
"expo-haptics": "~57.0.2",
9393
"expo-image": "~57.0.3",
94+
"expo-image-manipulator": "~57.0.17",
9495
"expo-image-picker": "~57.0.14",
9596
"expo-linking": "~57.0.8",
9697
"expo-network": "~57.0.1",

‎apps/mobile/src/lib/composerFiles.test.ts‎

Lines changed: 123 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ const mocks = vi.hoisted(() => ({
1111
open: vi.fn(),
1212
size: vi.fn(),
1313
readBase64: vi.fn(),
14+
manipulate: vi.fn(),
15+
release: vi.fn(),
1416
}));
1517

1618
vi.mock("expo-file-system", () => {
@@ -80,6 +82,10 @@ vi.mock("expo-file-system", () => {
8082

8183
vi.mock("expo-image-picker", () => ({ launchImageLibraryAsync: mocks.pickMedia }));
8284
vi.mock("expo-document-picker", () => ({ getDocumentAsync: mocks.pickFile }));
85+
vi.mock("expo-image-manipulator", () => ({
86+
SaveFormat: { JPEG: "jpeg", PNG: "png", WEBP: "webp" },
87+
ImageManipulator: { manipulate: mocks.manipulate },
88+
}));
8389
vi.mock("./uuid", () => ({ uuidv4: () => "attachment-id" }));
8490

8591
import {
@@ -106,20 +112,50 @@ describe("composer file attachments", () => {
106112
});
107113

108114
describe("photo library image conversion", () => {
109-
const jpeg = "/9j/2Q==";
115+
const rendered = { uri: "file:///cache/ImageManipulator/photo.jpg", base64: "/9j/2Q==" };
110116
const photo: ImagePickerAsset = {
111117
uri: "file:///picker/photo.heic",
112118
type: "image",
113119
fileName: "photo.HEIC",
114120
mimeType: "image/heic",
115-
fileSize: 20 * 1024 * 1024,
116-
base64: jpeg,
117-
width: 1,
118-
height: 1,
121+
fileSize: 4 * 1024 * 1024,
122+
width: 4032,
123+
height: 3024,
124+
};
125+
/**
126+
* Stands in for the native manipulator: `size` is what decoding the source yields,
127+
* `resizes` records every requested resize, and `saved` is what saving returns.
128+
*/
129+
const native = {
130+
size: { width: 1, height: 1 },
131+
resizes: [] as Array<{ width?: number | null; height?: number | null }>,
132+
saved: rendered as { uri: string; base64?: string },
119133
};
120134

135+
beforeEach(() => {
136+
native.size = { width: 1, height: 1 };
137+
native.resizes = [];
138+
native.saved = rendered;
139+
mocks.manipulate.mockReset();
140+
mocks.release.mockReset();
141+
mocks.manipulate.mockImplementation(() => {
142+
const context = {
143+
resize(size: { width?: number | null; height?: number | null }) {
144+
native.resizes.push(size);
145+
return context;
146+
},
147+
renderAsync: async () => ({
148+
...native.size,
149+
release: mocks.release,
150+
saveAsync: async () => native.saved,
151+
}),
152+
};
153+
return context;
154+
});
155+
});
156+
121157
it.each(["image/heic", "image/heif", undefined])(
122-
"attaches the native JPEG conversion with matching metadata when the source MIME is %s",
158+
"renders a %s photo to JPEG natively and previews the rendered file",
123159
async (mimeType) => {
124160
mocks.pickMedia.mockResolvedValue({
125161
canceled: false,
@@ -128,6 +164,9 @@ describe("composer file attachments", () => {
128164

129165
const result = await pickComposerImages({ existingCount: 0 });
130166

167+
expect(mocks.pickMedia).toHaveBeenCalledWith(expect.objectContaining({ base64: false }));
168+
expect(mocks.manipulate).toHaveBeenCalledWith(photo.uri);
169+
expect(mocks.readBase64).not.toHaveBeenCalled();
131170
expect(result).toEqual({
132171
images: [
133172
{
@@ -136,20 +175,35 @@ describe("composer file attachments", () => {
136175
name: "photo.jpg",
137176
mimeType: "image/jpeg",
138177
sizeBytes: 4,
139-
dataUrl: `data:image/jpeg;base64,${jpeg}`,
140-
previewUri: `data:image/jpeg;base64,${jpeg}`,
178+
dataUrl: `data:image/jpeg;base64,${rendered.base64}`,
179+
previewUri: rendered.uri,
141180
},
142181
],
143182
error: null,
144183
});
145184
},
146185
);
147186

187+
it.each([
188+
{ size: { width: 4032, height: 3024 }, resizes: [{ width: 2048 }] },
189+
{ size: { width: 3024, height: 4032 }, resizes: [{ height: 2048 }] },
190+
{ size: { width: 2048, height: 1536 }, resizes: [] },
191+
])("bounds a $size.width x $size.height photo to a 2048 px longest edge", async (input) => {
192+
native.size = input.size;
193+
mocks.pickMedia.mockResolvedValue({ canceled: false, assets: [photo] });
194+
195+
await pickComposerImages({ existingCount: 0 });
196+
197+
expect(native.resizes).toEqual(input.resizes);
198+
// Every decoded bitmap is released, including the full-size one a resize replaces.
199+
expect(mocks.release).toHaveBeenCalledTimes(input.resizes.length + 1);
200+
});
201+
148202
it.each([
149203
{ extension: "png", mimeType: "image/png", base64: "iVBORw0KGgo=" },
150204
{ extension: "gif", mimeType: "image/gif", base64: "R0lGODlh" },
151205
{ extension: "webp", mimeType: "image/webp", base64: "UklGRgQAAABXRUJQ" },
152-
])("preserves original $extension bytes instead of the picker's JPEG", async (original) => {
206+
])("keeps original $extension bytes from the picker file", async (original) => {
153207
const name = `photo.${original.extension}`;
154208
mocks.pickMedia.mockResolvedValue({
155209
canceled: false,
@@ -159,44 +213,72 @@ describe("composer file attachments", () => {
159213

160214
const result = await pickComposerImages({ existingCount: 0 });
161215

216+
expect(mocks.manipulate).not.toHaveBeenCalled();
217+
expect(mocks.readBase64).toHaveBeenCalledWith(photo.uri);
162218
expect(result.error).toBeNull();
163219
expect(result.images).toEqual([
164220
expect.objectContaining({
165221
name,
166222
mimeType: original.mimeType,
167223
dataUrl: `data:${original.mimeType};base64,${original.base64}`,
168224
sizeBytes: Buffer.from(original.base64, "base64").byteLength,
225+
previewUri: photo.uri,
169226
}),
170227
]);
171228
});
172229

173-
it("checks the converted JPEG size even when the HEIC source was smaller", async () => {
174-
const oversized =
175-
jpeg.slice(0, 4) + "A".repeat(Math.ceil(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES / 3) * 4);
230+
it("renders a supported original that exceeds the image limit instead of rejecting it", async () => {
176231
mocks.pickMedia.mockResolvedValue({
177232
canceled: false,
178-
assets: [{ ...photo, fileSize: 42, base64: oversized }],
233+
assets: [
234+
{
235+
...photo,
236+
fileName: "photo.jpg",
237+
mimeType: "image/jpeg",
238+
fileSize: PROVIDER_SEND_TURN_MAX_IMAGE_BYTES + 1,
239+
},
240+
],
179241
});
180242

181-
await expect(pickComposerImages({ existingCount: 0 })).resolves.toEqual({
182-
images: [],
183-
error: "'photo.HEIC' exceeds the 10 MB attachment limit.",
184-
});
243+
const result = await pickComposerImages({ existingCount: 0 });
244+
245+
expect(mocks.manipulate).toHaveBeenCalledWith(photo.uri);
246+
expect(result.error).toBeNull();
247+
expect(result.images).toEqual([
248+
expect.objectContaining({ name: "photo.jpg", mimeType: "image/jpeg", sizeBytes: 4 }),
249+
]);
185250
});
186251

187-
it("does not relabel unconverted HEIC bytes as JPEG", async () => {
252+
it("measures the picker file when the picker reports no size", async () => {
188253
mocks.pickMedia.mockResolvedValue({
189254
canceled: false,
190-
assets: [{ ...photo, base64: "AAAAGGZ0eXBoZWlj" }],
255+
assets: [{ ...photo, fileName: "photo.png", mimeType: "image/png", fileSize: undefined }],
191256
});
257+
mocks.size.mockReturnValue(3);
258+
mocks.readBase64.mockResolvedValue("YWJj");
192259

193260
const result = await pickComposerImages({ existingCount: 0 });
194261

195-
expect(result.images).toEqual([]);
196-
expect(result.error).toContain("not a supported image type");
262+
expect(mocks.manipulate).not.toHaveBeenCalled();
263+
expect(result.images).toEqual([expect.objectContaining({ mimeType: "image/png" })]);
197264
});
198265

199-
it("retains a converted photo when another original cannot be read", async () => {
266+
it("checks the rendered JPEG against the image limit", async () => {
267+
native.saved = {
268+
uri: rendered.uri,
269+
base64:
270+
rendered.base64.slice(0, 4) +
271+
"A".repeat(Math.ceil(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES / 3) * 4),
272+
};
273+
mocks.pickMedia.mockResolvedValue({ canceled: false, assets: [photo] });
274+
275+
await expect(pickComposerImages({ existingCount: 0 })).resolves.toEqual({
276+
images: [],
277+
error: "'photo.HEIC' exceeds the 10 MB attachment limit.",
278+
});
279+
});
280+
281+
it("retains a rendered photo when another original cannot be read", async () => {
200282
mocks.pickMedia.mockResolvedValue({
201283
canceled: false,
202284
assets: [{ ...photo, fileName: "missing.gif", mimeType: "image/gif" }, photo],
@@ -208,6 +290,21 @@ describe("composer file attachments", () => {
208290
expect(result.images).toEqual([expect.objectContaining({ name: "photo.jpg" })]);
209291
expect(result.error).toBe("Failed to read 'missing.gif'.");
210292
});
293+
294+
it("reports a photo the native renderer cannot decode", async () => {
295+
mocks.manipulate.mockImplementation(() => ({
296+
resize: () => {
297+
throw new Error("unreachable");
298+
},
299+
renderAsync: () => Promise.reject(new Error("corrupt")),
300+
}));
301+
mocks.pickMedia.mockResolvedValue({ canceled: false, assets: [photo] });
302+
303+
await expect(pickComposerImages({ existingCount: 0 })).resolves.toEqual({
304+
images: [],
305+
error: "Failed to read 'photo.HEIC'.",
306+
});
307+
});
211308
});
212309

213310
describe("photo library videos", () => {
@@ -217,10 +314,13 @@ describe("composer file attachments", () => {
217314
fileName: "photo.png",
218315
mimeType: "image/png",
219316
fileSize: 3,
220-
base64: "YWJj",
221317
width: 1,
222318
height: 1,
223319
};
320+
321+
beforeEach(() => {
322+
mocks.readBase64.mockResolvedValue("YWJj");
323+
});
224324
const video: ImagePickerAsset = {
225325
uri: "file:///picker/clip.mov",
226326
type: "video",

0 commit comments

Comments
 (0)