Skip to content

Commit 1dcd586

Browse files
author
Julius Marminge
committed
fix(mobile): attach HEIC photos using native JPEG conversion
1 parent 788901a commit 1dcd586

3 files changed

Lines changed: 151 additions & 12 deletions

File tree

‎apps/mobile/src/lib/composerFiles.test.ts‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { beforeEach, describe, expect, it, vi } from "vite-plus/test";
2+
import { PROVIDER_SEND_TURN_MAX_IMAGE_BYTES } from "@t3tools/contracts";
23
import type { ImagePickerAsset } from "expo-image-picker";
34

45
const mocks = vi.hoisted(() => ({
@@ -9,6 +10,7 @@ const mocks = vi.hoisted(() => ({
910
delete: vi.fn(),
1011
open: vi.fn(),
1112
size: vi.fn(),
13+
readBase64: vi.fn(),
1214
}));
1315

1416
vi.mock("expo-file-system", () => {
@@ -55,6 +57,10 @@ vi.mock("expo-file-system", () => {
5557
mocks.copy(this.uri, destination.uri);
5658
}
5759

60+
async base64(): Promise<string> {
61+
return mocks.readBase64(this.uri);
62+
}
63+
5864
delete(): void {
5965
mocks.delete(this.uri);
6066
}
@@ -95,9 +101,115 @@ describe("composer file attachments", () => {
95101
mocks.delete.mockReset();
96102
mocks.open.mockReset();
97103
mocks.size.mockReset();
104+
mocks.readBase64.mockReset();
98105
mocks.size.mockImplementation((uri: string) => (uri.startsWith("content:") ? null : 42));
99106
});
100107

108+
describe("photo library image conversion", () => {
109+
const jpeg = "/9j/2Q==";
110+
const photo: ImagePickerAsset = {
111+
uri: "file:///picker/photo.heic",
112+
type: "image",
113+
fileName: "photo.HEIC",
114+
mimeType: "image/heic",
115+
fileSize: 20 * 1024 * 1024,
116+
base64: jpeg,
117+
width: 1,
118+
height: 1,
119+
};
120+
121+
it.each(["image/heic", "image/heif", undefined])(
122+
"attaches the native JPEG conversion with matching metadata when the source MIME is %s",
123+
async (mimeType) => {
124+
mocks.pickMedia.mockResolvedValue({
125+
canceled: false,
126+
assets: [{ ...photo, mimeType }],
127+
});
128+
129+
const result = await pickComposerImages({ existingCount: 0 });
130+
131+
expect(result).toEqual({
132+
images: [
133+
{
134+
id: "attachment-id",
135+
type: "image",
136+
name: "photo.jpg",
137+
mimeType: "image/jpeg",
138+
sizeBytes: 4,
139+
dataUrl: `data:image/jpeg;base64,${jpeg}`,
140+
previewUri: `data:image/jpeg;base64,${jpeg}`,
141+
},
142+
],
143+
error: null,
144+
});
145+
},
146+
);
147+
148+
it.each([
149+
{ extension: "png", mimeType: "image/png", base64: "iVBORw0KGgo=" },
150+
{ extension: "gif", mimeType: "image/gif", base64: "R0lGODlh" },
151+
{ extension: "webp", mimeType: "image/webp", base64: "UklGRgQAAABXRUJQ" },
152+
])("preserves original $extension bytes instead of the picker's JPEG", async (original) => {
153+
const name = `photo.${original.extension}`;
154+
mocks.pickMedia.mockResolvedValue({
155+
canceled: false,
156+
assets: [{ ...photo, fileName: name, mimeType: original.mimeType }],
157+
});
158+
mocks.readBase64.mockResolvedValue(original.base64);
159+
160+
const result = await pickComposerImages({ existingCount: 0 });
161+
162+
expect(result.error).toBeNull();
163+
expect(result.images).toEqual([
164+
expect.objectContaining({
165+
name,
166+
mimeType: original.mimeType,
167+
dataUrl: `data:${original.mimeType};base64,${original.base64}`,
168+
sizeBytes: Buffer.from(original.base64, "base64").byteLength,
169+
}),
170+
]);
171+
});
172+
173+
it("checks the converted JPEG size even when the HEIC source was smaller", async () => {
174+
const oversized =
175+
jpeg.slice(0, 4) + "A".repeat(Math.ceil(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES / 3) * 4);
176+
mocks.pickMedia.mockResolvedValue({
177+
canceled: false,
178+
assets: [{ ...photo, fileSize: 42, base64: oversized }],
179+
});
180+
181+
await expect(pickComposerImages({ existingCount: 0 })).resolves.toEqual({
182+
images: [],
183+
error: "'photo.HEIC' exceeds the 10 MB attachment limit.",
184+
});
185+
});
186+
187+
it("does not relabel unconverted HEIC bytes as JPEG", async () => {
188+
mocks.pickMedia.mockResolvedValue({
189+
canceled: false,
190+
assets: [{ ...photo, base64: "AAAAGGZ0eXBoZWlj" }],
191+
});
192+
193+
const result = await pickComposerImages({ existingCount: 0 });
194+
195+
expect(result.images).toEqual([]);
196+
expect(result.error).toContain("not a supported image type");
197+
});
198+
199+
it("retains a converted photo when another original cannot be read", async () => {
200+
mocks.pickMedia.mockResolvedValue({
201+
canceled: false,
202+
assets: [{ ...photo, fileName: "missing.gif", mimeType: "image/gif" }, photo],
203+
});
204+
mocks.readBase64.mockRejectedValue(new Error("missing file"));
205+
206+
const result = await pickComposerImages({ existingCount: 0 });
207+
208+
expect(result.images).toEqual([expect.objectContaining({ name: "photo.jpg" })]);
209+
expect(result.error).toBe("Failed to read 'missing.gif'.");
210+
});
211+
});
212+
101213
describe("photo library videos", () => {
102214
const image: ImagePickerAsset = {
103215
uri: "file:///picker/photo.png",

‎apps/mobile/src/lib/composerImages.ts‎

Lines changed: 37 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -351,7 +351,7 @@ export async function pickComposerMedia(input: {
351351
error = `You can attach up to ${PROVIDER_SEND_TURN_MAX_ATTACHMENTS} attachments per message.`;
352352
break;
353353
}
354-
const mimeType = asset.mimeType?.toLowerCase();
354+
let mimeType = asset.mimeType?.toLowerCase();
355355
if (asset.type === "video" || mimeType?.startsWith("video/")) {
356356
if (input.maxVideoBytes === undefined) {
357357
error = "Video attachments are unavailable here.";
@@ -375,35 +375,61 @@ export async function pickComposerMedia(input: {
375375
}
376376
continue;
377377
}
378-
if (!mimeType?.startsWith("image/")) {
378+
if (asset.type !== "image" && !mimeType?.startsWith("image/")) {
379379
error = `Unsupported file type for '${asset.fileName ?? "image"}'.`;
380380
continue;
381381
}
382-
if (!isProviderSendTurnSupportedImageMimeType(mimeType)) {
383-
error = `'${asset.fileName ?? "image"}' is not a supported image type. Attach GIF, JPEG, PNG, or WebP images.`;
384-
continue;
385-
}
386382

387-
const base64 = asset.base64;
383+
let base64 = asset.base64;
388384
if (!base64) {
389385
error = `Failed to read '${asset.fileName ?? "image"}'.`;
390386
continue;
391387
}
392388

393-
const sizeBytes = asset.fileSize ?? estimateBase64ByteSize(base64);
389+
let name = asset.fileName?.trim() || "image";
390+
// The iOS picker returns JPEG base64 even when its metadata describes HEIC,
391+
// PNG, or GIF. Keep supported originals so transparency and animation survive;
392+
// use the native JPEG conversion for formats providers cannot accept.
393+
if (base64.startsWith("/9j/")) {
394+
if (
395+
mimeType &&
396+
mimeType !== "image/jpeg" &&
397+
isProviderSendTurnSupportedImageMimeType(mimeType)
398+
) {
399+
try {
400+
const { File } = await import("expo-file-system");
401+
base64 = await new File(asset.uri).base64();
402+
} catch {
403+
error = `Failed to read '${name}'.`;
404+
continue;
405+
}
406+
} else {
407+
mimeType = "image/jpeg";
408+
if (!/\.jpe?g$/i.test(name)) {
409+
name = `${name.replace(/\.[^.]+$/, "")}.jpg`;
410+
}
411+
}
412+
}
413+
if (!mimeType || !isProviderSendTurnSupportedImageMimeType(mimeType)) {
414+
error = `'${name}' is not a supported image type. Attach GIF, JPEG, PNG, or WebP images.`;
415+
continue;
416+
}
417+
418+
const sizeBytes = estimateBase64ByteSize(base64);
394419
if (sizeBytes <= 0 || sizeBytes > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) {
395420
error = `'${asset.fileName ?? "image"}' exceeds the 10 MB attachment limit.`;
396421
continue;
397422
}
398423

424+
const dataUrl = `data:${mimeType};base64,${base64}`;
399425
attachments.push({
400426
id: uuidv4(),
401427
type: "image",
402-
name: asset.fileName ?? "image",
428+
name,
403429
mimeType,
404430
sizeBytes,
405-
dataUrl: `data:${mimeType};base64,${base64}`,
406-
previewUri: asset.uri,
431+
dataUrl,
432+
previewUri: mimeType === asset.mimeType?.toLowerCase() ? asset.uri : dataUrl,
407433
});
408434
}
409435

‎docs/user/composer.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,8 @@ On web and desktop, if you reload before a file finishes uploading, the draft ke
4242
and shows **Attach again** next to it. Attach the file again or remove it, then send.
4343

4444
On web and desktop, HEIC and HEIF photos are automatically converted to JPEG when you drag them into
45-
the composer or paste them into a message.
45+
the composer or paste them into a message. On iOS, selecting them from **Photo Library** also
46+
converts them to JPEG. The 10 MB image limit applies to the converted photo.
4647

4748
On mobile, the model picker shows each OpenCode model's upstream provider, such as Anthropic,
4849
GitHub Copilot, or OpenCode Zen, beneath its name. Search by that provider name to narrow the list

0 commit comments

Comments
 (0)