Skip to content

Commit 8e13841

Browse files
ahalekellyclaude
andcommitted
perf(mobile): picked photos and videos preview as soon as the picker closes
Placeholders built from the picker's local copies show in the attachment strip while processing runs. Each keeps its final attachment's id, and the finished image keeps the picker copy as its preview, so the thumbnail does not reload. Removing a placeholder cancels that item, and sending waits for processing without locking the editor. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent c65f8d5 commit 8e13841

8 files changed

Lines changed: 145 additions & 31 deletions

‎apps/mobile/src/features/threads/NewTaskDraftScreen.tsx‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ import {
4040
} from "@t3tools/contracts";
4141

4242
import { ComposerEditor, type ComposerEditorHandle } from "../../components/ComposerEditor";
43-
import { composerContextImportsAtom } from "../../state/use-composer-drafts";
43+
import { composerContextImportsAtom, composerDraftBusy } from "../../state/use-composer-drafts";
4444
import { abortComposerDraftOperations } from "../../state/composerDraftOperations";
4545
import { composerContextSendBlockReason } from "../../lib/composerContext";
4646
import { ComposerInlineControl } from "../../components/ComposerToolbar";
@@ -93,7 +93,6 @@ import {
9393
import { incomingShareImports, useIncomingShare } from "../sharing/IncomingShareProvider";
9494
import { idleShareFlagAtom, idleShareImportPhaseAtom } from "./incomingShareImport";
9595
import { newTaskFlowKey } from "./new-task-flow-state";
96-
import { appAtomRegistry } from "../../state/atom-registry";
9796

9897
import { readEnvironmentScope, useEnvironmentScope } from "../../state/session";
9998

@@ -576,7 +575,7 @@ export function NewTaskDraftScreen(props: {
576575
return null;
577576
}
578577
const draft = getComposerDraftSnapshot(draftKey);
579-
if (appAtomRegistry.get(composerContextImportsAtom)[draftKey]) return null;
578+
if (composerDraftBusy(draftKey)) return null;
580579
const modelSelection =
581580
resolveSelectableModelSelection(
582581
selectedEnvironmentServerConfig,

‎apps/mobile/src/features/threads/ThreadComposer.tsx‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ import {
6060
composerDraftsAtom,
6161
setComposerDraftText,
6262
composerContextImportsAtom,
63+
composerPendingMediaAtom,
6364
countComposerDraftAttachmentsAfterSelection,
6465
useComposerDraft,
6566
} from "../../state/use-composer-drafts";
@@ -449,11 +450,19 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps
449450
const composerDraftKey = props.draftKey ?? composerOwnerKey;
450451
const { text: draftMessage, attachments: draftAttachments } = useComposerDraft(composerDraftKey);
451452
const queuedEdit = props.queuedEdit ?? null;
453+
const pendingMedia = useAtomValue(composerPendingMediaAtom)[composerDraftKey];
452454
const hasContent =
453455
draftMessage.trim().length > 0 ||
454456
draftAttachments.length > 0 ||
457+
pendingMedia !== undefined ||
455458
(queuedEdit?.existingAttachments.length ?? 0) > 0;
456-
const stripAttachments = composerStripAttachments(draftAttachments);
459+
// A finished attachment keeps its placeholder's id; skip the placeholder once it lands.
460+
const stripAttachments = [
461+
...composerStripAttachments(draftAttachments),
462+
...(pendingMedia ?? []).filter(
463+
(placeholder) => !draftAttachments.some(({ id }) => id === placeholder.id),
464+
),
465+
];
457466
const showStopAction = !hasContent && props.canStopThread && queuedEdit === null;
458467
const onResumeQueue = props.onResumeQueue;
459468
const [resumingQueue, setResumingQueue] = useState(false);
@@ -608,6 +617,7 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps
608617
(props.canOperateThread || props.connectionState !== "connected") &&
609618
hasContent &&
610619
!contextImports[composerDraftKey] &&
620+
pendingMedia === undefined &&
611621
!isSubmissionLocked &&
612622
sendBlockedReason === null &&
613623
!modelUnavailable;
@@ -625,7 +635,7 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps
625635
(current) =>
626636
current ?? {
627637
...source,
628-
gallery: filePreviewGallery(source, composerImageGallery(draftAttachments)),
638+
gallery: filePreviewGallery(source, composerImageGallery(stripAttachments)),
629639
},
630640
);
631641
};

‎apps/mobile/src/lib/composerFiles.test.ts‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ describe("composer file attachments", () => {
160160
});
161161

162162
it.each(["image/heic", "image/heif", undefined])(
163-
"renders a %s photo to JPEG natively and previews the rendered file",
163+
"renders a %s photo to JPEG natively and previews the picker copy",
164164
async (mimeType) => {
165165
mocks.pickMedia.mockResolvedValue({
166166
canceled: false,
@@ -181,7 +181,7 @@ describe("composer file attachments", () => {
181181
mimeType: "image/jpeg",
182182
sizeBytes: 4,
183183
dataUrl: `data:image/jpeg;base64,${rendered.base64}`,
184-
previewUri: rendered.uri,
184+
previewUri: photo.uri,
185185
},
186186
],
187187
error: null,
@@ -248,12 +248,14 @@ describe("composer file attachments", () => {
248248
name: "IMG_9280.png",
249249
mimeType: "image/png",
250250
sizeBytes: last.width * last.height * 4,
251-
previewUri: `file:///cache/fitted-${last.width}x${last.height}.png`,
251+
previewUri: photo.uri,
252252
}),
253253
);
254254
expect(image!.sizeBytes).toBeLessThanOrEqual(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES);
255255
expect(mocks.delete).toHaveBeenCalledWith("file:///cache/fitted-1397x2000.png");
256-
expect(mocks.delete).not.toHaveBeenCalledWith(image!.previewUri);
256+
expect(mocks.delete).toHaveBeenCalledWith(
257+
`file:///cache/fitted-${last.width}x${last.height}.png`,
258+
);
257259
});
258260

259261
it("measures the picker file instead of trusting the reported size", async () => {
@@ -362,8 +364,17 @@ describe("composer file attachments", () => {
362364
uri.endsWith("clip.mov") ? video.fileSize : 3,
363365
);
364366

365-
const result = await pickComposerMedia({ existingCount: 0, maxVideoBytes: 50 * 1024 * 1024 });
367+
const onPicked = vi.fn();
368+
const result = await pickComposerMedia({
369+
existingCount: 0,
370+
maxVideoBytes: 50 * 1024 * 1024,
371+
onPicked,
372+
});
366373

374+
expect(onPicked).toHaveBeenCalledWith([
375+
expect.objectContaining({ id: "attachment-id", type: "image", previewUri: image.uri }),
376+
expect.objectContaining({ id: "attachment-id", type: "file", fileUri: video.uri }),
377+
]);
367378
expect(mocks.pickMedia).toHaveBeenCalledWith(
368379
expect.objectContaining({
369380
mediaTypes: ["images", "videos"],

‎apps/mobile/src/lib/composerImages.ts‎

Lines changed: 41 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -375,6 +375,8 @@ export async function pickComposerImages(input: { readonly existingCount: number
375375
export async function pickComposerMedia(input: {
376376
readonly existingCount: number;
377377
readonly maxVideoBytes?: number;
378+
/** Receives placeholders over the picker's local copies before processing; each keeps its attachment's id. */
379+
readonly onPicked?: (placeholders: ReadonlyArray<DraftComposerAttachment>) => void;
378380
}): Promise<{
379381
readonly attachments: ReadonlyArray<DraftComposerAttachment>;
380382
readonly error: string | null;
@@ -425,11 +427,28 @@ export async function pickComposerMedia(input: {
425427
error: null,
426428
};
427429
}
430+
const ids = result.assets.map(() => uuidv4());
431+
input.onPicked?.(
432+
result.assets.flatMap((asset, index): DraftComposerAttachment[] => {
433+
const common = {
434+
id: ids[index]!,
435+
name: asset.fileName ?? "",
436+
mimeType: asset.mimeType ?? "",
437+
sizeBytes: asset.fileSize ?? 0,
438+
};
439+
if (asset.type === "image" || asset.mimeType?.startsWith("image/")) {
440+
return [{ ...common, type: "image", previewUri: asset.uri }];
441+
}
442+
return input.maxVideoBytes === undefined
443+
? []
444+
: [{ ...common, type: "file", fileUri: asset.uri }];
445+
}),
446+
);
428447

429448
const attachments: DraftComposerAttachment[] = [];
430449
let error: string | null = null;
431450

432-
for (const asset of result.assets) {
451+
for (const [index, asset] of result.assets.entries()) {
433452
if (attachments.length >= remainingSlots) {
434453
error = `You can attach up to ${PROVIDER_SEND_TURN_MAX_ATTACHMENTS} attachments per message.`;
435454
break;
@@ -443,15 +462,16 @@ export async function pickComposerMedia(input: {
443462
try {
444463
const { File } = await import("expo-file-system");
445464
const file = new File(asset.uri);
446-
attachments.push(
447-
await createComposerFileAttachment({
465+
attachments.push({
466+
...(await createComposerFileAttachment({
448467
uri: asset.uri,
449468
name: asset.fileName?.trim() || file.name || "video",
450469
mimeType: mimeType || file.type || "application/octet-stream",
451470
sizeBytes: asset.fileSize ?? null,
452471
maxBytes: clampFileAttachmentUploadBytes(input.maxVideoBytes),
453-
}),
454-
);
472+
})),
473+
id: ids[index]!,
474+
});
455475
} catch (cause) {
456476
error =
457477
cause instanceof Error ? cause.message : `Could not read '${asset.fileName ?? "video"}'.`;
@@ -482,7 +502,8 @@ export async function pickComposerMedia(input: {
482502
? mimeType
483503
: null;
484504

485-
let image: { base64: string; mimeType: string; name: string; previewUri: string };
505+
let image: { base64: string; mimeType: string; name: string };
506+
let renderedUri: string | undefined;
486507
try {
487508
if (originalMimeType !== null) {
488509
const { File } = await import("expo-file-system");
@@ -492,25 +513,34 @@ export async function pickComposerMedia(input: {
492513
mimeType: originalMimeType,
493514
base64,
494515
});
516+
renderedUri = fitted?.uri;
495517
image = {
496518
base64: fitted?.base64 ?? base64,
497519
mimeType: originalMimeType,
498520
name,
499-
previewUri: fitted?.uri ?? asset.uri,
500521
};
501522
} else {
502523
const rendered = await renderPhoto(asset.uri);
524+
renderedUri = rendered.uri;
503525
image = {
504526
base64: rendered.base64,
505527
mimeType: "image/jpeg",
506528
name: /\.jpe?g$/i.test(name) ? name : `${name.replace(/\.[^.]+$/, "")}.jpg`,
507-
previewUri: rendered.uri,
508529
};
509530
}
510531
} catch {
511532
error = `Failed to read '${name}'.`;
512533
continue;
513534
}
535+
// The bytes now live in the data URL, and the preview uses the picker copy.
536+
if (renderedUri !== undefined) {
537+
try {
538+
const { File } = await import("expo-file-system");
539+
new File(renderedUri).delete();
540+
} catch (cause) {
541+
console.warn("[composer-images] could not remove rendered photo", cause);
542+
}
543+
}
514544

515545
const sizeBytes = estimateBase64ByteSize(image.base64);
516546
if (sizeBytes <= 0 || sizeBytes > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) {
@@ -519,13 +549,14 @@ export async function pickComposerMedia(input: {
519549
}
520550

521551
attachments.push({
522-
id: uuidv4(),
552+
id: ids[index]!,
523553
type: "image",
524554
name: image.name,
525555
mimeType: image.mimeType,
526556
sizeBytes,
527557
dataUrl: `data:${image.mimeType};base64,${image.base64}`,
528-
previewUri: image.previewUri,
558+
// The picker copy is already on screen as the placeholder; reusing it avoids a reload flash.
559+
previewUri: asset.uri,
529560
});
530561
}
531562

‎apps/mobile/src/state/thread-task-permissions.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,7 @@ vi.mock("./use-composer-drafts", async () => {
141141
return {
142142
composerDraftsAtom: Atom.make({}),
143143
composerContextImportsAtom: Atom.make({}),
144+
composerDraftBusy: () => false,
144145
getComposerDraftSnapshot: () => state.draft,
145146
clearComposerDraftContent: state.clearDraft,
146147
clearComposerDraft: state.clearDraft,

‎apps/mobile/src/state/use-composer-attachment-actions.ts‎

Lines changed: 26 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,13 @@ import {
1717
captureComposerDraftInsertion,
1818
countComposerDraftAttachmentsAfterSelection,
1919
getComposerDraftSnapshot,
20+
composerPendingMediaAtom,
21+
releaseUnusedComposerAttachmentFiles,
22+
addComposerPendingMedia,
23+
removeComposerPendingMedia,
2024
type ComposerDraftInsertion,
2125
} from "./use-composer-drafts";
26+
import { appAtomRegistry } from "./atom-registry";
2227
import { startComposerDraftOperation } from "./composerDraftOperations";
2328

2429
export function useComposerAttachmentActions(props: {
@@ -63,15 +68,29 @@ export function useComposerAttachmentActions(props: {
6368
if (!key || props.locked) return;
6469
const insertion = captureComposerDraftInsertion(key);
6570
await withDraftOperation(key, async (signal) => {
66-
const result = await pickComposerMedia({
71+
let placeholderIds: ReadonlyArray<string> = [];
72+
await pickComposerMedia({
6773
existingCount: countComposerDraftAttachmentsAfterSelection(key, insertion),
6874
maxVideoBytes: capabilities?.fileAttachments?.maxUploadBytes,
69-
});
70-
report(
71-
"Could not attach photo or video",
72-
result.error,
73-
await append(key, signal, result.attachments, insertion),
74-
);
75+
onPicked: (placeholders) => {
76+
placeholderIds = placeholders.map(({ id }) => id);
77+
addComposerPendingMedia(key, placeholders);
78+
},
79+
})
80+
.then(async (result) => {
81+
// Every attachment has a placeholder; one the user removed is no longer pending.
82+
const pending = appAtomRegistry.get(composerPendingMediaAtom)[key] ?? [];
83+
const kept = result.attachments.filter(({ id }) => pending.some((p) => p.id === id));
84+
await releaseUnusedComposerAttachmentFiles(
85+
result.attachments.filter((attachment) => !kept.includes(attachment)),
86+
);
87+
report(
88+
"Could not attach photo or video",
89+
result.error,
90+
await append(key, signal, kept, insertion),
91+
);
92+
})
93+
.finally(() => removeComposerPendingMedia(key, placeholderIds));
7594
});
7695
};
7796
const onPickDraftFiles = async () => {

‎apps/mobile/src/state/use-composer-drafts.ts‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,46 @@ export function setComposerContextImporting(draftKey: string, importing: boolean
8888
appAtomRegistry.set(composerContextImportsAtom, next);
8989
}
9090

91+
/** Placeholders for picked media still being processed, shown in the strip until their attachments land. */
92+
export const composerPendingMediaAtom = Atom.make<
93+
Record<string, ReadonlyArray<DraftComposerAttachment>>
94+
>({}).pipe(Atom.keepAlive);
95+
96+
function updateComposerPendingMedia(
97+
draftKey: string,
98+
update: (current: ReadonlyArray<DraftComposerAttachment>) => DraftComposerAttachment[],
99+
): void {
100+
const all = appAtomRegistry.get(composerPendingMediaAtom);
101+
const current = all[draftKey] ?? [];
102+
const placeholders = update(current);
103+
if (placeholders.length === current.length) return;
104+
const next = { ...all };
105+
if (placeholders.length > 0) next[draftKey] = placeholders;
106+
else delete next[draftKey];
107+
appAtomRegistry.set(composerPendingMediaAtom, next);
108+
}
109+
110+
export function addComposerPendingMedia(
111+
draftKey: string,
112+
placeholders: ReadonlyArray<DraftComposerAttachment>,
113+
): void {
114+
updateComposerPendingMedia(draftKey, (current) => [...current, ...placeholders]);
115+
}
116+
117+
export function removeComposerPendingMedia(draftKey: string, ids: ReadonlyArray<string>): void {
118+
updateComposerPendingMedia(draftKey, (current) =>
119+
current.filter((placeholder) => !ids.includes(placeholder.id)),
120+
);
121+
}
122+
123+
/** A draft can't be sent while context is importing or picked media is still processing. */
124+
export function composerDraftBusy(draftKey: string): boolean {
125+
return (
126+
appAtomRegistry.get(composerContextImportsAtom)[draftKey] === true ||
127+
appAtomRegistry.get(composerPendingMediaAtom)[draftKey] !== undefined
128+
);
129+
}
130+
91131
interface ComposerDraftSelection {
92132
readonly text: string;
93133
readonly start: number;
@@ -1419,6 +1459,7 @@ export function replaceComposerDraftAttachments(
14191459
}
14201460

14211461
export function removeComposerDraftAttachment(draftKey: string, imageId: string): void {
1462+
removeComposerPendingMedia(draftKey, [imageId]);
14221463
const previousAttachments = getComposerDraftSnapshot(draftKey).attachments;
14231464
updateComposerDrafts((current) => {
14241465
const existing = normalizeDraft(current[draftKey]);

0 commit comments

Comments
 (0)