Skip to content

Commit 62d858d

Browse files
committed
fix(mobile): handle image drops in the Android composer
Route native image drops through the existing attachment flow on supported Android versions, keeping URI permissions available for asynchronous reads. Resolve content URI MIME types and convert unsupported formats before attaching.
1 parent 3f94487 commit 62d858d

5 files changed

Lines changed: 239 additions & 5 deletions

File tree

‎apps/mobile/modules/t3-composer-editor/android/src/main/java/expo/modules/t3composereditor/T3ComposerEditorView.kt‎

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package expo.modules.t3composereditor
22

3+
import android.app.Activity
34
import android.content.ClipData
45
import android.content.ClipboardManager
56
import android.content.Context
@@ -16,6 +17,7 @@ import android.text.TextUtils
1617
import android.text.TextWatcher
1718
import android.text.style.ReplacementSpan
1819
import android.util.TypedValue
20+
import android.view.DragEvent
1921
import android.view.Gravity
2022
import android.view.GestureDetector
2123
import android.view.MotionEvent
@@ -39,7 +41,7 @@ class T3ComposerEditorView(context: Context, appContext: AppContext) : ExpoView(
3941
// React Native ignores the editor's requestLayout(), which makes TextView defer caret scrolls.
4042
override val shouldUseAndroidLayout = true
4143

42-
private val editor = SelectionAwareEditText(context)
44+
private val editor = SelectionAwareEditText(context) { appContext.currentActivity }
4345
private val defaultHighlightColor = editor.highlightColor
4446
private val defaultSelectionColor = context.resolveThemeColor(
4547
android.R.attr.colorAccent,
@@ -574,7 +576,10 @@ private fun parseTokens(value: String): List<ComposerToken> = try {
574576
emptyList()
575577
}
576578

577-
internal class SelectionAwareEditText(context: Context) : EditText(context) {
579+
internal class SelectionAwareEditText(
580+
context: Context,
581+
private val currentActivity: () -> Activity? = { context as? Activity }
582+
) : EditText(context) {
578583
var readOnly = false
579584
var selectionListener: ((Int, Int) -> Unit)? = null
580585
var pasteImagesListener: ((List<String>) -> Unit)? = null
@@ -669,6 +674,39 @@ internal class SelectionAwareEditText(context: Context) : EditText(context) {
669674
selectionListener?.invoke(selStart, selEnd)
670675
}
671676

677+
override fun onDragEvent(event: DragEvent): Boolean = when {
678+
readOnly || !isEnabled -> false
679+
event.action == DragEvent.ACTION_DROP && event.clipDescription?.hasMimeType(
680+
"image/*"
681+
) == true ->
682+
dropImages(event)
683+
else -> super.onDragEvent(event)
684+
}
685+
686+
private fun dropImages(event: DragEvent): Boolean {
687+
val clip = event.clipData
688+
val activity = currentActivity()
689+
val listener = pasteImagesListener
690+
if (clip == null || activity == null || listener == null) return false
691+
val imageUris = try {
692+
// Keep the activity-scoped grant while JavaScript asynchronously reads the dropped images.
693+
activity.requestDragAndDropPermissions(event)
694+
(0 until clip.itemCount).mapNotNull { index ->
695+
clip.getItemAt(index).uri?.takeIf { uri ->
696+
context.contentResolver.getType(uri)?.startsWith("image/") == true
697+
}?.toString()
698+
}
699+
} catch (_: SecurityException) {
700+
emptyList()
701+
}
702+
return if (imageUris.size == clip.itemCount) {
703+
listener(imageUris)
704+
true
705+
} else {
706+
false
707+
}
708+
}
709+
672710
override fun onTextContextMenuItem(id: Int): Boolean {
673711
val pasting = id == android.R.id.paste || id == android.R.id.pasteAsPlainText
674712
if (readOnly && (id == android.R.id.cut || pasting)) {

‎apps/mobile/modules/t3-composer-editor/android/src/test/java/expo/modules/t3composereditor/ComposerPasteTest.kt‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,27 @@
11
package expo.modules.t3composereditor
22

3+
import android.app.Activity
4+
import android.content.ContentProvider
5+
import android.content.ContentValues
6+
import android.database.Cursor
37
import android.content.ClipData
48
import android.content.ClipDescription
59
import android.content.ClipboardManager
610
import android.content.Context
11+
import android.content.pm.ProviderInfo
712
import android.net.Uri
13+
import android.view.DragEvent
14+
import android.view.View
815
import android.view.KeyEvent
916
import org.junit.Assert.assertEquals
1017
import org.junit.Assert.assertFalse
1118
import org.junit.Assert.assertNull
1219
import org.junit.Assert.assertTrue
1320
import org.junit.Test
1421
import org.junit.runner.RunWith
22+
import org.robolectric.Robolectric
23+
import org.robolectric.shadows.DragEventBuilder
24+
import org.robolectric.shadows.ShadowContentResolver
1525
import org.robolectric.RobolectricTestRunner
1626
import org.robolectric.RuntimeEnvironment
1727
import org.robolectric.annotation.Config
@@ -169,4 +179,140 @@ class ComposerPasteTest {
169179
assertEquals("unchanged", editor.text.toString())
170180
assertNull(intercepted)
171181
}
182+
183+
private fun dropEditor(): SelectionAwareEditText {
184+
val activity = Robolectric.buildActivity(Activity::class.java).setup().get()
185+
return SelectionAwareEditText(activity).apply {
186+
activity.setContentView(this)
187+
isFocusableInTouchMode = true
188+
setText("before")
189+
setSelection(length())
190+
measure(
191+
View.MeasureSpec.makeMeasureSpec(500, View.MeasureSpec.EXACTLY),
192+
View.MeasureSpec.makeMeasureSpec(200, View.MeasureSpec.EXACTLY),
193+
)
194+
layout(0, 0, 500, 200)
195+
requestFocus()
196+
}
197+
}
198+
199+
private fun imageClip(vararg names: String): ClipData {
200+
val provider = DropImageProvider().apply {
201+
attachInfo(this@ComposerPasteTest.context, ProviderInfo().apply { authority = "images" })
202+
}
203+
ShadowContentResolver.registerProviderInternal("images", provider)
204+
return ClipData(
205+
"images",
206+
arrayOf("image/png", "image/jpeg"),
207+
ClipData.Item(Uri.parse("content://images/${names.first()}")),
208+
).apply {
209+
names.drop(1).forEach { addItem(ClipData.Item(Uri.parse("content://images/$it"))) }
210+
}
211+
}
212+
213+
private fun dragEvent(clip: ClipData, action: Int = DragEvent.ACTION_DROP): DragEvent =
214+
DragEventBuilder.newBuilder()
215+
.setAction(action)
216+
.setX(400f)
217+
.setY(10f)
218+
.setClipDescription(clip.description)
219+
.setClipData(if (action == DragEvent.ACTION_DROP) clip else null)
220+
.build()
221+
222+
@Test
223+
@Config(sdk = [24, 36])
224+
fun imageDropAttachesEveryImageOnceWithoutInsertingUriText() {
225+
val target = dropEditor()
226+
val received = mutableListOf<List<String>>()
227+
target.pasteImagesListener = { received.add(it) }
228+
val clip = imageClip("screenshot.png", "photo.jpg")
229+
230+
assertTrue(target.onDragEvent(dragEvent(clip, DragEvent.ACTION_DRAG_STARTED)))
231+
assertTrue(target.onDragEvent(dragEvent(clip)))
232+
assertEquals(
233+
listOf(listOf("content://images/screenshot.png", "content://images/photo.jpg")),
234+
received
235+
)
236+
assertEquals("before", target.text.toString())
237+
}
238+
239+
@Test
240+
@Config(sdk = [24, 36])
241+
fun imageDropIsRejectedWhenTheComposerLocksDuringTheDrag() {
242+
val target = dropEditor()
243+
target.pasteImagesListener = { error("A locked composer must not receive images") }
244+
val clip = imageClip("screenshot.png")
245+
assertTrue(target.onDragEvent(dragEvent(clip, DragEvent.ACTION_DRAG_STARTED)))
246+
target.readOnly = true
247+
248+
assertFalse(target.onDragEvent(dragEvent(clip)))
249+
assertEquals("before", target.text.toString())
250+
}
251+
252+
@Test
253+
@Config(sdk = [24, 36])
254+
fun disabledComposerRejectsImageDrags() {
255+
val target = dropEditor()
256+
target.isEnabled = false
257+
target.pasteImagesListener = { error("A disabled composer must not receive images") }
258+
val clip = imageClip("screenshot.png")
259+
260+
assertFalse(target.onDragEvent(dragEvent(clip, DragEvent.ACTION_DRAG_STARTED)))
261+
assertFalse(target.onDragEvent(dragEvent(clip)))
262+
assertEquals("before", target.text.toString())
263+
}
264+
265+
@Test
266+
@Config(sdk = [24, 36])
267+
fun mixedImageAndTextDropDoesNotSilentlyDiscardItems() {
268+
val target = dropEditor()
269+
target.pasteImagesListener = { error("Mixed drops must not partially attach") }
270+
val clip = imageClip("screenshot.png").apply { addItem(ClipData.Item("hello")) }
271+
272+
assertFalse(target.onDragEvent(dragEvent(clip)))
273+
assertEquals("before", target.text.toString())
274+
}
275+
276+
@Test
277+
@Config(sdk = [24, 36])
278+
fun unreadableImageDropDoesNotCrashOrInsertUriText() {
279+
val target = dropEditor()
280+
target.pasteImagesListener = { error("Unreadable images must not attach") }
281+
282+
assertFalse(target.onDragEvent(dragEvent(imageClip("denied.png"))))
283+
assertEquals("before", target.text.toString())
284+
}
285+
286+
@Test
287+
@Config(sdk = [24, 36])
288+
fun ordinaryTextDropsKeepNativeInsertion() {
289+
val target = dropEditor()
290+
target.pasteImagesListener = { error("Text drops must stay native") }
291+
292+
assertTrue(target.onDragEvent(dragEvent(ClipData.newPlainText("text", "hello"))))
293+
assertEquals("beforehello", target.text.toString())
294+
}
295+
296+
private class DropImageProvider : ContentProvider() {
297+
override fun onCreate() = true
298+
override fun getType(uri: Uri): String {
299+
if (uri.lastPathSegment == "denied.png") throw SecurityException("No image read grant")
300+
return if (uri.lastPathSegment?.endsWith(".jpg") == true) "image/jpeg" else "image/png"
301+
}
302+
override fun query(
303+
uri: Uri,
304+
projection: Array<out String>?,
305+
selection: String?,
306+
selectionArgs: Array<out String>?,
307+
sortOrder: String?
308+
): Cursor? = null
309+
override fun insert(uri: Uri, values: ContentValues?): Uri? = null
310+
override fun delete(uri: Uri, selection: String?, selectionArgs: Array<out String>?) = 0
311+
override fun update(
312+
uri: Uri,
313+
values: ContentValues?,
314+
selection: String?,
315+
selectionArgs: Array<out String>?
316+
) = 0
317+
}
172318
}

‎apps/mobile/src/lib/composerImages.test.ts‎

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,10 @@ import {
44
PROVIDER_SEND_TURN_MAX_IMAGE_BYTES,
55
} from "@t3tools/contracts";
66

7-
const files = new Map<string, { base64: string; deleted: boolean; text?: string }>();
7+
const files = new Map<
8+
string,
9+
{ base64: string; deleted: boolean; text?: string; mimeType?: string }
10+
>();
811

912
const clipboard = vi.hoisted(() => ({
1013
hasImageAsync: vi.fn(),
@@ -60,6 +63,10 @@ vi.mock("expo-file-system", () => ({
6063
return files.has(this.uri) && files.get(this.uri)?.deleted === false;
6164
}
6265

66+
get type(): string {
67+
return files.get(this.uri)?.mimeType ?? "";
68+
}
69+
6370
async base64(): Promise<string> {
6471
const entry = files.get(this.uri);
6572
if (!entry || entry.deleted) {
@@ -193,6 +200,40 @@ describe("native pasted image cleanup", () => {
193200
files.clear();
194201
});
195202

203+
it("preserves the reported image type of extensionless Android content URIs", async () => {
204+
const uri = "content://media/external/images/media/42";
205+
files.set(uri, { base64: "aGVsbG8=", deleted: false, mimeType: "image/jpeg" });
206+
207+
const attachments = await convertPastedImagesToAttachments({ uris: [uri], existingCount: 0 });
208+
209+
expect(attachments).toEqual([
210+
expect.objectContaining({
211+
name: "pasted-image.jpeg",
212+
mimeType: "image/jpeg",
213+
dataUrl: "data:image/jpeg;base64,aGVsbG8=",
214+
previewUri: uri,
215+
}),
216+
]);
217+
expect(files.get(uri)?.deleted).toBe(false);
218+
});
219+
220+
it("renders an unsupported Android image format to JPEG before attaching it", async () => {
221+
const uri = "content://media/external/images/media/43";
222+
files.set(uri, { base64: "b3JpZ2luYWw=", deleted: false, mimeType: "image/heic" });
223+
manipulator.state.sourceSize = { width: 100, height: 100 };
224+
225+
const attachments = await convertPastedImagesToAttachments({ uris: [uri], existingCount: 0 });
226+
227+
expect(attachments).toEqual([
228+
expect.objectContaining({
229+
name: "pasted-image.jpeg",
230+
mimeType: "image/jpeg",
231+
dataUrl: "data:image/jpeg;base64,cmVuZGVyZWQ=",
232+
}),
233+
]);
234+
expect(files.get(uri)?.deleted).toBe(false);
235+
});
236+
196237
it("converts owned files to data-backed previews and deletes the source", async () => {
197238
const uri =
198239
"file:///private/var/mobile/Containers/Data/Application/app/tmp/t3-composer-paste/id.png";

‎apps/mobile/src/lib/composerImages.ts‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -685,8 +685,15 @@ export async function convertPastedImagesToAttachments(input: {
685685
continue;
686686
}
687687
const file = new File(uri);
688-
const mimeType = mimeTypeFromUri(uri);
689-
const pastedBase64 = await file.base64();
688+
const reportedMimeType = file.type;
689+
const sourceMimeType = reportedMimeType?.startsWith("image/")
690+
? reportedMimeType
691+
: mimeTypeFromUri(uri);
692+
const mimeType = isProviderSendTurnSupportedImageMimeType(sourceMimeType)
693+
? sourceMimeType
694+
: "image/jpeg";
695+
const pastedBase64 =
696+
mimeType === sourceMimeType ? await file.base64() : (await renderPhoto(uri)).base64;
690697
const fitted = await fitComposerImage({ uri, mimeType, base64: pastedBase64 });
691698
const base64 = fitted?.base64 ?? pastedBase64;
692699
const sizeBytes = estimateBase64ByteSize(base64);

‎docs/user/composer.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,8 @@ You can drag or paste images into the web or desktop composer. HEIC and HEIF
4646
photos are converted to JPEG there and when selected from the mobile photo
4747
library; photos over the image limit are also resized to fit. On mobile, you can
4848
also send files to T3 Code through another app's system share sheet.
49+
On Android, you can also drag images from another app into the message input
50+
in split screen.
4951

5052
See [images and videos](#images-and-videos-in-messages) for previewing and saving media.
5153

0 commit comments

Comments
 (0)