Skip to content

Commit 9ee7da6

Browse files
12yamaria-rcks
andauthored
fix(mobile): handle image drops in the Android composer (#25573)
Co-authored-by: maria-rcks <[email protected]>
1 parent 5ff1482 commit 9ee7da6

5 files changed

Lines changed: 272 additions & 16 deletions

File tree

‎apps/mobile/modules/t3-composer-editor/android/src/main/java/expo/modules/t3composereditor/T3ComposerEditorView.kt‎

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package expo.modules.t3composereditor
22

3+
import android.app.Activity
34
import android.content.ClipData
45
import android.content.ClipboardManager
56
import android.content.Context
@@ -16,6 +17,7 @@ import android.text.TextUtils
1617
import android.text.TextWatcher
1718
import android.text.style.ReplacementSpan
1819
import android.util.TypedValue
20+
import android.view.DragEvent
1921
import android.view.Gravity
2022
import android.view.GestureDetector
2123
import android.view.MotionEvent
@@ -39,7 +41,7 @@ class T3ComposerEditorView(context: Context, appContext: AppContext) : ExpoView(
3941
// React Native ignores the editor's requestLayout(), which makes TextView defer caret scrolls.
4042
override val shouldUseAndroidLayout = true
4143

42-
private val editor = SelectionAwareEditText(context)
44+
private val editor = SelectionAwareEditText(context) { appContext.currentActivity }
4345
private val defaultHighlightColor = editor.highlightColor
4446
private val defaultSelectionColor = context.resolveThemeColor(
4547
android.R.attr.colorAccent,
@@ -574,7 +576,10 @@ private fun parseTokens(value: String): List<ComposerToken> = try {
574576
emptyList()
575577
}
576578

577-
internal class SelectionAwareEditText(context: Context) : EditText(context) {
579+
internal class SelectionAwareEditText(
580+
context: Context,
581+
private val currentActivity: () -> Activity? = { context as? Activity }
582+
) : EditText(context) {
578583
var readOnly = false
579584
var selectionListener: ((Int, Int) -> Unit)? = null
580585
var pasteImagesListener: ((List<String>) -> Unit)? = null
@@ -669,6 +674,39 @@ internal class SelectionAwareEditText(context: Context) : EditText(context) {
669674
selectionListener?.invoke(selStart, selEnd)
670675
}
671676

677+
override fun onDragEvent(event: DragEvent): Boolean = when {
678+
readOnly || !isEnabled -> false
679+
event.action == DragEvent.ACTION_DROP && event.clipDescription?.hasMimeType(
680+
"image/*"
681+
) == true ->
682+
dropImages(event)
683+
else -> super.onDragEvent(event)
684+
}
685+
686+
private fun dropImages(event: DragEvent): Boolean {
687+
val clip = event.clipData
688+
val activity = currentActivity()
689+
val listener = pasteImagesListener
690+
if (clip == null || activity == null || listener == null) return false
691+
val imageUris = try {
692+
// Keep the activity-scoped grant while JavaScript asynchronously reads the dropped images.
693+
activity.requestDragAndDropPermissions(event)
694+
(0 until clip.itemCount).mapNotNull { index ->
695+
clip.getItemAt(index).uri?.takeIf { uri ->
696+
context.contentResolver.getType(uri)?.startsWith("image/") == true
697+
}?.toString()
698+
}
699+
} catch (_: SecurityException) {
700+
emptyList()
701+
}
702+
return if (imageUris.size == clip.itemCount) {
703+
listener(imageUris)
704+
true
705+
} else {
706+
false
707+
}
708+
}
709+
672710
override fun onTextContextMenuItem(id: Int): Boolean {
673711
val pasting = id == android.R.id.paste || id == android.R.id.pasteAsPlainText
674712
if (readOnly && (id == android.R.id.cut || pasting)) {

‎apps/mobile/modules/t3-composer-editor/android/src/test/java/expo/modules/t3composereditor/ComposerPasteTest.kt‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,27 @@
11
package expo.modules.t3composereditor
22

3+
import android.app.Activity
4+
import android.content.ContentProvider
5+
import android.content.ContentValues
6+
import android.database.Cursor
37
import android.content.ClipData
48
import android.content.ClipDescription
59
import android.content.ClipboardManager
610
import android.content.Context
11+
import android.content.pm.ProviderInfo
712
import android.net.Uri
13+
import android.view.DragEvent
14+
import android.view.View
815
import android.view.KeyEvent
916
import org.junit.Assert.assertEquals
1017
import org.junit.Assert.assertFalse
1118
import org.junit.Assert.assertNull
1219
import org.junit.Assert.assertTrue
1320
import org.junit.Test
1421
import org.junit.runner.RunWith
22+
import org.robolectric.Robolectric
23+
import org.robolectric.shadows.DragEventBuilder
24+
import org.robolectric.shadows.ShadowContentResolver
1525
import org.robolectric.RobolectricTestRunner
1626
import org.robolectric.RuntimeEnvironment
1727
import org.robolectric.annotation.Config
@@ -169,4 +179,140 @@ class ComposerPasteTest {
169179
assertEquals("unchanged", editor.text.toString())
170180
assertNull(intercepted)
171181
}
182+
183+
private fun dropEditor(): SelectionAwareEditText {
184+
val activity = Robolectric.buildActivity(Activity::class.java).setup().get()
185+
return SelectionAwareEditText(activity).apply {
186+
activity.setContentView(this)
187+
isFocusableInTouchMode = true
188+
setText("before")
189+
setSelection(length())
190+
measure(
191+
View.MeasureSpec.makeMeasureSpec(500, View.MeasureSpec.EXACTLY),
192+
View.MeasureSpec.makeMeasureSpec(200, View.MeasureSpec.EXACTLY),
193+
)
194+
layout(0, 0, 500, 200)
195+
requestFocus()
196+
}
197+
}
198+
199+
private fun imageClip(vararg names: String): ClipData {
200+
val provider = DropImageProvider().apply {
201+
attachInfo(this@ComposerPasteTest.context, ProviderInfo().apply { authority = "images" })
202+
}
203+
ShadowContentResolver.registerProviderInternal("images", provider)
204+
return ClipData(
205+
"images",
206+
arrayOf("image/png", "image/jpeg"),
207+
ClipData.Item(Uri.parse("content://images/${names.first()}")),
208+
).apply {
209+
names.drop(1).forEach { addItem(ClipData.Item(Uri.parse("content://images/$it"))) }
210+
}
211+
}
212+
213+
private fun dragEvent(clip: ClipData, action: Int = DragEvent.ACTION_DROP): DragEvent =
214+
DragEventBuilder.newBuilder()
215+
.setAction(action)
216+
.setX(400f)
217+
.setY(10f)
218+
.setClipDescription(clip.description)
219+
.setClipData(if (action == DragEvent.ACTION_DROP) clip else null)
220+
.build()
221+
222+
@Test
223+
@Config(sdk = [24, 36])
224+
fun imageDropAttachesEveryImageOnceWithoutInsertingUriText() {
225+
val target = dropEditor()
226+
val received = mutableListOf<List<String>>()
227+
target.pasteImagesListener = { received.add(it) }
228+
val clip = imageClip("screenshot.png", "photo.jpg")
229+
230+
assertTrue(target.onDragEvent(dragEvent(clip, DragEvent.ACTION_DRAG_STARTED)))
231+
assertTrue(target.onDragEvent(dragEvent(clip)))
232+
assertEquals(
233+
listOf(listOf("content://images/screenshot.png", "content://images/photo.jpg")),
234+
received
235+
)
236+
assertEquals("before", target.text.toString())
237+
}
238+
239+
@Test
240+
@Config(sdk = [24, 36])
241+
fun imageDropIsRejectedWhenTheComposerLocksDuringTheDrag() {
242+
val target = dropEditor()
243+
target.pasteImagesListener = { error("A locked composer must not receive images") }
244+
val clip = imageClip("screenshot.png")
245+
assertTrue(target.onDragEvent(dragEvent(clip, DragEvent.ACTION_DRAG_STARTED)))
246+
target.readOnly = true
247+
248+
assertFalse(target.onDragEvent(dragEvent(clip)))
249+
assertEquals("before", target.text.toString())
250+
}
251+
252+
@Test
253+
@Config(sdk = [24, 36])
254+
fun disabledComposerRejectsImageDrags() {
255+
val target = dropEditor()
256+
target.isEnabled = false
257+
target.pasteImagesListener = { error("A disabled composer must not receive images") }
258+
val clip = imageClip("screenshot.png")
259+
260+
assertFalse(target.onDragEvent(dragEvent(clip, DragEvent.ACTION_DRAG_STARTED)))
261+
assertFalse(target.onDragEvent(dragEvent(clip)))
262+
assertEquals("before", target.text.toString())
263+
}
264+
265+
@Test
266+
@Config(sdk = [24, 36])
267+
fun mixedImageAndTextDropDoesNotSilentlyDiscardItems() {
268+
val target = dropEditor()
269+
target.pasteImagesListener = { error("Mixed drops must not partially attach") }
270+
val clip = imageClip("screenshot.png").apply { addItem(ClipData.Item("hello")) }
271+
272+
assertFalse(target.onDragEvent(dragEvent(clip)))
273+
assertEquals("before", target.text.toString())
274+
}
275+
276+
@Test
277+
@Config(sdk = [24, 36])
278+
fun unreadableImageDropDoesNotCrashOrInsertUriText() {
279+
val target = dropEditor()
280+
target.pasteImagesListener = { error("Unreadable images must not attach") }
281+
282+
assertFalse(target.onDragEvent(dragEvent(imageClip("denied.png"))))
283+
assertEquals("before", target.text.toString())
284+
}
285+
286+
@Test
287+
@Config(sdk = [24, 36])
288+
fun ordinaryTextDropsKeepNativeInsertion() {
289+
val target = dropEditor()
290+
target.pasteImagesListener = { error("Text drops must stay native") }
291+
292+
assertTrue(target.onDragEvent(dragEvent(ClipData.newPlainText("text", "hello"))))
293+
assertEquals("beforehello", target.text.toString())
294+
}
295+
296+
private class DropImageProvider : ContentProvider() {
297+
override fun onCreate() = true
298+
override fun getType(uri: Uri): String {
299+
if (uri.lastPathSegment == "denied.png") throw SecurityException("No image read grant")
300+
return if (uri.lastPathSegment?.endsWith(".jpg") == true) "image/jpeg" else "image/png"
301+
}
302+
override fun query(
303+
uri: Uri,
304+
projection: Array<out String>?,
305+
selection: String?,
306+
selectionArgs: Array<out String>?,
307+
sortOrder: String?
308+
): Cursor? = null
309+
override fun insert(uri: Uri, values: ContentValues?): Uri? = null
310+
override fun delete(uri: Uri, selection: String?, selectionArgs: Array<out String>?) = 0
311+
override fun update(
312+
uri: Uri,
313+
values: ContentValues?,
314+
selection: String?,
315+
selectionArgs: Array<out String>?
316+
) = 0
317+
}
172318
}

‎apps/mobile/src/lib/composerImages.test.ts‎

Lines changed: 66 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,13 @@ import {
44
PROVIDER_SEND_TURN_MAX_IMAGE_BYTES,
55
} from "@t3tools/contracts";
66

7-
const files = new Map<string, { base64: string; deleted: boolean; text?: string }>();
7+
const files = new Map<
8+
string,
9+
{ base64: string; deleted: boolean; text?: string; mimeType?: string }
10+
>();
811

912
const manipulator = vi.hoisted(() => {
10-
const state = { sourceSize: { width: 0, height: 0 } };
13+
const state = { sourceSize: { width: 0, height: 0 }, renderedBase64: "cmVuZGVyZWQ=" };
1114
const manipulate = vi.fn((source: string | { width: number; height: number }) => {
1215
let size = typeof source === "string" ? state.sourceSize : source;
1316
const context = {
@@ -18,10 +21,11 @@ const manipulator = vi.hoisted(() => {
1821
renderAsync: vi.fn(async () => ({
1922
...size,
2023
release: vi.fn(),
21-
saveAsync: vi.fn(async (options: { format: string }) => ({
22-
uri: `file:///cache/rendered.${options.format}`,
23-
base64: `cmVuZGVyZWQ=`,
24-
})),
24+
saveAsync: vi.fn(async (options: { format: string }) => {
25+
const uri = `file:///cache/rendered.${options.format}`;
26+
files.set(uri, { base64: state.renderedBase64, deleted: false });
27+
return { uri, base64: state.renderedBase64 };
28+
}),
2529
})),
2630
};
2731
return context;
@@ -51,6 +55,10 @@ vi.mock("expo-file-system", () => ({
5155
return files.has(this.uri) && files.get(this.uri)?.deleted === false;
5256
}
5357

58+
get type(): string {
59+
return files.get(this.uri)?.mimeType ?? "";
60+
}
61+
5462
async base64(): Promise<string> {
5563
const entry = files.get(this.uri);
5664
if (!entry || entry.deleted) {
@@ -102,6 +110,58 @@ import { convertPastedImagesToAttachments } from "./composerImages";
102110
describe("native pasted image cleanup", () => {
103111
beforeEach(() => {
104112
files.clear();
113+
manipulator.state.renderedBase64 = "cmVuZGVyZWQ=";
114+
});
115+
116+
it("preserves the reported image type of extensionless Android content URIs", async () => {
117+
const uri = "content://media/external/images/media/42";
118+
files.set(uri, { base64: "aGVsbG8=", deleted: false, mimeType: "image/jpeg" });
119+
120+
const attachments = await convertPastedImagesToAttachments({ uris: [uri], existingCount: 0 });
121+
122+
expect(attachments).toEqual([
123+
expect.objectContaining({
124+
name: "pasted-image.jpeg",
125+
mimeType: "image/jpeg",
126+
dataUrl: "data:image/jpeg;base64,aGVsbG8=",
127+
previewUri: "data:image/jpeg;base64,aGVsbG8=",
128+
}),
129+
]);
130+
expect(files.get(uri)?.deleted).toBe(false);
131+
});
132+
133+
it("renders unsupported images to JPEG and removes the cache file after attaching", async () => {
134+
const uri = "content://media/external/images/media/43";
135+
files.set(uri, { base64: "b3JpZ2luYWw=", deleted: false, mimeType: "image/heic" });
136+
manipulator.state.sourceSize = { width: 100, height: 100 };
137+
138+
const attachments = await convertPastedImagesToAttachments({ uris: [uri], existingCount: 0 });
139+
140+
expect(attachments).toEqual([
141+
expect.objectContaining({
142+
name: "pasted-image.jpeg",
143+
mimeType: "image/jpeg",
144+
dataUrl: "data:image/jpeg;base64,cmVuZGVyZWQ=",
145+
previewUri: "data:image/jpeg;base64,cmVuZGVyZWQ=",
146+
}),
147+
]);
148+
expect(files.get("file:///cache/rendered.jpeg")?.deleted).toBe(true);
149+
expect(files.get(uri)?.deleted).toBe(false);
150+
});
151+
152+
it("removes the rendered JPEG when its bytes exceed the image limit", async () => {
153+
const uri = "content://media/external/images/media/43";
154+
files.set(uri, { base64: "b3JpZ2luYWw=", deleted: false, mimeType: "image/heic" });
155+
manipulator.state.sourceSize = { width: 100, height: 100 };
156+
manipulator.state.renderedBase64 = "A".repeat(
157+
Math.ceil((PROVIDER_SEND_TURN_MAX_IMAGE_BYTES + 1) / 3) * 4,
158+
);
159+
160+
const attachments = await convertPastedImagesToAttachments({ uris: [uri], existingCount: 0 });
161+
162+
expect(attachments).toEqual([]);
163+
expect(files.get("file:///cache/rendered.jpeg")?.deleted).toBe(true);
164+
expect(files.get(uri)?.deleted).toBe(false);
105165
});
106166

107167
it("converts owned files to data-backed previews and deletes the source", async () => {

0 commit comments

Comments
 (0)