Skip to content

Commit 3394d5b

Browse files
authored
fix(android): keep a scroll's swipe out of the IME window (#2514)
* fix(android): keep a scroll's swipe out of the IME window Android is the case the clip exists for beyond iOS: an `adjustPan` or `adjustNothing` activity keeps a window whose recorded bounds already run under the IME, so a plan built from them aims at the keys. The helper now reports the largest input method window beside the application window, in absolute screen pixels like the window next to it, and `scroll` clips its band with the shared rule or refuses when the keyboard owns the surface. An older helper reports no keyboard keys, and a provider-supplied viewport has no IME channel at all. Both read as "nothing to avoid", which is what the shared rule already does with a missing frame; neither turns into a refusal. `UiAutomation.getWindows()` answers with an empty list until the service asks for interactive windows, so the read applies the seam the tree capture already uses rather than depending on a snapshot capture having run first in the same instrumentation; the one-shot fallback below it has no such neighbour. Measured on a Pixel 7 emulator with an `adjust=pan` contact editor, the application window keeps its full 2400px height while the IME window reports `[0,1517][1080,2400]`, and `scroll down` answers with `referenceHeight: 1505`, `keyboardMinY: 1517`, `keyboardAvoided: true` and a swipe ending at 301 instead of starting at 1920 under the keys. * fix(android): clear the composer, not just the key plane, before a scroll swipes The helper kept the largest `TYPE_INPUT_METHOD` rectangle as the keyboard. A composer bar and its key plane can arrive as separate windows and the key plane is the larger one, so the earlier top edge was discarded and the clipped band still ended inside the composer: the swipe landed on keys the rule exists to keep it off. The read now copies every input method window and unions the ones the swipe's centre line crosses, which is the same line the shared clip rule tests. A candidate strip at the edge of the screen that the swipe can never reach no longer shortens the band either. The selection runs on plain window edges, because `Rect` is a device type whose constructors throw off-device, so the two-window case is a unit test rather than a simulator-only path. * refactor(android): pass the measured occlusion explicitly and drop the duplicate rect check The clip rule already fails open on a keyboard frame it cannot measure, so the helper reader no longer needs its own copy of the check, and the gesture-viewport validator goes back to its original body. The refusal names its three numbers instead of spreading the clip variant, so the discriminator never leaks into error details.
1 parent ab0c7a4 commit 3394d5b

13 files changed

Lines changed: 554 additions & 42 deletions

File tree

‎android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/GestureViewportReader.java‎

Lines changed: 106 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,61 @@
44
import android.graphics.Rect;
55
import android.view.accessibility.AccessibilityNodeInfo;
66
import android.view.accessibility.AccessibilityWindowInfo;
7+
import java.util.ArrayList;
78
import java.util.List;
89
import java.util.concurrent.TimeoutException;
910

10-
/** Resolves the active application window bounds used to validate planned gestures. */
11+
/**
12+
* Resolves the active application window bounds used to validate planned gestures, plus the input
13+
* method window's bounds when one is on screen. The keyboard half is what lets a scroll keep its
14+
* swipe above the keys instead of flinging into them (#2500): the same {@code getWindows()} pass
15+
* already lists {@code TYPE_INPUT_METHOD}, so reading it costs no extra automation round trip, and
16+
* it is the live window list rather than a cached frame.
17+
*/
1118
final class GestureViewportReader {
1219
private GestureViewportReader() {}
1320

21+
/** The application viewport a gesture may target, and the IME's share of the screen, if any. */
22+
static final class Reading {
23+
final Rect application;
24+
/** Null when no input method window is on screen; an unmeasurable keyboard is not occlusion. */
25+
final Rect inputMethod;
26+
27+
Reading(Rect application, Rect inputMethod) {
28+
this.application = application;
29+
this.inputMethod = inputMethod;
30+
}
31+
}
32+
33+
/**
34+
* One reported window's edges in screen pixels. Plain fields because {@code Rect} is a device type
35+
* whose constructors throw off-device, and which of several input method windows a swipe strikes is
36+
* arithmetic that has to be testable without one.
37+
*/
38+
static final class WindowEdges {
39+
final int left;
40+
final int top;
41+
final int right;
42+
final int bottom;
43+
44+
WindowEdges(int left, int top, int right, int bottom) {
45+
this.left = left;
46+
this.top = top;
47+
this.right = right;
48+
this.bottom = bottom;
49+
}
50+
51+
static WindowEdges of(Rect rect) {
52+
return new WindowEdges(rect.left, rect.top, rect.right, rect.bottom);
53+
}
54+
55+
Rect toRect() {
56+
return new Rect(left, top, right, bottom);
57+
}
58+
}
59+
1460
@SuppressWarnings("deprecation")
15-
static Rect read(UiAutomation automation) {
61+
static Reading readReading(UiAutomation automation) {
1662
try {
1763
automation.waitForIdle(100, 2_000);
1864
} catch (TimeoutException ignored) {
@@ -21,12 +67,27 @@ static Rect read(UiAutomation automation) {
2167
// UiAutomation.getWindows() transfers recyclable AccessibilityWindowInfo instances, and this
2268
// read runs repeatedly inside the persistent helper session: copy the bounds the precedence
2369
// below needs, then recycle every window before resolving.
70+
// UiAutomation.getWindows() answers with an empty list until interactive retrieval is on, which
71+
// is the same seam the tree capture already uses. Without it this read sees no windows at all and
72+
// the keyboard below is invisible to it.
73+
AccessibilityTreeCapture.enableInteractiveWindowRetrieval(automation);
2474
Rect activeBounds = null;
2575
Rect fallbackBounds = null;
76+
List<WindowEdges> inputMethodWindows = new ArrayList<>();
2677
List<AccessibilityWindowInfo> windows = automation.getWindows();
2778
try {
2879
for (AccessibilityWindowInfo window : windows) {
29-
if (window.getType() != AccessibilityWindowInfo.TYPE_APPLICATION) continue;
80+
int type = window.getType();
81+
if (type == AccessibilityWindowInfo.TYPE_INPUT_METHOD) {
82+
// Copy every input method window. Which of them a swipe has to clear depends on the
83+
// application window, which this loop has not finished reading, so they are collected here
84+
// and resolved once it has.
85+
Rect bounds = new Rect();
86+
window.getBoundsInScreen(bounds);
87+
if (!bounds.isEmpty()) inputMethodWindows.add(WindowEdges.of(bounds));
88+
continue;
89+
}
90+
if (type != AccessibilityWindowInfo.TYPE_APPLICATION) continue;
3091
Rect bounds = new Rect();
3192
window.getBoundsInScreen(bounds);
3293
if (activeBounds == null
@@ -41,6 +102,48 @@ static Rect read(UiAutomation automation) {
41102
window.recycle();
42103
}
43104
}
105+
Rect application = resolveApplication(automation, activeBounds, fallbackBounds);
106+
WindowEdges struck = struckInputMethod(
107+
inputMethodWindows, application == null ? null : WindowEdges.of(application));
108+
return new Reading(application, struck == null ? null : struck.toRect());
109+
}
110+
111+
/**
112+
* The input method share a swipe has to stay above, or null when none of it is in the way.
113+
*
114+
* <p>A composer bar and its key plane can arrive as separate windows, and the larger rectangle is
115+
* usually the lower key plane: keeping only that leaves the swipe inside the composer reaching
116+
* further up the screen. So this unions the windows the swipe's centre line crosses — the same line
117+
* the shared clip rule tests — and ignores the ones beside it that the swipe cannot reach.
118+
*/
119+
static WindowEdges struckInputMethod(List<WindowEdges> inputMethodWindows, WindowEdges application) {
120+
WindowEdges struck = null;
121+
for (WindowEdges bounds : inputMethodWindows) {
122+
if (application != null) {
123+
double swipeCenterX = application.left + (application.right - application.left) / 2.0;
124+
boolean strikesSwipePath = swipeCenterX >= bounds.left && swipeCenterX < bounds.right;
125+
boolean overlapsWindow = bounds.bottom > application.top && bounds.top < application.bottom;
126+
if (!strikesSwipePath || !overlapsWindow) continue;
127+
}
128+
if (struck == null) {
129+
struck = bounds;
130+
continue;
131+
}
132+
struck = new WindowEdges(
133+
Math.min(struck.left, bounds.left),
134+
Math.min(struck.top, bounds.top),
135+
Math.max(struck.right, bounds.right),
136+
Math.max(struck.bottom, bounds.bottom));
137+
}
138+
return struck;
139+
}
140+
141+
static Rect read(UiAutomation automation) {
142+
return readReading(automation).application;
143+
}
144+
145+
private static Rect resolveApplication(
146+
UiAutomation automation, Rect activeBounds, Rect fallbackBounds) {
44147
if (activeBounds != null) return activeBounds;
45148
AccessibilityNodeInfo activeRoot = automation.getRootInActiveWindow();
46149
if (activeRoot != null) {

‎android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/TouchCommandHandler.java‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,11 @@ final class TouchCommandHandler {
1515
private TouchCommandHandler() {}
1616

1717
static void populateViewport(Bundle result, UiAutomation automation) {
18-
Rect viewport = GestureViewportReader.read(automation);
18+
GestureViewportReader.Reading reading = GestureViewportReader.readReading(automation);
1919
result.putString("ok", "true");
2020
result.putString("kind", "viewport");
21-
putViewportMetadata(result, viewport);
21+
putViewportMetadata(result, reading.application);
22+
putKeyboardMetadata(result, reading.inputMethod);
2223
}
2324

2425
static void populateGesture(Bundle result, UiAutomation automation, String payloadBase64)
@@ -69,4 +70,17 @@ private static void putViewportMetadata(Bundle result, Rect viewport) {
6970
result.putString("width", Integer.toString(viewport.width()));
7071
result.putString("height", Integer.toString(viewport.height()));
7172
}
73+
74+
/**
75+
* The input method window's screen bounds, reported only when one is on screen. An absent keyboard
76+
* is reported by absence: a keyboard this helper cannot see is not evidence that a surface is
77+
* blocked, so the caller must not read a zero frame as an occlusion.
78+
*/
79+
private static void putKeyboardMetadata(Bundle result, Rect inputMethod) {
80+
if (inputMethod == null) return;
81+
result.putString("keyboardX", Integer.toString(inputMethod.left));
82+
result.putString("keyboardY", Integer.toString(inputMethod.top));
83+
result.putString("keyboardWidth", Integer.toString(inputMethod.width()));
84+
result.putString("keyboardHeight", Integer.toString(inputMethod.height()));
85+
}
7286
}
Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
package com.callstack.agentdevice.snapshothelper;
2+
3+
import java.util.Arrays;
4+
import java.util.Collections;
5+
import java.util.List;
6+
7+
public final class GestureViewportReaderTest {
8+
private GestureViewportReaderTest() {}
9+
10+
private static final GestureViewportReader.WindowEdges APPLICATION =
11+
new GestureViewportReader.WindowEdges(0, 0, 1080, 2400);
12+
private static final GestureViewportReader.WindowEdges KEY_PLANE =
13+
new GestureViewportReader.WindowEdges(0, 1517, 1080, 2400);
14+
private static final GestureViewportReader.WindowEdges COMPOSER =
15+
new GestureViewportReader.WindowEdges(0, 1400, 1080, 1517);
16+
private static final GestureViewportReader.WindowEdges SIDE_STRIP =
17+
new GestureViewportReader.WindowEdges(900, 1200, 1080, 2400);
18+
19+
static void run() {
20+
assertNoInputMethodOnScreen();
21+
assertComposerAboveItsKeyPlaneKeepsItsEarlierTopEdge();
22+
assertWindowBesideTheSwipePathIsIgnored();
23+
}
24+
25+
private static void assertNoInputMethodOnScreen() {
26+
assertEdges(
27+
GestureViewportReader.struckInputMethod(
28+
Collections.<GestureViewportReader.WindowEdges>emptyList(), APPLICATION),
29+
null,
30+
"no input method window on screen");
31+
}
32+
33+
private static void assertComposerAboveItsKeyPlaneKeepsItsEarlierTopEdge() {
34+
List<GestureViewportReader.WindowEdges> both = Arrays.asList(KEY_PLANE, COMPOSER);
35+
// The key plane is the larger rectangle. Keeping only it would plan a swipe ending inside the
36+
// composer, whose top edge reaches 117px further up the screen.
37+
assertEdges(
38+
GestureViewportReader.struckInputMethod(both, APPLICATION),
39+
new GestureViewportReader.WindowEdges(0, 1400, 1080, 2400),
40+
"composer above its key plane");
41+
assertEdges(
42+
GestureViewportReader.struckInputMethod(Arrays.asList(COMPOSER, KEY_PLANE), APPLICATION),
43+
new GestureViewportReader.WindowEdges(0, 1400, 1080, 2400),
44+
"composer listed after its key plane");
45+
}
46+
47+
private static void assertWindowBesideTheSwipePathIsIgnored() {
48+
// A floating candidate strip at the right edge never crosses the centre line a vertical swipe
49+
// travels, so its higher top edge must not shorten the band.
50+
assertEdges(
51+
GestureViewportReader.struckInputMethod(Arrays.asList(KEY_PLANE, SIDE_STRIP), APPLICATION),
52+
KEY_PLANE,
53+
"input method window beside the swipe path");
54+
assertEdges(
55+
GestureViewportReader.struckInputMethod(
56+
Collections.singletonList(SIDE_STRIP), APPLICATION),
57+
null,
58+
"only an unreachable input method window on screen");
59+
}
60+
61+
private static void assertEdges(
62+
GestureViewportReader.WindowEdges actual,
63+
GestureViewportReader.WindowEdges expected,
64+
String label) {
65+
if (expected == null) {
66+
if (actual != null) {
67+
throw new AssertionError(
68+
"Expected no input method rect for " + label + ", got " + describe(actual));
69+
}
70+
return;
71+
}
72+
if (actual == null) {
73+
throw new AssertionError("Expected " + describe(expected) + " for " + label + ", got none");
74+
}
75+
if (actual.left != expected.left
76+
|| actual.top != expected.top
77+
|| actual.right != expected.right
78+
|| actual.bottom != expected.bottom) {
79+
throw new AssertionError(
80+
"Expected " + describe(expected) + " for " + label + ", got " + describe(actual));
81+
}
82+
}
83+
84+
private static String describe(GestureViewportReader.WindowEdges edges) {
85+
return "[" + edges.left + "," + edges.top + "][" + edges.right + "," + edges.bottom + "]";
86+
}
87+
}

‎android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/SnapshotHelperTestSuite.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,5 +7,6 @@ public static void main(String[] args) throws Exception {
77
PointerEventScheduleTest.run();
88
AccessibilityCaptureStabilizerTest.run();
99
BoundedUiAutomationConnectionTest.run();
10+
GestureViewportReaderTest.run();
1011
}
1112
}

0 commit comments

Comments
 (0)