Skip to content

Commit f63e72f

Browse files
thymikeethymikee
committed
fix(android): keep a scroll's swipe out of the IME window
Android is the case the clip exists for beyond iOS: an `adjustPan` or `adjustNothing` activity keeps a window whose recorded bounds already run under the IME, so a plan built from them aims at the keys. The helper now reports the largest input method window beside the application window, in absolute screen pixels like the window next to it, and `scroll` clips its band with the shared rule or refuses when the keyboard owns the surface. An older helper reports no keyboard keys, and a provider-supplied viewport has no IME channel at all. Both read as "nothing to avoid", which is what the shared rule already does with a missing frame; neither turns into a refusal. `UiAutomation.getWindows()` answers with an empty list until the service asks for interactive windows, so the read applies the seam the tree capture already uses rather than depending on a snapshot capture having run first in the same instrumentation; the one-shot fallback below it has no such neighbour. Measured on a Pixel 7 emulator with an `adjust=pan` contact editor, the application window keeps its full 2400px height while the IME window reports `[0,1517][1080,2400]`, and `scroll down` answers with `referenceHeight: 1505`, `keyboardMinY: 1517`, `keyboardAvoided: true` and a swipe ending at 301 instead of starting at 1920 under the keys.
1 parent faa75e7 commit f63e72f

11 files changed

Lines changed: 415 additions & 50 deletions

File tree

‎android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/GestureViewportReader.java‎

Lines changed: 47 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,30 @@
77
import java.util.List;
88
import java.util.concurrent.TimeoutException;
99

10-
/** Resolves the active application window bounds used to validate planned gestures. */
10+
/**
11+
* Resolves the active application window bounds used to validate planned gestures, plus the input
12+
* method window's bounds when one is on screen. The keyboard half is what lets a scroll keep its
13+
* swipe above the keys instead of flinging into them (#2500): the same {@code getWindows()} pass
14+
* already lists {@code TYPE_INPUT_METHOD}, so reading it costs no extra automation round trip, and
15+
* it is the live window list rather than a cached frame.
16+
*/
1117
final class GestureViewportReader {
1218
private GestureViewportReader() {}
1319

20+
/** The application viewport a gesture may target, and the IME's share of the screen, if any. */
21+
static final class Reading {
22+
final Rect application;
23+
/** Null when no input method window is on screen; an unmeasurable keyboard is not occlusion. */
24+
final Rect inputMethod;
25+
26+
Reading(Rect application, Rect inputMethod) {
27+
this.application = application;
28+
this.inputMethod = inputMethod;
29+
}
30+
}
31+
1432
@SuppressWarnings("deprecation")
15-
static Rect read(UiAutomation automation) {
33+
static Reading readReading(UiAutomation automation) {
1634
try {
1735
automation.waitForIdle(100, 2_000);
1836
} catch (TimeoutException ignored) {
@@ -21,12 +39,29 @@ static Rect read(UiAutomation automation) {
2139
// UiAutomation.getWindows() transfers recyclable AccessibilityWindowInfo instances, and this
2240
// read runs repeatedly inside the persistent helper session: copy the bounds the precedence
2341
// below needs, then recycle every window before resolving.
42+
// UiAutomation.getWindows() answers with an empty list until interactive retrieval is on, which
43+
// is the same seam the tree capture already uses. Without it this read sees no windows at all and
44+
// the keyboard below is invisible to it.
45+
AccessibilityTreeCapture.enableInteractiveWindowRetrieval(automation);
2446
Rect activeBounds = null;
2547
Rect fallbackBounds = null;
48+
Rect inputMethodBounds = null;
2649
List<AccessibilityWindowInfo> windows = automation.getWindows();
2750
try {
2851
for (AccessibilityWindowInfo window : windows) {
29-
if (window.getType() != AccessibilityWindowInfo.TYPE_APPLICATION) continue;
52+
int type = window.getType();
53+
if (type == AccessibilityWindowInfo.TYPE_INPUT_METHOD) {
54+
// Keep the largest IME window: a composer bar and its key plane can be reported as
55+
// separate windows, and the scroll only needs how far down the free surface reaches.
56+
Rect bounds = new Rect();
57+
window.getBoundsInScreen(bounds);
58+
if (!bounds.isEmpty() && (inputMethodBounds == null || bounds.height() * bounds.width()
59+
> inputMethodBounds.height() * inputMethodBounds.width())) {
60+
inputMethodBounds = bounds;
61+
}
62+
continue;
63+
}
64+
if (type != AccessibilityWindowInfo.TYPE_APPLICATION) continue;
3065
Rect bounds = new Rect();
3166
window.getBoundsInScreen(bounds);
3267
if (activeBounds == null
@@ -41,6 +76,15 @@ static Rect read(UiAutomation automation) {
4176
window.recycle();
4277
}
4378
}
79+
return new Reading(resolveApplication(automation, activeBounds, fallbackBounds), inputMethodBounds);
80+
}
81+
82+
static Rect read(UiAutomation automation) {
83+
return readReading(automation).application;
84+
}
85+
86+
private static Rect resolveApplication(
87+
UiAutomation automation, Rect activeBounds, Rect fallbackBounds) {
4488
if (activeBounds != null) return activeBounds;
4589
AccessibilityNodeInfo activeRoot = automation.getRootInActiveWindow();
4690
if (activeRoot != null) {

‎android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/TouchCommandHandler.java‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,11 @@ final class TouchCommandHandler {
1515
private TouchCommandHandler() {}
1616

1717
static void populateViewport(Bundle result, UiAutomation automation) {
18-
Rect viewport = GestureViewportReader.read(automation);
18+
GestureViewportReader.Reading reading = GestureViewportReader.readReading(automation);
1919
result.putString("ok", "true");
2020
result.putString("kind", "viewport");
21-
putViewportMetadata(result, viewport);
21+
putViewportMetadata(result, reading.application);
22+
putKeyboardMetadata(result, reading.inputMethod);
2223
}
2324

2425
static void populateGesture(Bundle result, UiAutomation automation, String payloadBase64)
@@ -69,4 +70,17 @@ private static void putViewportMetadata(Bundle result, Rect viewport) {
6970
result.putString("width", Integer.toString(viewport.width()));
7071
result.putString("height", Integer.toString(viewport.height()));
7172
}
73+
74+
/**
75+
* The input method window's screen bounds, reported only when one is on screen. An absent keyboard
76+
* is reported by absence: a keyboard this helper cannot see is not evidence that a surface is
77+
* blocked, so the caller must not read a zero frame as an occlusion.
78+
*/
79+
private static void putKeyboardMetadata(Bundle result, Rect inputMethod) {
80+
if (inputMethod == null) return;
81+
result.putString("keyboardX", Integer.toString(inputMethod.left));
82+
result.putString("keyboardY", Integer.toString(inputMethod.top));
83+
result.putString("keyboardWidth", Integer.toString(inputMethod.width()));
84+
result.putString("keyboardHeight", Integer.toString(inputMethod.height()));
85+
}
7286
}

‎packages/platform-android/src/__tests__/input-actions.test.ts‎

Lines changed: 138 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { test, vi } from 'vitest';
1+
import { afterEach, beforeEach, test, vi } from 'vitest';
22
import assert from 'node:assert/strict';
33
import {
44
backAndroid,
@@ -13,10 +13,147 @@ import { ANDROID_EMULATOR } from './test-utils/device-fixtures.ts';
1313
import { withFakeAdb } from './test-utils/fake-adb.ts';
1414
import { withAndroidAdbProvider } from '../adb-executor.ts';
1515
import type { AndroidTouchInjector } from '../adb-executor.ts';
16+
import { resetAndroidSnapshotHelperSessions } from '../snapshot-helper-session-lifecycle.ts';
17+
import { ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT } from './test-utils/android-snapshot-helper.ts';
18+
import {
19+
ANDROID_TOUCH_HELPER_MANIFEST as HELPER_MANIFEST,
20+
androidTouchHelperResultRecord as helperRecord,
21+
} from './touch-helper.fixtures.ts';
1622

1723
// The fake adb provider installs through the production withAndroidAdbProvider
1824
// scope, so `calls` records device-scoped args without a leading `-s <serial>`.
1925

26+
// The keyboard-aware viewport read goes through the snapshot helper rather than a touch provider,
27+
// so the scroll tests below resolve the fixture APK instead of a bundled one.
28+
vi.mock('../helper-package-install.ts', async (importOriginal) => {
29+
const actual = await importOriginal<typeof import('../helper-package-install.ts')>();
30+
return {
31+
...actual,
32+
resolveAndroidHelperArtifact: async () => ({
33+
apkPath: ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT.apkPath,
34+
manifest: {
35+
...HELPER_MANIFEST,
36+
sha256: ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT.manifest.sha256,
37+
},
38+
}),
39+
};
40+
});
41+
42+
beforeEach(async () => {
43+
delete process.env.AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION;
44+
await resetAndroidSnapshotHelperSessions();
45+
});
46+
47+
afterEach(async () => {
48+
delete process.env.AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION;
49+
await resetAndroidSnapshotHelperSessions();
50+
});
51+
52+
/**
53+
* Answers one helper session: the version probe says "current" so no install is faked, the viewport
54+
* read reports `window` plus an optional IME window, and the gesture accepts whatever it is given.
55+
*/
56+
function helperRouteAdb(window: {
57+
x: number;
58+
y: number;
59+
width: number;
60+
height: number;
61+
keyboard?: { x: number; y: number; width: number; height: number };
62+
}) {
63+
return (args: string[]) => {
64+
if (args.includes('--show-versioncode')) {
65+
return {
66+
stdout: `package:${HELPER_MANIFEST.packageName} versionCode:999999`,
67+
stderr: '',
68+
};
69+
}
70+
if (args.includes('viewport')) {
71+
const { keyboard, ...app } = window;
72+
return {
73+
stdout: [
74+
helperRecord({
75+
ok: 'true',
76+
x: String(app.x),
77+
y: String(app.y),
78+
width: String(app.width),
79+
height: String(app.height),
80+
...(keyboard
81+
? {
82+
keyboardX: String(keyboard.x),
83+
keyboardY: String(keyboard.y),
84+
keyboardWidth: String(keyboard.width),
85+
keyboardHeight: String(keyboard.height),
86+
}
87+
: {}),
88+
}),
89+
'INSTRUMENTATION_CODE: 0',
90+
].join('\n'),
91+
stderr: '',
92+
};
93+
}
94+
if (args[0] === 'shell' && args[1] === 'am') {
95+
return {
96+
stdout: [
97+
helperRecord({ ok: 'true', kind: 'pan', injectedEvents: '18', elapsedMs: '320' }),
98+
'INSTRUMENTATION_CODE: 0',
99+
].join('\n'),
100+
stderr: '',
101+
};
102+
}
103+
return undefined;
104+
};
105+
}
106+
107+
const PORTRAIT_WINDOW = { x: 0, y: 0, width: 1080, height: 2280 };
108+
const LOWER_HALF_KEYBOARD = { x: 0, y: 1600, width: 1080, height: 680 };
109+
110+
test('scrollAndroid keeps the swipe above the IME window and names the clipped band', async () => {
111+
// The full window would place a center-symmetric swipe at y 1140..~1500 — on the keys. Clipping
112+
// first means the injected path and the reported reference height both stop above the keyboard
113+
// by the accessory allowance, so a focused field no longer swallows the gesture (#2500).
114+
await withFakeAdb(
115+
helperRouteAdb({ ...PORTRAIT_WINDOW, keyboard: LOWER_HALF_KEYBOARD }),
116+
async ({ device }) => {
117+
const result = await scrollAndroid(device, 'down', { pixels: 600 });
118+
const lowest = Math.max(Number(result.y1), Number(result.y2));
119+
assert.equal(result.keyboardAvoided, true);
120+
assert.equal(result.keyboardMinY, 1600);
121+
assert.equal(result.referenceHeight, 1588, 'clipped axis is the band above the allowance');
122+
assert.equal(result.pixels, 600, 'requested travel fits the clipped band');
123+
assert.ok(lowest <= 1588, `swipe endpoint ${lowest} landed under the keyboard`);
124+
},
125+
);
126+
});
127+
128+
test('scrollAndroid swipes the whole window when no IME window is on screen', async () => {
129+
await withFakeAdb(helperRouteAdb({ ...PORTRAIT_WINDOW }), async ({ device }) => {
130+
const result = await scrollAndroid(device, 'down', { pixels: 600 });
131+
assert.equal('keyboardAvoided' in result, false);
132+
assert.equal(result.referenceHeight, 2280);
133+
});
134+
});
135+
136+
test('scrollAndroid refuses rather than flinging into a keyboard that owns the window', async () => {
137+
// A landscape IME leaves 40px of a 900px window: a swipe there reads as a stuck surface, so the
138+
// command refuses with its own typed reason instead of the generic no-progress stop.
139+
await withFakeAdb(
140+
helperRouteAdb({
141+
...PORTRAIT_WINDOW,
142+
keyboard: { x: 0, y: 120, width: 1080, height: 2160 },
143+
}),
144+
async ({ device }) => {
145+
await assert.rejects(scrollAndroid(device, 'down', { pixels: 600 }), (error: unknown) => {
146+
assert.ok(error instanceof Error);
147+
assert.equal(
148+
(error as { details?: { reason?: string } }).details?.reason,
149+
'scroll_keyboard_occludes_surface',
150+
);
151+
return true;
152+
});
153+
},
154+
);
155+
});
156+
20157
test('scrollAndroid plans explicit pixel travel through semantic touch injection', async () => {
21158
const touchCalls: Parameters<AndroidTouchInjector>[0][] = [];
22159
const result = await withAndroidAdbProvider(

‎packages/platform-android/src/__tests__/touch-helper-session.test.ts‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,10 @@ import { captureAndroidSnapshotWithHelperSession } from '../snapshot-helper-sess
2222
import { resetAndroidSnapshotHelperSessions } from '../snapshot-helper-session-lifecycle.ts';
2323
import { getAndroidSnapshotHelperSessionDeviceKey } from '../snapshot-helper-retirement.ts';
2424
import { lowerAndroidTouchPlan } from '../touch-plan-lowering.ts';
25-
import { executeAndroidTouchHelperPlan, readAndroidTouchHelperViewport } from '../touch-helper.ts';
25+
import {
26+
executeAndroidTouchHelperPlan,
27+
readAndroidTouchHelperViewportReading,
28+
} from '../touch-helper.ts';
2629
import { ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT } from './test-utils/android-snapshot-helper.ts';
2730
import {
2831
ANDROID_TOUCH_HELPER_MANIFEST as manifest,
@@ -250,7 +253,7 @@ test('a daemon-session viewport read starts the session so the gesture reuses it
250253
},
251254
{ serial: device.id },
252255
async () => {
253-
const viewport = await readAndroidTouchHelperViewport(device, {
256+
const viewport = await readAndroidTouchHelperViewportReading(device, {
254257
helperSessionScope: 'daemon-session',
255258
});
256259
const gesture = await executeAndroidTouchHelperPlan(
@@ -261,7 +264,7 @@ test('a daemon-session viewport read starts the session so the gesture reuses it
261264
},
262265
);
263266

264-
assert.deepEqual(result.viewport, { x: 0, y: 0, width: 400, height: 800 });
267+
assert.deepEqual(result.viewport, { viewport: { x: 0, y: 0, width: 400, height: 800 } });
265268
assert.equal(result.gesture.helperTransport, 'persistent-session');
266269
assert.equal(viewportCommands, 1);
267270
assert.equal(gestureCommands, 1);
@@ -297,10 +300,10 @@ test('a command-scoped viewport read stays one-shot and starts no session', asyn
297300
}),
298301
},
299302
{ serial: device.id },
300-
async () => await readAndroidTouchHelperViewport(device),
303+
async () => await readAndroidTouchHelperViewportReading(device),
301304
);
302305

303-
assert.deepEqual(viewport, { x: 5, y: 6, width: 300, height: 400 });
306+
assert.deepEqual(viewport, { viewport: { x: 5, y: 6, width: 300, height: 400 } });
304307
assert.ok(oneShotArgs?.includes('viewport'));
305308
assert.equal(sessionCommands, 0);
306309
});
@@ -673,10 +676,10 @@ test('viewport falls back to one-shot instrumentation after a session error', as
673676
}),
674677
},
675678
{ serial: device.id },
676-
async () => await readAndroidTouchHelperViewport(device),
679+
async () => await readAndroidTouchHelperViewportReading(device),
677680
);
678681

679-
assert.deepEqual(viewportResult, { x: 5, y: 6, width: 300, height: 400 });
682+
assert.deepEqual(viewportResult, { viewport: { x: 5, y: 6, width: 300, height: 400 } });
680683
assert.ok(oneShotArgs?.includes('viewport'));
681684
assert.equal(await session.isSessionAlive(), false);
682685
});
@@ -719,9 +722,9 @@ test('a structured ok=false viewport response stops the session before the one-s
719722
}),
720723
},
721724
{ serial: device.id },
722-
async () => await readAndroidTouchHelperViewport(device),
725+
async () => await readAndroidTouchHelperViewportReading(device),
723726
);
724727

725-
assert.deepEqual(viewportResult, { x: 5, y: 6, width: 300, height: 400 });
728+
assert.deepEqual(viewportResult, { viewport: { x: 5, y: 6, width: 300, height: 400 } });
726729
assert.ok(oneShotArgs?.includes('viewport'));
727730
});

0 commit comments

Comments
 (0)