Skip to content

Commit ca99252

Browse files
committed
perf(android): batch text entry whenever the device is on the helper IME
The IME helper's broadcast channel writes a whole string in one `am broadcast`, but it was reachable only through this daemon process's activation cache. A device already carrying the helper as its active input method — left by a crashed run, switched by another daemon on the same emulator, or opened with `--no-test-ime` — silently fell back to ceil(n/8) `input text` spawns. Type and fill now read the active input method from the `dumpsys input_method` probe the shell path already runs, and route to the batch channel when that input method IS the helper. Zero extra spawns: the read that decides the channel is the read that already decided IME ownership. The broadcast targets the observed package, so the promoted route does not depend on a packaged artifact being on disk. `ANDROID_INPUT_TEXT_CHUNK_SIZE` stays at 8 and now says why in the code: it is the #531 truncation fix, not a tuning knob. `text-entry.ts` extracts the shell mechanics (chunking, encoding, typed unsupported/ime_capture classification, ownership) that `input-actions.ts` had outgrown; `assertAndroidShellInputIsAppOwned` splits into a probe and a pure assertion so one read can answer both questions.
1 parent cc1c2bc commit ca99252

3 files changed

Lines changed: 214 additions & 16 deletions

File tree

‎src/platforms/android/__tests__/text-input-test-ime.test.ts‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,128 @@ test('fillAndroid clears then commits non-ASCII text through the test IME and ve
123123
);
124124
});
125125

126+
// The device, not this daemon process, decides which IME is active. A previous run (or another
127+
// daemon on the same emulator) can leave the agent-device helper as the active input method with
128+
// nothing in this process's activation cache — the batch channel is still the right channel there.
129+
130+
test('typeAndroid batches ASCII text when the device is already on the helper IME', async () => {
131+
const calls: string[][] = [];
132+
await withAndroidAdbProvider(
133+
async (args) => {
134+
calls.push(args);
135+
return {
136+
exitCode: 0,
137+
stdout: args.join(' ') === 'shell dumpsys input_method' ? helperImeInputMethodDump() : '',
138+
stderr: '',
139+
};
140+
},
141+
{ serial: ANDROID_EMULATOR.id },
142+
async () => {
143+
await typeAndroid(ANDROID_EMULATOR, 'filed the expense');
144+
},
145+
);
146+
147+
const broadcasts = calls.filter((args) => args[1] === 'am' && args[2] === 'broadcast');
148+
assert.equal(broadcasts.length, 1, 'the whole string should land in one broadcast');
149+
assert.deepEqual(broadcasts[0]?.slice(3, 5), ['-p', PACKAGE]);
150+
const textIndex = broadcasts[0]?.indexOf('text') ?? -1;
151+
assert.equal(
152+
Buffer.from(broadcasts[0]?.[textIndex + 1] ?? '', 'base64').toString('utf8'),
153+
'filed the expense',
154+
);
155+
assert.equal(
156+
calls.filter((args) => args[1] === 'input' && args[2] === 'text').length,
157+
0,
158+
'shell chunking must not run when the helper IME owns text entry',
159+
);
160+
});
161+
162+
test('fillAndroid batches ASCII text when the device is already on the helper IME', async () => {
163+
let currentText = 'stale value';
164+
const calls: string[][] = [];
165+
const adb: AndroidAdbExecutor = createAndroidSnapshotHelperExecutor({
166+
exec: async (args) => {
167+
calls.push(args);
168+
if (args[1] === 'am' && args[2] === 'broadcast') {
169+
const action = args[args.indexOf('-a') + 1];
170+
if (action === 'com.callstack.agentdevice.imehelper.ACTION_CLEAR_TEXT') {
171+
currentText = '';
172+
} else if (action === 'com.callstack.agentdevice.imehelper.ACTION_INPUT_TEXT_B64') {
173+
const textIndex = args.indexOf('text');
174+
currentText += Buffer.from(args[textIndex + 1] ?? '', 'base64').toString('utf8');
175+
}
176+
return { exitCode: 0, stdout: '', stderr: '' };
177+
}
178+
return {
179+
exitCode: 0,
180+
stdout: args.join(' ') === 'shell dumpsys input_method' ? helperImeInputMethodDump() : '',
181+
stderr: '',
182+
};
183+
},
184+
captureXml: () => androidInputXml({ text: currentText }),
185+
});
186+
187+
await withAndroidAdbProvider(
188+
{ exec: adb, snapshotHelperArtifact: ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT },
189+
{ serial: ANDROID_EMULATOR.id },
190+
async () => {
191+
await fillAndroid(ANDROID_EMULATOR, 10, 10, 'filed the expense');
192+
},
193+
);
194+
195+
assert.equal(currentText, 'filed the expense');
196+
assert.equal(
197+
calls.filter((args) => args[1] === 'input' && args[2] === 'text').length,
198+
0,
199+
'shell chunking must not run when the helper IME owns text entry',
200+
);
201+
assert.equal(
202+
calls.some((args) => args.includes('KEYCODE_DEL')),
203+
false,
204+
'the helper clears the field over the broadcast channel, not with delete keyevents',
205+
);
206+
});
207+
208+
test('a third-party active IME keeps ASCII text on the chunked shell path', async () => {
209+
const calls: string[][] = [];
210+
await withAndroidAdbProvider(
211+
async (args) => {
212+
calls.push(args);
213+
return {
214+
exitCode: 0,
215+
stdout:
216+
args.join(' ') === 'shell dumpsys input_method'
217+
? [
218+
'mInputShown=true',
219+
'mCurMethodId=com.vendor.keyboard/.VendorIme',
220+
'packageName=com.example.shop',
221+
'resourceId=com.example.shop:id/search',
222+
'inputType=0x1',
223+
].join('\n')
224+
: '',
225+
stderr: '',
226+
};
227+
},
228+
{ serial: ANDROID_EMULATOR.id },
229+
async () => {
230+
await typeAndroid(ANDROID_EMULATOR, 'filed the expense');
231+
},
232+
);
233+
234+
assert.equal(calls.filter((args) => args[1] === 'am' && args[2] === 'broadcast').length, 0);
235+
assert.equal(calls.filter((args) => args[1] === 'input' && args[2] === 'text').length, 3);
236+
});
237+
238+
function helperImeInputMethodDump(): string {
239+
return [
240+
'mInputShown=true',
241+
`mCurMethodId=${PACKAGE}/.TestInputMethodService`,
242+
'packageName=com.example.shop',
243+
'resourceId=com.example.shop:id/search',
244+
'inputType=0x1',
245+
].join('\n');
246+
}
247+
126248
function androidInputXml(options: { text: string }): string {
127249
return `<?xml version="1.0" encoding="UTF-8"?><hierarchy><node package="com.example" class="android.widget.EditText" text="${options.text}" focused="true" bounds="[0,0][200,100]"/></hierarchy>`;
128250
}

‎src/platforms/android/ime-helper.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,16 @@ export function getAndroidImeHelperDeviceKey(device: DeviceInfo): string {
121121
return `${device.platform}:${device.id}`;
122122
}
123123

124+
/**
125+
* Whether a device's active input method IS this helper. The batch broadcast channel below works
126+
* whenever that holds — which is not the same as "this daemon process switched the IME". A crashed
127+
* run, a second daemon on the same emulator, or an `open --no-test-ime` onto a device that already
128+
* carries the helper all leave the channel available with an empty activation cache.
129+
*/
130+
export function isAndroidImeHelperPackage(packageName: string | undefined): boolean {
131+
return packageName === ANDROID_IME_HELPER_PACKAGE;
132+
}
133+
124134
// --- Broadcast text-entry channel -----------------------------------------------------------
125135
// The IME's receiver requires the WRITE_SECURE_SETTINGS sender permission, which adb shell holds
126136
// but a co-installed third-party app cannot — that permission gate (not the transport) is the

‎src/platforms/android/text-input.ts‎

Lines changed: 82 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -26,13 +26,19 @@ import {
2626
} from './fill-verification.ts';
2727
import {
2828
clearAndroidImeHelperText,
29+
isAndroidImeHelperPackage,
2930
selectAndroidImeHelperArtifact,
3031
sendAndroidImeHelperText,
3132
} from './ime-helper.ts';
3233
import { isAndroidTestImeActive } from './ime-lifecycle.ts';
3334
import { focusAndroid } from './input-actions.ts';
3435
import type { AndroidHelperSessionOptions } from './snapshot-helper-types.ts';
3536

37+
/**
38+
* `input text` truncates long strings in some app/IME states (#531), so ASCII is written in short
39+
* chunks rather than one command. Do not raise this to buy back spawns — the IME helper's batch
40+
* broadcast is the lever for that, and it does not truncate.
41+
*/
3642
const ANDROID_INPUT_TEXT_CHUNK_SIZE = 8;
3743

3844
export async function typeAndroid(device: DeviceInfo, text: string, delayMs = 0): Promise<void> {
@@ -43,11 +49,25 @@ export async function typeAndroid(device: DeviceInfo, text: string, delayMs = 0)
4349
return;
4450
}
4551
if (isAndroidTestImeActive(device)) {
46-
await typeAndroidTestIme(device, text, delayMs);
52+
await typeAndroidImeHelper(
53+
device,
54+
await activatedAndroidImeHelperPackage(device),
55+
text,
56+
delayMs,
57+
);
4758
return;
4859
}
4960
assertAndroidShellTextSupported(text);
50-
await assertAndroidShellInputIsAppOwned(device, 'type');
61+
// The shell path needs the input-method read anyway, and it also names the device's active IME.
62+
// If that is the helper, its batch channel writes the whole string in one broadcast instead of
63+
// ceil(n/8) `input text` spawns — no flag, no IME switch, nothing this process had to arrange.
64+
const inputState = await readAndroidShellTextInputState(device, 'type');
65+
const helperPackage = androidImeHelperInputMethod(inputState);
66+
if (helperPackage) {
67+
await typeAndroidImeHelper(device, helperPackage, text, delayMs);
68+
return;
69+
}
70+
assertAndroidShellInputIsAppOwned(inputState, 'type');
5171
if (delayMs > 0 && Array.from(text).length > 1) {
5272
await typeAndroidShell(device, { action: 'type', text, chunkSize: 1, delayMs });
5373
return;
@@ -77,7 +97,15 @@ export async function fillAndroid(
7797
return completeAndroidFillVerification(text, beforeTarget, verification);
7898
}
7999
if (isAndroidTestImeActive(device)) {
80-
const verification = await fillAndroidTestIme(device, x, y, text, beforeTarget, helper);
100+
const verification = await fillAndroidImeHelper(
101+
device,
102+
await activatedAndroidImeHelperPackage(device),
103+
x,
104+
y,
105+
text,
106+
beforeTarget,
107+
helper,
108+
);
81109
return completeAndroidFillVerification(text, beforeTarget, verification);
82110
}
83111
assertAndroidShellTextSupported(text);
@@ -110,7 +138,23 @@ export async function fillAndroid(
110138

111139
for (const attempt of attempts) {
112140
await focusAndroid(device, x, y);
113-
await assertAndroidShellInputIsAppOwned(device, 'fill');
141+
// Same read, same reason as `typeAndroid`: when the helper is the active IME its channel
142+
// replaces both the delete-key clear and the chunked write for the rest of this fill.
143+
const inputState = await readAndroidShellTextInputState(device, 'fill');
144+
const helperPackage = androidImeHelperInputMethod(inputState);
145+
if (helperPackage) {
146+
const verification = await fillAndroidImeHelper(
147+
device,
148+
helperPackage,
149+
x,
150+
y,
151+
text,
152+
beforeTarget,
153+
helper,
154+
);
155+
return completeAndroidFillVerification(text, beforeTarget, verification);
156+
}
157+
assertAndroidShellInputIsAppOwned(inputState, 'fill');
114158
const clearCount = clampCount(
115159
textCodePointLength + attempt.clearPadding,
116160
attempt.minClear,
@@ -136,14 +180,27 @@ export async function fillAndroid(
136180
return completeAndroidFillVerification(text, beforeTarget, lastVerification);
137181
}
138182

139-
async function typeAndroidTestIme(
183+
/**
184+
* The helper package this process switched the device to. The observed-IME route reads the package
185+
* off the device instead, so it never depends on a packaged artifact being present.
186+
*/
187+
async function activatedAndroidImeHelperPackage(device: DeviceInfo): Promise<string> {
188+
const artifact = await selectAndroidImeHelperArtifact(resolveAndroidAdbProvider(device));
189+
return artifact.manifest.packageName;
190+
}
191+
192+
function androidImeHelperInputMethod(state: AndroidKeyboardState | null): string | undefined {
193+
const packageName = state?.inputMethodPackage;
194+
return isAndroidImeHelperPackage(packageName) ? packageName : undefined;
195+
}
196+
197+
async function typeAndroidImeHelper(
140198
device: DeviceInfo,
199+
packageName: string,
141200
text: string,
142201
delayMs: number,
143202
): Promise<void> {
144203
const adb = resolveAndroidAdbExecutor(device);
145-
const artifact = await selectAndroidImeHelperArtifact(resolveAndroidAdbProvider(device));
146-
const packageName = artifact.manifest.packageName;
147204
const parts = text.split('\n');
148205
for (const [partIndex, part] of parts.entries()) {
149206
const chunks = delayMs > 0 ? chunkAndroidInputText(part, 1) : [part];
@@ -160,17 +217,16 @@ async function typeAndroidTestIme(
160217
emitAndroidTextDiagnostic('type', 'test-ime', text);
161218
}
162219

163-
async function fillAndroidTestIme(
220+
async function fillAndroidImeHelper(
164221
device: DeviceInfo,
222+
packageName: string,
165223
x: number,
166224
y: number,
167225
text: string,
168226
beforeTarget: AndroidFillVerification['targetInput'],
169227
helper: AndroidHelperSessionOptions,
170228
): Promise<AndroidFillVerification> {
171229
const adb = resolveAndroidAdbExecutor(device);
172-
const artifact = await selectAndroidImeHelperArtifact(resolveAndroidAdbProvider(device));
173-
const packageName = artifact.manifest.packageName;
174230
let lastVerification: AndroidFillVerification | null = null;
175231
// One retry covers the rare not-yet-bound InputConnection right after focus.
176232
for (let attempt = 0; attempt < 2; attempt += 1) {
@@ -241,13 +297,17 @@ async function clearFocusedText(device: DeviceInfo, count: number): Promise<void
241297
}
242298
}
243299

244-
async function assertAndroidShellInputIsAppOwned(
300+
/**
301+
* Reads the device's input-method state for a text-entry decision, or `null` when the probe itself
302+
* fails. A failed probe is not a refusal: the shell path has always continued without this evidence
303+
* rather than blocking text entry on a diagnostic read.
304+
*/
305+
async function readAndroidShellTextInputState(
245306
device: DeviceInfo,
246307
action: AndroidTextInputAction,
247-
): Promise<void> {
248-
let state: AndroidKeyboardState;
308+
): Promise<AndroidKeyboardState | null> {
249309
try {
250-
state = await getAndroidKeyboardState(device);
310+
return await getAndroidKeyboardState(device);
251311
} catch (error) {
252312
emitDiagnostic({
253313
level: 'warn',
@@ -257,9 +317,15 @@ async function assertAndroidShellInputIsAppOwned(
257317
error: error instanceof Error ? error.message : String(error),
258318
},
259319
});
260-
return;
320+
return null;
261321
}
262-
if (state.inputOwner !== 'ime') return;
322+
}
323+
324+
function assertAndroidShellInputIsAppOwned(
325+
state: AndroidKeyboardState | null,
326+
action: AndroidTextInputAction,
327+
): void {
328+
if (!state || state.inputOwner !== 'ime') return;
263329
throw new AppError(
264330
'COMMAND_FAILED',
265331
'KEYBOARD_OVERLAY_BLOCKING: Android text input is blocked because the focused input belongs to the active keyboard/IME.',

0 commit comments

Comments
 (0)