Skip to content

Commit 11f8561

Browse files
committed
fix(limrun): witness the field a fill taps before typing
Limrun's typeText focus scan looks for a globally focused accessibility element, which a Flutter form never publishes, so every fill on those screens failed with "No focused accessibility element found" (#2578). The iOS interactor now reads the per-node IsEditing trait itself: it records what covered the point before tapping, taps, and types only once an editing element covers that point or was among the elements that did. An editing element elsewhere on the screen is refused as a missed target rather than typed into (#1658). Replace and clear ride the selection the field already offers, and the SDK moves to 0.49.3 for the typed element tree whose per-node traits make the witness possible.
1 parent 730b201 commit 11f8561

16 files changed

Lines changed: 658 additions & 75 deletions

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -301,7 +301,7 @@
301301
"@agent-device/xml": "workspace:*",
302302
"@arethetypeswrong/cli": "^0.18.5",
303303
"@chenglou/freerange": "^0.0.4",
304-
"@limrun/api": "^0.24.5",
304+
"@limrun/api": "^0.49.3",
305305
"@nkzw/eslint-plugin": "^2.0.0",
306306
"@nkzw/oxlint-config": "^1.3.0",
307307
"@stryker-mutator/core": "10.0.0",

‎packages/contracts/src/interactor-types.ts‎

Lines changed: 19 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -116,15 +116,17 @@ export type TypeTextBackendResult = {
116116
};
117117

118118
/**
119-
* What a cloud-WebDriver `fill` could establish about its target taking text
120-
* entry focus before it sent the keys (#1658). The local Apple runner owns this
121-
* discipline in Swift and reports it as a `textEntryRoute`; the cloud path has
122-
* no runner, only what the driver will tell it, so it names what it observed
123-
* instead of which channel it typed through. Listed strongest evidence first:
119+
* What a cloud `fill` could establish about its target taking text entry focus
120+
* before it sent the keys (#1658). The local Apple runner owns this discipline
121+
* in Swift and reports it as a `textEntryRoute`; a cloud session has no runner,
122+
* only what its provider will tell it, so it names what it observed instead of
123+
* which channel it typed through. Listed strongest evidence first:
124124
*
125-
* - `focused-element`: the element holding text-entry focus after the tap
126-
* contains the tapped point — positive evidence that THIS field, not merely
127-
* some field, took focus.
125+
* - `focused-element`: after the tap, the element holding text-entry focus is
126+
* not the one that held it before, or it contains the tapped point — positive
127+
* evidence that THIS field, not merely some field, took focus. Identity is
128+
* asked first because focusing a field can re-lay it out, so geometry alone
129+
* would refuse a fill that plainly worked.
128130
* - `keyboard-shown`: no active-element route, but the software keyboard went
129131
* from hidden to shown after our tap. Witnesses that focus arrived somewhere,
130132
* which only means our field when the keyboard was down beforehand.
@@ -141,9 +143,9 @@ export type TypeTextBackendResult = {
141143
* wait could establish anything. `press` + `type` remains the deliberate way
142144
* to enter text unwitnessed.
143145
*
144-
* Closed set: the cloud interactor is its only producer and the boundary that
145-
* narrows it (readFillBackendResult in the touch handler) drops a
146-
* value it cannot name.
146+
* Closed set: the two cloud interactors are its only producers, each reading
147+
* focus through its own provider's route, and the boundary that narrows it
148+
* (readFillBackendResult in the touch handler) drops a value it cannot name.
147149
*/
148150
export const CLOUD_TEXT_ENTRY_READINESS = ['focused-element', 'keyboard-shown'] as const;
149151

@@ -165,11 +167,12 @@ export type FillUnconfirmedVerification = {
165167
};
166168

167169
/**
168-
* What `Interactor.fill` reports back about the entry it performed. The
169-
* cloud-WebDriver interactor populates `textEntryReadiness`; Android may return
170-
* target-bound `verification: 'unconfirmed'` evidence when an app-owned field
171-
* changed but formatting prevented raw equality. The Apple runner carries its
172-
* own readiness equivalent in Swift.
170+
* What `Interactor.fill` reports back about the entry it performed. The two
171+
* cloud interactors (WebDriver and the Limrun iOS direct session) populate
172+
* `textEntryReadiness`; Android may return target-bound
173+
* `verification: 'unconfirmed'` evidence when an app-owned field changed but
174+
* formatting prevented raw equality. The Apple runner carries its own readiness
175+
* equivalent in Swift.
173176
*/
174177
export type FillBackendResult =
175178
| {

‎packages/provider-limrun/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,6 @@
1616
}
1717
},
1818
"devDependencies": {
19-
"@limrun/api": "^0.24.5"
19+
"@limrun/api": "^0.49.3"
2020
}
2121
}

‎packages/provider-limrun/src/ios-interactor-snapshot.test.ts‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,11 @@ test('limrun iOS snapshot stamps the xctest channel with its own producer', asyn
55
const session = {
66
platform: 'ios',
77
client: {
8-
elementTree: async () =>
9-
JSON.stringify({
10-
elementType: 'Application',
11-
frame: { x: 0, y: 0, width: 320, height: 240 },
12-
children: [{ elementType: 'Button', label: 'Continue', enabled: true }],
13-
}),
8+
elementTree: async () => ({
9+
elementType: 'Application',
10+
frame: { x: 0, y: 0, width: 320, height: 240 },
11+
children: [{ elementType: 'Button', label: 'Continue', enabled: true }],
12+
}),
1413
deviceInfo: { screenWidth: 320, screenHeight: 240 },
1514
},
1615
} as unknown as LimrunIosSession;
Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
import { expect, test, vi } from 'vitest';
2+
import { createLimrunIosInteractor, type LimrunIosSession } from './ios.ts';
3+
import type { IosTreeNode } from './snapshot.ts';
4+
5+
const FIELD_RECT = Object.freeze({ x: 24, y: 142, width: 354, height: 56 });
6+
7+
function emailField(editing: boolean): IosTreeNode {
8+
return {
9+
type: 'TextField',
10+
AXLabel: 'Email',
11+
pid: 4242,
12+
frame: FIELD_RECT,
13+
traits: editing
14+
? ['TextEntry', 'TextOperationsAvailable', 'IsEditing']
15+
: ['TextEntry', 'TextOperationsAvailable'],
16+
};
17+
}
18+
19+
function sessionWithTrees(trees: IosTreeNode[]) {
20+
const calls: string[] = [];
21+
let reads = 0;
22+
const client = {
23+
elementTree: vi.fn(async () => {
24+
calls.push('elementTree');
25+
const tree = trees[Math.min(reads, trees.length - 1)];
26+
reads += 1;
27+
return tree;
28+
}),
29+
tap: vi.fn(async () => {
30+
calls.push('tap');
31+
}),
32+
pressKey: vi.fn(async (key: string, modifiers?: string[]) => {
33+
calls.push(`pressKey:${key}${modifiers?.length ? `+${modifiers.join('+')}` : ''}`);
34+
}),
35+
typeText: vi.fn(async () => {
36+
calls.push('typeText');
37+
}),
38+
};
39+
const session = {
40+
platform: 'ios',
41+
instanceId: 'limrun-text-entry-instance',
42+
client,
43+
} as unknown as LimrunIosSession;
44+
return { interactor: createLimrunIosInteractor(session), client, calls };
45+
}
46+
47+
test('type enters the focused field without the provider focus scan', async () => {
48+
const { interactor, client } = sessionWithTrees([]);
49+
50+
await interactor.type('[email protected]');
51+
52+
expect(client.typeText).toHaveBeenCalledWith('[email protected]', false, { requireFocus: false });
53+
});
54+
55+
test('a delayed type sends one focused-blind key press per character', async () => {
56+
const { interactor, client } = sessionWithTrees([]);
57+
58+
await interactor.type('abc', 1);
59+
60+
expect(client.typeText.mock.calls).toEqual([
61+
['a', false, { requireFocus: false }],
62+
['b', false, { requireFocus: false }],
63+
['c', false, { requireFocus: false }],
64+
]);
65+
});
66+
67+
test('fill waits for text-entry focus to arrive and only then sends keys', async () => {
68+
const { interactor, client, calls } = sessionWithTrees([
69+
emailField(false),
70+
emailField(false),
71+
emailField(true),
72+
]);
73+
74+
const result = await interactor.fill(200, 170, '[email protected]');
75+
76+
expect(calls).toEqual([
77+
'elementTree',
78+
'tap',
79+
'elementTree',
80+
'elementTree',
81+
'pressKey:a+command',
82+
'typeText',
83+
]);
84+
expect(result).toEqual({ textEntryReadiness: 'focused-element' });
85+
expect(client.typeText).toHaveBeenCalledWith('[email protected]', false, { requireFocus: false });
86+
});
87+
88+
test('fill selects the field value first, so entry replaces it', async () => {
89+
const { interactor, calls } = sessionWithTrees([emailField(false), emailField(true)]);
90+
91+
await interactor.fill(200, 170, 'replacement');
92+
93+
expect(calls.indexOf('pressKey:a+command')).toBeLessThan(calls.indexOf('typeText'));
94+
});
95+
96+
test('the clear request empties the focused field instead of typing nothing', async () => {
97+
const { interactor, client, calls } = sessionWithTrees([emailField(false), emailField(true)]);
98+
99+
const result = await interactor.fill(200, 170, '');
100+
101+
expect(calls).toEqual([
102+
'elementTree',
103+
'tap',
104+
'elementTree',
105+
'pressKey:a+command',
106+
'pressKey:delete',
107+
]);
108+
expect(client.typeText).not.toHaveBeenCalled();
109+
expect(result).toEqual({ textEntryReadiness: 'focused-element' });
110+
});
111+
112+
test('a session that cannot answer the focus read fails before the device is touched', async () => {
113+
const { interactor, client, calls } = sessionWithTrees([emailField(false)]);
114+
client.elementTree.mockImplementationOnce(async () => {
115+
calls.push('elementTree');
116+
throw new Error('instance connection lost');
117+
});
118+
119+
await expect(interactor.fill(200, 170, '[email protected]')).rejects.toThrow(
120+
'instance connection lost',
121+
);
122+
expect(calls).toEqual(['elementTree']);
123+
expect(client.tap).not.toHaveBeenCalled();
124+
expect(client.typeText).not.toHaveBeenCalled();
125+
});

‎packages/provider-limrun/src/ios-snapshot-adapter.fixtures.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,8 @@ export function createLimrunSnapshotSession(
5959
return {
6060
instanceId: 'limrun-snapshot-test-instance',
6161
client: {
62-
elementTree: async () => JSON.stringify(tree),
62+
elementTree: async () => tree,
6363
deviceInfo: { screenWidth: screen.width, screenHeight: screen.height },
6464
},
65-
} as Pick<LimrunIosSession, 'client' | 'instanceId'>;
65+
} as unknown as Pick<LimrunIosSession, 'client' | 'instanceId'>;
6666
}

‎packages/provider-limrun/src/ios-snapshot-adapter.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import { flattenIosTree, type IosTreeNode } from './snapshot.ts';
99
export async function captureLimrunIosSnapshot(
1010
session: Pick<LimrunIosSession, 'client' | 'instanceId'>,
1111
): Promise<SnapshotRuntimeAcquiredResult> {
12-
const tree = JSON.parse(await session.client.elementTree()) as IosTreeNode | IosTreeNode[];
12+
const tree = (await session.client.elementTree()) as IosTreeNode[];
1313
const viewport = readLimrunViewport(tree, session.client.deviceInfo);
1414
return createIosSnapshotAcquisition({
1515
producer: 'limrun-ios-tree',

‎packages/provider-limrun/src/ios.ts‎

Lines changed: 50 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import type {
55
ProviderDeviceInstallOptions,
66
ProviderDeviceInstallResult,
77
} from '@agent-device/contracts/device';
8-
import type { Interactor } from '@agent-device/contracts/interactor-types';
8+
import type { FillBackendResult, Interactor } from '@agent-device/contracts/interactor-types';
99
import type { DeviceInfo } from '@agent-device/kernel/device';
1010
import { AppError } from '@agent-device/kernel/errors';
1111
import type Limrun from '@limrun/api';
@@ -226,19 +226,61 @@ class LimrunIosInteractor implements Interactor {
226226
}
227227

228228
async type(text: string, delayMs?: number): Promise<void> {
229+
await this.enterText(text, delayMs);
230+
}
231+
232+
async fill(x: number, y: number, text: string, delayMs?: number): Promise<FillBackendResult> {
233+
// The witness is loaded on the fill path instead of at module evaluation, so this
234+
// provider keeps the import-time closure budget it declares.
235+
const { awaitLimrunTextEntryFocus, readLimrunTapTargetIdentities, readLimrunTextEntryFocus } =
236+
await import('./text-entry-focus.ts');
237+
// The screen is read before the tap so the witness can say what was under this
238+
// point at all: an editing element that appears somewhere else proves nothing
239+
// about this tap (#1658).
240+
const targetsAtPoint = readLimrunTapTargetIdentities(
241+
await this.session.client.elementTree(),
242+
x,
243+
y,
244+
);
245+
await this.tap(x, y);
246+
const textEntryReadiness = await awaitLimrunTextEntryFocus({
247+
targetsAtPoint,
248+
readFocus: async () => readLimrunTextEntryFocus(await this.session.client.elementTree()),
249+
sleep: (milliseconds) => sleep(milliseconds),
250+
x,
251+
y,
252+
});
253+
// Select what the field holds first: iOS replaces a selection on the next
254+
// key, so this is what makes `fill` a replacement rather than an append, and
255+
// `fill <target> ""` a clear rather than a no-op.
256+
await this.session.client.pressKey('a', ['command']);
257+
if (text.length === 0) {
258+
await this.session.client.pressKey('delete');
259+
return { textEntryReadiness };
260+
}
261+
await this.enterText(text, delayMs);
262+
return { textEntryReadiness };
263+
}
264+
265+
/**
266+
* Types into whatever holds text-entry focus, character by character when a
267+
* delay is asked for. `requireFocus: false` because the provider's own scan
268+
* asks a question agent-device already answered: it looks for a globally
269+
* reported focused element, which an app that exposes its fields without one
270+
* (#2578's Flutter form) can never satisfy even with its field focused and
271+
* taking keys. Focus targeting belongs to the caller here — `fill` witnesses it
272+
* against the editing element, and `type` is agent-device's deliberate
273+
* entry-into-the-current-field route on every other platform too.
274+
*/
275+
private async enterText(text: string, delayMs?: number): Promise<void> {
229276
if (delayMs && delayMs > 0) {
230277
for (const char of Array.from(text)) {
231-
await this.session.client.typeText(char);
278+
await this.session.client.typeText(char, false, { requireFocus: false });
232279
await sleep(delayMs);
233280
}
234281
return;
235282
}
236-
await this.session.client.typeText(text);
237-
}
238-
239-
async fill(x: number, y: number, text: string): Promise<void> {
240-
await this.tap(x, y);
241-
await this.session.client.typeText(text);
283+
await this.session.client.typeText(text, false, { requireFocus: false });
242284
}
243285

244286
async scroll(direction: 'up' | 'down' | 'left' | 'right', options?: { pixels?: number }) {
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
import { expect, test } from 'vitest';
2+
import { toIosSelector } from './snapshot.ts';
3+
4+
test('agent-device selector keys become the accessibility keys the iOS instance matches', () => {
5+
expect(toIosSelector({ key: 'id', value: 'email-field' })).toEqual({
6+
AXUniqueId: 'email-field',
7+
});
8+
expect(toIosSelector({ key: 'value', value: '[email protected]' })).toEqual({
9+
AXValue: '[email protected]',
10+
});
11+
expect(toIosSelector({ key: 'label', value: 'Email' })).toEqual({ AXLabel: 'Email' });
12+
});
13+
14+
test('visible text targets the provider label field, which is where the tree exposes it', () => {
15+
expect(toIosSelector({ key: 'text', value: 'Sign In' })).toEqual({ AXLabel: 'Sign In' });
16+
});

‎packages/provider-limrun/src/snapshot.ts‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ export type IosTreeNode = {
1919
role?: string;
2020
selected?: boolean;
2121
hittable?: boolean;
22+
traits?: string[];
23+
pid?: number;
2224
children?: IosTreeNode[];
2325
nodes?: IosTreeNode[];
2426
elements?: IosTreeNode[];
@@ -78,7 +80,8 @@ function readIosNodeIdentifier(node: IosTreeNode): string | undefined {
7880
return node.identifier ?? node.AXUniqueId ?? undefined;
7981
}
8082

81-
function readIosNodeRect(node: IosTreeNode): RawSnapshotNode['rect'] {
83+
/** The node's screen-space rect, or undefined when the provider omitted a complete one. */
84+
export function readIosNodeRect(node: IosTreeNode): RawSnapshotNode['rect'] {
8285
const rect = node.rect ?? node.frame;
8386
if (
8487
!rect ||
@@ -97,16 +100,16 @@ function readIosNodeRect(node: IosTreeNode): RawSnapshotNode['rect'] {
97100
};
98101
}
99102

100-
function readIosNodeChildren(node: IosTreeNode): IosTreeNode[] {
103+
export function readIosNodeChildren(node: IosTreeNode): IosTreeNode[] {
101104
return node.children ?? node.nodes ?? node.elements ?? [];
102105
}
103106

104107
export function toIosSelector(selector: LimrunSelector) {
105-
if (selector.key === 'id') return { accessibilityId: selector.value };
106-
if (selector.key === 'value') return { value: selector.value };
108+
if (selector.key === 'id') return { AXUniqueId: selector.value };
109+
if (selector.key === 'value') return { AXValue: selector.value };
107110
// The Limrun iOS tree exposes visible text through AXLabel, so both
108111
// agent-device label and text selectors target the provider's label field.
109-
return { label: selector.value };
112+
return { AXLabel: selector.value };
110113
}
111114

112115
export async function writeBase64File(filePath: string, base64: string): Promise<void> {

0 commit comments

Comments
 (0)