Skip to content

Commit a79dad6

Browse files
authored
feat(interaction): surface targetHittable/hint for non-hittable press/fill targets (#1040)
When a resolved press/fill/click selector or ref target lands on a node that reports hittable: false (after any hittable-ancestor promotion), the success response now carries targetHittable: false and a hint suggesting the interaction may have had no visible effect. Resolution semantics are unchanged: no failing, no filtering, no strict-unique matching — iOS AX hittable flags are unreliable on deep RN trees, and react-navigation's maestro suite depends on today's auto-disambiguation behavior (#1037). Also documents the selector disambiguation policy (deepest node, then smallest area, exact ties fail) in `help workflow`, including that replay/replay-heal apply the same policy for touch/fill/get-text.
1 parent 9b19aee commit a79dad6

8 files changed

Lines changed: 135 additions & 0 deletions

File tree

‎src/__tests__/cli-help.test.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,18 @@ test('help workflow preserves known device workaround guidance', async () => {
106106
assert.match(result.stdout, /Avoid broad restore-key fallbacks/);
107107
});
108108

109+
test('help workflow documents the selector disambiguation policy (#1037)', async () => {
110+
const result = await runCliCapture(['help', 'workflow']);
111+
assert.equal(result.code, 0);
112+
assert.equal(result.calls.length, 0);
113+
assert.match(result.stdout, /does not fail by default/);
114+
assert.match(result.stdout, /deepest node first/);
115+
assert.match(result.stdout, /then smallest on-screen area/);
116+
assert.match(result.stdout, /Selector did not resolve uniquely/);
117+
assert.match(result.stdout, /replay and replay-heal apply the same depth-then-area policy/);
118+
assert.match(result.stdout, /targetHittable: false/);
119+
});
120+
109121
test('help unknown command prints error plus global usage and skips daemon dispatch', async () => {
110122
const result = await runCliCapture(['help', 'not-a-command']);
111123
assert.equal(result.code, 1);

‎src/cli/parser/cli-help.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,8 @@ Selectors:
189189
agent-device press 'id="submit-order"'
190190
agent-device is visible 'label="Online"'
191191
agent-device get text 'id="quantity-value"'
192+
Ambiguous selector disambiguation: a selector on an interactive command (press/click/fill/focus/longpress/scroll/swipe/pinch) that matches multiple elements does not fail by default. It auto-resolves deepest node first (largest depth in the tree), then smallest on-screen area; only an exact tie on both depth and area fails with "Selector did not resolve uniquely". replay and replay-heal apply the same depth-then-area policy for touch/fill/get-text, so recorded flows and live commands pick the same candidate. This exists because short/reused labels (tab + header + button with the same text, or a duplicated list-row label) are common in real apps; add id="..." or a longer/more specific text to force a different match instead of assuming ambiguous selectors always fail.
193+
Selector match can still land on a non-interactive node (for example an off-screen map annotation that exact-matches text= while the real control has a longer label). Success responses for press/fill/click/ref targets carry targetHittable: false and a hint when the resolved element reports hittable: false, since the tap may have had no visible effect; treat that as a signal to verify with a snapshot or re-target by @ref/longer text, not as a command failure.
192194
193195
Text entry:
194196
fill replaces; type appends to focused field.

‎src/commands/interaction/runtime/interactions.test.ts‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -234,6 +234,60 @@ test('runtime click keeps non-button semantic targets at their own center', asyn
234234
assert.equal(result.node?.label, 'Account');
235235
});
236236

237+
test('runtime press surfaces targetHittable and a hint when the final tap node is non-hittable (#1037)', async () => {
238+
const calls: Point[] = [];
239+
const device = createInteractionDevice(nonHittableCellSnapshot(), {
240+
tap: async (_context, point) => {
241+
calls.push(point);
242+
},
243+
});
244+
245+
const result = await device.interactions.press(ref('@e2'), { session: 'default' });
246+
247+
// Press still proceeds and reports success — non-hittable is informational only.
248+
assert.deepEqual(calls, [{ x: 70, y: 30 }]);
249+
assert.equal(result.kind, 'ref');
250+
assert.equal(result.node?.label, 'Account');
251+
assert.equal(result.targetHittable, false);
252+
assert.match(result.hint ?? '', /hittable: false/);
253+
assert.match(result.hint ?? '', /@ref/);
254+
});
255+
256+
test('runtime press omits targetHittable and hint when the resolved node is hittable', async () => {
257+
const device = createInteractionDevice(selectorSnapshot(), {
258+
tap: async () => {},
259+
});
260+
261+
const result = await device.interactions.press(selector('label=Continue'), {
262+
session: 'default',
263+
});
264+
265+
assert.equal(result.kind, 'selector');
266+
assert.equal(result.targetHittable, undefined);
267+
assert.equal(result.hint, undefined);
268+
});
269+
270+
test('runtime fill surfaces targetHittable and a hint for a non-hittable selector match (Maps pin case, #1037)', async () => {
271+
const calls: Array<{ point: Point; text: string }> = [];
272+
const device = createInteractionDevice(mapPinAnnotationSnapshot(), {
273+
fill: async (_context, point, text) => {
274+
calls.push({ point, text });
275+
},
276+
});
277+
278+
const result = await device.interactions.fill(
279+
selector('text="Anthropic - Headquarters"'),
280+
'ignored',
281+
{ session: 'default' },
282+
);
283+
284+
assert.equal(result.kind, 'selector');
285+
assert.equal(result.node?.label, 'Anthropic - Headquarters');
286+
assert.equal(result.targetHittable, false);
287+
assert.match(result.hint ?? '', /hittable: false/);
288+
assert.deepEqual(calls, [{ point: { x: 192, y: 461 }, text: 'ignored' }]);
289+
});
290+
237291
test('runtime click still promotes non-touchable nodes to hittable ancestors', async () => {
238292
const calls: Point[] = [];
239293
const device = createInteractionDevice(nonTouchableGroupSnapshot(), {
@@ -1092,6 +1146,31 @@ function nonHittableCellSnapshot(): SnapshotState {
10921146
]);
10931147
}
10941148

1149+
// Mirrors the #1037 Maps repro: a non-hittable map-pin annotation exact-matches
1150+
// `text="Anthropic - Headquarters"` while the real, longer-labeled row is a
1151+
// separate node. Fill/press must still proceed against the unique match (no
1152+
// stricter resolution) but should flag it as likely non-actionable.
1153+
function mapPinAnnotationSnapshot(): SnapshotState {
1154+
return makeSnapshotState([
1155+
{
1156+
index: 0,
1157+
depth: 0,
1158+
type: 'Other',
1159+
label: 'Anthropic - Headquarters',
1160+
rect: { x: 177, y: 446, width: 30, height: 30 },
1161+
hittable: false,
1162+
},
1163+
{
1164+
index: 1,
1165+
depth: 1,
1166+
type: 'Button',
1167+
label: 'Anthropic - Headquarters, 548 Market St',
1168+
rect: { x: 0, y: 786, width: 390, height: 60 },
1169+
hittable: false,
1170+
},
1171+
]);
1172+
}
1173+
10951174
function nonTouchableGroupSnapshot(): SnapshotState {
10961175
return makeSnapshotState([
10971176
{

‎src/commands/interaction/runtime/resolution.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ async function resolveRefInteractionTarget(
9393
action: params.action === 'fill' ? 'fill' : 'click',
9494
}),
9595
refLabel: resolveRefLabel(node, capture.snapshot.nodes),
96+
...describeNonHittableTarget(node, params.action),
9697
};
9798
}
9899

@@ -159,6 +160,26 @@ async function resolveSelectorInteractionTarget(
159160
action: params.action === 'fill' ? 'fill' : 'click',
160161
}),
161162
refLabel: resolveRefLabel(node, capture.snapshot.nodes),
163+
...describeNonHittableTarget(node, params.action),
164+
};
165+
}
166+
167+
/**
168+
* iOS AX `hittable` flags are unreliable on deep React Native trees (see #1037:
169+
* a map-pin annotation exact-matched a longer recents row label and reported tap
170+
* success while doing nothing visible). We deliberately do NOT fail or filter on
171+
* this signal — that would break selectors that only ever resolve to nodes the
172+
* platform marks non-hittable. Instead, surface it so the caller can notice a
173+
* likely no-op tap and re-target with a ref or a more specific selector/longer text.
174+
*/
175+
function describeNonHittableTarget(
176+
node: SnapshotNode,
177+
action: InteractionAction,
178+
): { targetHittable?: boolean; hint?: string } {
179+
if (node.hittable !== false) return {};
180+
return {
181+
targetHittable: false,
182+
hint: `The resolved element reports hittable: false, so this ${action} may have had no visible effect. Verify with a snapshot, or prefer a @ref or a longer/more specific selector to target the intended element.`,
162183
};
163184
}
164185

‎src/contracts/interaction.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@ export type ResolvedInteractionTarget =
4343
node?: SnapshotNode;
4444
selectorChain?: string[];
4545
refLabel?: string;
46+
targetHittable?: boolean;
47+
hint?: string;
4648
}
4749
| {
4850
kind: 'selector';
@@ -51,6 +53,8 @@ export type ResolvedInteractionTarget =
5153
node: SnapshotNode;
5254
selectorChain: string[];
5355
refLabel?: string;
56+
targetHittable?: boolean;
57+
hint?: string;
5458
};
5559

5660
export type PressCommandResult = ResolvedInteractionTarget & {

‎src/daemon/handlers/__tests__/interaction.test.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1739,6 +1739,9 @@ test('press @ref promotes a non-hittable node to its hittable ancestor before ta
17391739
expect(response.data?.ref).toBe('e2');
17401740
expect(response.data?.x).toBe(180);
17411741
expect(response.data?.y).toBe(136);
1742+
// Promotion landed on a hittable ancestor, so there is nothing to flag.
1743+
expect(response.data?.targetHittable).toBeUndefined();
1744+
expect(response.data?.hint).toBeUndefined();
17421745
}
17431746
expect(mockDispatch).toHaveBeenCalledTimes(1);
17441747
expect(mockDispatch.mock.calls[0]?.[1]).toBe('press');
@@ -1798,6 +1801,12 @@ test('press @ref does not promote to a full-screen hittable ancestor', async ()
17981801
if (response?.ok) {
17991802
expect(response.data?.x).toBe(201);
18001803
expect(response.data?.y).toBe(319);
1804+
// #1037: the press still proceeds against the non-hittable node (no stricter
1805+
// resolution), but the daemon response flags it so agents can react instead
1806+
// of assuming the tap had a visible effect.
1807+
expect(response.data?.targetHittable).toBe(false);
1808+
expect(typeof response.data?.hint).toBe('string');
1809+
expect(response.data?.hint as string).toMatch(/hittable: false/);
18011810
}
18021811
expect(mockDispatch).toHaveBeenCalledTimes(1);
18031812
expect(mockDispatch.mock.calls[0]?.[2]).toEqual(['201', '319']);

‎src/daemon/handlers/interaction-touch-targets.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,13 +132,17 @@ export function interactionResultExtra(
132132
ref: stripAtPrefix(result.target?.kind === 'ref' ? result.target.ref : undefined),
133133
refLabel: result.refLabel,
134134
selectorChain: result.selectorChain,
135+
targetHittable: result.targetHittable,
136+
hint: result.hint,
135137
};
136138
}
137139
if (result.kind === 'selector') {
138140
return {
139141
selector: result.target?.kind === 'selector' ? result.target.selector : undefined,
140142
selectorChain: result.selectorChain,
141143
refLabel: result.refLabel,
144+
targetHittable: result.targetHittable,
145+
hint: result.hint,
142146
};
143147
}
144148
return {};

‎src/mcp/command-output-schemas.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,8 @@ function interactionResultSchema(extra: InteractionExtra = {}): JsonSchema {
140140
node: snapshotNodeSchema,
141141
selectorChain: stringArraySchema,
142142
refLabel: stringSchema(),
143+
targetHittable: booleanSchema(),
144+
hint: stringSchema(),
143145
...extraProperties,
144146
},
145147
['kind', 'target', ...extraRequired],
@@ -152,6 +154,8 @@ function interactionResultSchema(extra: InteractionExtra = {}): JsonSchema {
152154
node: snapshotNodeSchema,
153155
selectorChain: stringArraySchema,
154156
refLabel: stringSchema(),
157+
targetHittable: booleanSchema(),
158+
hint: stringSchema(),
155159
...extraProperties,
156160
},
157161
['kind', 'point', 'target', 'node', 'selectorChain', ...extraRequired],

0 commit comments

Comments
 (0)