Skip to content

Commit f7a36b4

Browse files
committed
fix(selectors): type rect-requiring targets as addressable
1 parent 971d24d commit f7a36b4

14 files changed

Lines changed: 431 additions & 114 deletions

‎packages/daemon/src/execution/interaction/internal/find.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import { SELECTOR_PIPELINE_POLICIES } from '@agent-device/selectors/selector-pip
1010
import { coveredInteractionDetails } from '@agent-device/selectors/interaction-error';
1111
import {
1212
centerOfRect,
13+
type SnapshotAddressableNode,
1314
type SnapshotKeyboardBandFact,
1415
type SnapshotState,
1516
} from '@agent-device/kernel/snapshot';
@@ -62,7 +63,7 @@ type FindContext = {
6263

6364
type ResolvedMatch = {
6465
node: SnapshotState['nodes'][number];
65-
resolvedNode: SnapshotState['nodes'][number];
66+
resolvedNode: SnapshotAddressableNode;
6667
ref: string;
6768
nodes: SnapshotState['nodes'];
6869
/** The in-place iOS system surface the target capture described (#2438), if any. */
@@ -75,7 +76,7 @@ type ResolvedMatch = {
7576
* seam surfaces it: click/fill re-enter the interaction leaf, which owns
7677
* that refusal's shape and raises it against the same node.
7778
*/
78-
occludedNode?: SnapshotState['nodes'][number];
79+
occludedNode?: SnapshotAddressableNode;
7980
};
8081

8182
export async function handleFindCommands(params: FindRouteInput): Promise<DaemonResponse | null> {

‎packages/kernel/src/snapshot.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -447,6 +447,8 @@ export type SnapshotNode = RawSnapshotNode & {
447447
inheritsIdentifier?: true;
448448
};
449449

450+
export type SnapshotAddressableNode = SnapshotNode & { ref: string };
451+
450452
/**
451453
* Platform role vocabulary → the text presenter's normalized label. Owned here (rather than
452454
* capture-kit) so `attachRefs` can publish the same `kind` on every node without capture-kit
@@ -837,7 +839,7 @@ export function attachRefs(
837839
* address (an overlay label, a settle tail entry, a ref pin). A ref-less node is readable and
838840
* searchable, never addressable.
839841
*/
840-
export function hasSnapshotRef(node: SnapshotNode): node is SnapshotNode & { ref: string } {
842+
export function hasSnapshotRef(node: SnapshotNode): node is SnapshotAddressableNode {
841843
return node.ref !== undefined;
842844
}
843845

‎packages/selectors/src/interaction-error.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
1-
import type { Rect, SnapshotCoveringNode, SnapshotNode } from '@agent-device/kernel/snapshot';
1+
import type {
2+
Rect,
3+
SnapshotAddressableNode,
4+
SnapshotCoveringNode,
5+
} from '@agent-device/kernel/snapshot';
26

37
/**
48
* Machine-readable `error.details.reason` values shared by interaction producers and adapters.
@@ -44,7 +48,7 @@ export const INTERACTION_ERROR_REASONS = {
4448
} as const;
4549

4650
/** The part of a covered node a refusal reports. */
47-
type CoveredNodeFacts = Pick<SnapshotNode, 'ref' | 'interactionBlocked' | 'coveredBy'>;
51+
type CoveredNodeFacts = Pick<SnapshotAddressableNode, 'ref' | 'interactionBlocked' | 'coveredBy'>;
4852

4953
/**
5054
* The whole `error.details` shape of a covered refusal, shared by every producer (the interaction

‎packages/selectors/src/internal/public-resolution-types.ts‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,21 +12,21 @@ export type SelectorDiagnostics = {
1212
* The disclosure for an ambiguous selector that was resolved by the heuristic;
1313
* present only when the heuristic picked among N>1 matches (ADR 0012).
1414
*/
15-
export type SelectorDisambiguationDisclosure = {
15+
export type SelectorDisambiguationDisclosure<TNode extends SnapshotNode = SnapshotNode> = {
1616
matchCount: number;
1717
tiebreak: DisambiguationTiebreak;
1818
/** Every losing matched node, document order, uncapped (response layer caps). */
19-
alternatives: SnapshotNode[];
19+
alternatives: TNode[];
2020
};
2121

2222
/** String-only resolution result; the parser representation remains package-private. */
23-
export type SelectorResolution = {
24-
node: SnapshotNode;
23+
export type SelectorResolution<TNode extends SnapshotNode = SnapshotNode> = {
24+
node: TNode;
2525
selector: string;
2626
selectorIndex: number;
2727
matches: number;
2828
diagnostics: SelectorDiagnostics[];
29-
disambiguation?: SelectorDisambiguationDisclosure;
29+
disambiguation?: SelectorDisambiguationDisclosure<TNode>;
3030
};
3131

3232
/**

‎packages/selectors/src/selector-pipeline.test.ts‎

Lines changed: 222 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
import assert from 'node:assert/strict';
22
import { test } from 'vitest';
3-
import type { RawSnapshotNode, SnapshotNode } from '@agent-device/kernel/snapshot';
3+
import type {
4+
RawSnapshotNode,
5+
SnapshotAddressableNode,
6+
SnapshotNode,
7+
} from '@agent-device/kernel/snapshot';
48
import { SELECTOR_RESOLUTION_POLICIES } from '@agent-device/selectors';
59
import { makeSnapshotState } from './snapshot-geometry.fixtures.ts';
610
import {
@@ -14,6 +18,7 @@ import {
1418
} from './interaction-targeting.fixtures.ts';
1519
import {
1620
listSelectorPipelineMatches,
21+
resolveActionSelectorPipeline,
1722
resolveSelectorPipeline,
1823
runNodePipelineStages,
1924
selectorPollBudget,
@@ -121,6 +126,90 @@ const ROOT_ANCESTOR_TREE: RawSnapshotNode[] = [
121126
},
122127
];
123128

129+
const REFLESS_RECT_TREE: SnapshotNode[] = [
130+
{
131+
index: 0,
132+
depth: 0,
133+
type: 'XCUIElementTypeButton',
134+
label: 'Addressless',
135+
rect: { x: 20, y: 30, width: 100, height: 40 },
136+
hittable: true,
137+
kind: 'button',
138+
},
139+
];
140+
141+
const REFLESS_PROMOTION_TREE: SnapshotNode[] = [
142+
{
143+
index: 0,
144+
depth: 0,
145+
type: 'XCUIElementTypeApplication',
146+
rect: { x: 0, y: 0, width: 390, height: 844 },
147+
hittable: true,
148+
kind: 'application',
149+
ref: 'e1',
150+
},
151+
{
152+
index: 1,
153+
depth: 1,
154+
parentIndex: 0,
155+
type: 'XCUIElementTypeCell',
156+
label: 'Addressless Row',
157+
rect: { x: 20, y: 30, width: 180, height: 50 },
158+
hittable: true,
159+
kind: 'cell',
160+
},
161+
{
162+
index: 2,
163+
depth: 2,
164+
parentIndex: 1,
165+
type: 'XCUIElementTypeStaticText',
166+
label: 'Addressless Row',
167+
rect: { x: 30, y: 40, width: 120, height: 20 },
168+
hittable: false,
169+
kind: 'text',
170+
ref: 'e3',
171+
},
172+
];
173+
174+
const REFLESS_STRUCTURAL_ANCESTOR_TREE: SnapshotNode[] = [
175+
{
176+
index: 0,
177+
depth: 0,
178+
type: 'XCUIElementTypeApplication',
179+
rect: { x: 0, y: 0, width: 390, height: 844 },
180+
kind: 'application',
181+
ref: 'e1',
182+
},
183+
{
184+
index: 1,
185+
depth: 1,
186+
parentIndex: 0,
187+
type: 'XCUIElementTypeOther',
188+
identifier: 'bridged-control',
189+
rect: { x: 20, y: 30, width: 36, height: 36 },
190+
kind: 'other',
191+
ref: 'e2',
192+
},
193+
{
194+
index: 2,
195+
depth: 2,
196+
parentIndex: 1,
197+
type: 'XCUIElementTypeOther',
198+
rect: { x: 20, y: 30, width: 36, height: 36 },
199+
kind: 'other',
200+
},
201+
{
202+
index: 3,
203+
depth: 3,
204+
parentIndex: 2,
205+
type: 'XCUIElementTypeButton',
206+
identifier: 'bridged-control',
207+
rect: { x: 20, y: 30, width: 36, height: 36 },
208+
kind: 'button',
209+
ref: 'e4',
210+
},
211+
];
212+
124213
function nodesOf(raw: RawSnapshotNode[]): SnapshotNode[] {
125214
return makeSnapshotState(raw).nodes;
126215
}
@@ -455,6 +544,138 @@ test('a listing row cannot be handed the node stages it does not declare', () =>
455544
void [nodeStages, pollBudget];
456545
});
457546

547+
test('rect-requiring pipeline entries narrow outcome nodes to addressable nodes', async () => {
548+
const nodes = nodesOf(PROMOTABLE_TREE);
549+
const staged = await runNodePipelineStages(
550+
SELECTOR_PIPELINE_POLICIES.promotedTarget,
551+
nodes,
552+
nodes[0]!,
553+
{ offscreen: async (node) => node },
554+
);
555+
const stagedAddressable: SnapshotAddressableNode = staged.node;
556+
const stagedRef: string = staged.node.ref;
557+
558+
const resolved = await resolveSelectorPipeline(
559+
SELECTOR_PIPELINE_POLICIES.coveredDiagnosis,
560+
nodes,
561+
'label="Account"',
562+
MATCH,
563+
);
564+
if (resolved.kind === 'target') {
565+
const resolvedAddressable: SnapshotAddressableNode = resolved.node;
566+
const resolvedRef: string = resolved.node.ref;
567+
void [resolvedAddressable, resolvedRef];
568+
}
569+
570+
const listed = listSelectorPipelineMatches(
571+
SELECTOR_PIPELINE_POLICIES.findAct,
572+
nodes,
573+
'label="Account"',
574+
MATCH,
575+
);
576+
const listedAddressable: SnapshotAddressableNode = listed.list!.matchedNodes[0]!;
577+
const listedRef: string = listed.list!.matchedNodes[0]!.ref;
578+
579+
const action = resolveActionSelectorPipeline(
580+
SELECTOR_PIPELINE_POLICIES.promotedTarget,
581+
nodes,
582+
'label="Account"',
583+
MATCH,
584+
);
585+
if (action.kind === 'resolved') {
586+
const actionAddressable: SnapshotAddressableNode = action.resolution.node;
587+
const actionRef: string = action.resolution.node.ref;
588+
void [actionAddressable, actionRef];
589+
}
590+
591+
const read = await runNodePipelineStages(SELECTOR_PIPELINE_POLICIES.readAny, nodes, nodes[0]!);
592+
// @ts-expect-error read rows do not prove that a selectable node is addressable.
593+
const readRef: string = read.node.ref;
594+
595+
const readList = listSelectorPipelineMatches(
596+
SELECTOR_PIPELINE_POLICIES.readList,
597+
nodes,
598+
'label="Account"',
599+
MATCH,
600+
);
601+
// @ts-expect-error listing rows without rect requirements do not prove addressability.
602+
const readListRef: string = readList.list!.matchedNodes[0]!.ref;
603+
void [stagedAddressable, stagedRef, listedAddressable, listedRef, readRef, readListRef];
604+
});
605+
606+
test('rect-requiring pipeline entries do not expose ref-less rect nodes as addressable', async () => {
607+
const listed = listSelectorPipelineMatches(
608+
SELECTOR_PIPELINE_POLICIES.findAct,
609+
REFLESS_RECT_TREE,
610+
'label="Addressless"',
611+
MATCH,
612+
);
613+
assert.equal(listed.list, null);
614+
615+
const resolved = await resolveSelectorPipeline(
616+
SELECTOR_PIPELINE_POLICIES.coveredDiagnosis,
617+
REFLESS_RECT_TREE,
618+
'label="Addressless"',
619+
MATCH,
620+
);
621+
assert.equal(resolved.kind, 'none');
622+
623+
const read = await resolveSelectorPipeline(
624+
SELECTOR_PIPELINE_POLICIES.readAny,
625+
REFLESS_RECT_TREE,
626+
'label="Addressless"',
627+
MATCH,
628+
);
629+
assert.equal(read.kind, 'target');
630+
if (read.kind === 'target') assert.equal(read.node.ref, undefined);
631+
632+
const action = resolveActionSelectorPipeline(
633+
SELECTOR_PIPELINE_POLICIES.promotedTarget,
634+
REFLESS_RECT_TREE,
635+
'label="Addressless"',
636+
MATCH,
637+
);
638+
assert.equal(action.kind, 'none');
639+
});
640+
641+
test('acting selector classification keeps full ancestry after candidate addressability filtering', () => {
642+
const action = resolveActionSelectorPipeline(
643+
SELECTOR_PIPELINE_POLICIES.promotedTarget,
644+
REFLESS_STRUCTURAL_ANCESTOR_TREE,
645+
'id="bridged-control"',
646+
MATCH,
647+
);
648+
assert.equal(action.kind, 'resolved');
649+
if (action.kind !== 'resolved') return;
650+
assert.equal(action.resolution.node.index, 3);
651+
assert.equal(action.resolution.matches, 2);
652+
assert.equal(action.resolution.disambiguation?.tiebreak, 'structural-equivalence');
653+
});
654+
655+
test('rect-requiring node stages fail closed if promotion lands on a ref-less node', async () => {
656+
await assert.rejects(
657+
() =>
658+
runNodePipelineStages(
659+
SELECTOR_PIPELINE_POLICIES.resolvedTarget,
660+
REFLESS_RECT_TREE,
661+
REFLESS_RECT_TREE[0]!,
662+
{ offscreen: async (node) => node },
663+
),
664+
/requires an addressable target/,
665+
);
666+
667+
await assert.rejects(
668+
() =>
669+
runNodePipelineStages(
670+
SELECTOR_PIPELINE_POLICIES.promotedTarget,
671+
REFLESS_PROMOTION_TREE,
672+
REFLESS_PROMOTION_TREE[2]!,
673+
{ offscreen: async (node) => node },
674+
),
675+
/requires an addressable target/,
676+
);
677+
});
678+
458679
test('pipeline rows declare only the stages their kind enforces', () => {
459680
// The #1649 rule, carried to this table: a column nothing consumes is an
460681
// unverifiable claim that reads as truth. A row declares the stages ITS KIND

0 commit comments

Comments
 (0)