Skip to content

Commit 68f9a85

Browse files
committed
feat: direct iOS selector falls back to tree resolution on semantic failures (ADR 0011)
Runner ELEMENT_NOT_FOUND/AMBIGUOUS_MATCH on the direct iOS selector dispatch now delegate to the tree-based runtime path, which supplies runtime disambiguation, occlusion refusal, non-hittable promotion/annotation, and rich selector diagnostics/hints. Maestro replay dispatches (allowNonHittableCoordinateFallback) keep the runner-native error shapes: the new delegateSemanticFailures option is threaded from the dispatch flag, and the query path (allowElementNotFound at selector-runtime) keeps its exact semantics. Registry: direct-ios-selector errorTaxonomy/nonHittable/occlusion flip from gap waivers to delegated cells; the pinned gap list shrinks 6 -> 3. Success-path parity cells (disambiguation, responseIdentity) intentionally remain gaps per ADR 0011. Refs #1081
1 parent f3e07ff commit 68f9a85

7 files changed

Lines changed: 437 additions & 38 deletions

File tree

‎src/contracts/__tests__/interaction-guarantees.test.ts‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -180,9 +180,6 @@ test('acknowledged gaps are visible and bounded', () => {
180180
// (umbrella: https://github.com/callstack/agent-device/issues/1081).
181181
assert.deepEqual(gaps.sort(), [
182182
'direct-ios-selector/disambiguation',
183-
'direct-ios-selector/errorTaxonomy',
184-
'direct-ios-selector/nonHittable',
185-
'direct-ios-selector/occlusion',
186183
'direct-ios-selector/responseIdentity',
187184
'maestro-non-hittable-fallback/errorTaxonomy',
188185
]);

‎src/contracts/interaction-guarantees.ts‎

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -215,10 +215,9 @@ export const INTERACTION_DISPATCH_PATHS: Record<InteractionPathId, InteractionPa
215215
trackingIssue: GAPS_UMBRELLA_ISSUE,
216216
},
217217
occlusion: {
218-
kind: 'waived',
219-
reason:
220-
'gap: no explicit covered-element check on the direct path; closure strategy is delegation-on-error plus contract scenarios, since XCTest isHittable only approximates occlusion.',
221-
trackingIssue: GAPS_UMBRELLA_ISSUE,
218+
kind: 'delegated',
219+
to: 'runtime-selector',
220+
via: 'runner ELEMENT_NOT_FOUND/AMBIGUOUS_MATCH fall back to tree-based resolution (isDirectIosSelectorFallbackError delegateSemanticFailures; non-maestro dispatches only) — XCTest skips covered/non-hittable matches, so the runtime path raises the covered-element refusal with its hint',
222221
},
223222
offscreen: {
224223
// Decision: TapPointPolicy (pure geometry, parity-tested against the
@@ -229,10 +228,9 @@ export const INTERACTION_DISPATCH_PATHS: Record<InteractionPathId, InteractionPa
229228
parityTable: 'contracts/fixtures/tap-point-policy.json',
230229
},
231230
nonHittable: {
232-
kind: 'waived',
233-
reason:
234-
'gap: non-hittable matches are skipped runner-side (ELEMENT_NOT_FOUND) instead of promoted/annotated; closure strategy is delegation-on-error into the runtime path.',
235-
trackingIssue: GAPS_UMBRELLA_ISSUE,
231+
kind: 'delegated',
232+
to: 'runtime-selector',
233+
via: 'runner ELEMENT_NOT_FOUND (non-hittable matches are skipped runner-side) falls back to tree-based resolution (isDirectIosSelectorFallbackError delegateSemanticFailures; non-maestro dispatches only), which promotes to a hittable ancestor or annotates targetHittable/hint',
236234
},
237235
responseConstruction: SHARED_RESPONSE_CONSTRUCTION,
238236
responseIdentity: {
@@ -246,10 +244,9 @@ export const INTERACTION_DISPATCH_PATHS: Record<InteractionPathId, InteractionPa
246244
via: '--verify disables the direct path (readDirectIosSelectorTapTarget / fill flags.verify check)',
247245
},
248246
errorTaxonomy: {
249-
kind: 'waived',
250-
reason:
251-
'gap: ELEMENT_NOT_FOUND/AMBIGUOUS_MATCH lack the selector diagnostics and hints the runtime path attaches; closure strategy is delegation-on-error for the failure shapes.',
252-
trackingIssue: GAPS_UMBRELLA_ISSUE,
247+
kind: 'delegated',
248+
to: 'runtime-selector',
249+
via: 'runner ELEMENT_NOT_FOUND/AMBIGUOUS_MATCH fall back to tree-based resolution (isDirectIosSelectorFallbackError delegateSemanticFailures; non-maestro dispatches only), which attaches the shared no-match diagnostics, ambiguous shape, and hints',
253250
},
254251
},
255252
},

‎src/daemon/__tests__/direct-ios-selector.test.ts‎

Lines changed: 34 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,47 @@ test('runner ELEMENT_OFFSCREEN always falls back to tree-based resolution', () =
77
const error = new AppError('ELEMENT_OFFSCREEN', 'element resolved off-screen at (-161, 265)');
88
assert.equal(isDirectIosSelectorFallbackError(error), true);
99
assert.equal(isDirectIosSelectorFallbackError(error, { allowElementNotFound: false }), true);
10+
assert.equal(isDirectIosSelectorFallbackError(error, { delegateSemanticFailures: false }), true);
1011
});
1112

12-
test('runner ELEMENT_NOT_FOUND falls back only when the caller allows it', () => {
13+
test('runner ELEMENT_NOT_FOUND falls back for query callers that allow it', () => {
1314
const error = new AppError('ELEMENT_NOT_FOUND', 'element not found');
1415
assert.equal(isDirectIosSelectorFallbackError(error), false);
1516
assert.equal(isDirectIosSelectorFallbackError(error, { allowElementNotFound: true }), true);
1617
});
1718

19+
test('semantic failures delegate to the runtime path for interaction dispatches (ADR 0011)', () => {
20+
const notFound = new AppError('ELEMENT_NOT_FOUND', 'element not found');
21+
const ambiguous = new AppError('AMBIGUOUS_MATCH', 'multiple');
22+
assert.equal(
23+
isDirectIosSelectorFallbackError(notFound, { delegateSemanticFailures: true }),
24+
true,
25+
);
26+
assert.equal(
27+
isDirectIosSelectorFallbackError(ambiguous, { delegateSemanticFailures: true }),
28+
true,
29+
);
30+
});
31+
32+
test('maestro replay dispatches preserve the runner semantic error shapes (no fallback)', () => {
33+
const notFound = new AppError('ELEMENT_NOT_FOUND', 'element not found');
34+
const ambiguous = new AppError('AMBIGUOUS_MATCH', 'multiple');
35+
assert.equal(
36+
isDirectIosSelectorFallbackError(notFound, { delegateSemanticFailures: false }),
37+
false,
38+
);
39+
assert.equal(
40+
isDirectIosSelectorFallbackError(ambiguous, { delegateSemanticFailures: false }),
41+
false,
42+
);
43+
});
44+
45+
test('AMBIGUOUS_MATCH does not fall back on the query path (allowElementNotFound callers)', () => {
46+
const ambiguous = new AppError('AMBIGUOUS_MATCH', 'multiple');
47+
assert.equal(isDirectIosSelectorFallbackError(ambiguous), false);
48+
assert.equal(isDirectIosSelectorFallbackError(ambiguous, { allowElementNotFound: true }), false);
49+
});
50+
1851
test('transport-level COMMAND_FAILED errors fall back, semantic ones do not', () => {
1952
assert.equal(
2053
isDirectIosSelectorFallbackError(new AppError('COMMAND_FAILED', 'fetch failed')),
@@ -30,8 +63,4 @@ test('transport-level COMMAND_FAILED errors fall back, semantic ones do not', ()
3063
isDirectIosSelectorFallbackError(new AppError('COMMAND_FAILED', 'element covered by overlay')),
3164
false,
3265
);
33-
assert.equal(
34-
isDirectIosSelectorFallbackError(new AppError('AMBIGUOUS_MATCH', 'multiple')),
35-
false,
36-
);
3766
});

‎src/daemon/direct-ios-selector.ts‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,16 +31,43 @@ function isRunnerNativeSelectorKey(key: string): key is DirectIosSelectorTarget[
3131

3232
export function isDirectIosSelectorFallbackError(
3333
error: unknown,
34-
options: { allowElementNotFound?: boolean } = {},
34+
options: {
35+
/**
36+
* Read/query callers (wait/get/is): a runner ELEMENT_NOT_FOUND re-resolves
37+
* against the daemon tree, but AMBIGUOUS_MATCH still surfaces as-is.
38+
*/
39+
allowElementNotFound?: boolean;
40+
/**
41+
* ADR 0011 delegation-on-error for interaction dispatches: the runner's
42+
* semantic failure shapes (ELEMENT_NOT_FOUND, AMBIGUOUS_MATCH) fall back
43+
* to the tree-based runtime path, which supplies runtime disambiguation,
44+
* non-hittable promotion/annotation, occlusion refusal, and rich selector
45+
* diagnostics/hints. Must stay OFF for Maestro replay dispatches
46+
* (allowNonHittableCoordinateFallback): replay matching is intentionally
47+
* runner-native, so those error shapes must surface unchanged.
48+
*/
49+
delegateSemanticFailures?: boolean;
50+
} = {},
3551
): boolean {
3652
const appError = asAppError(error);
37-
if (appError.code === 'ELEMENT_NOT_FOUND') return options.allowElementNotFound === true;
53+
if (appError.code === 'ELEMENT_NOT_FOUND') {
54+
return options.delegateSemanticFailures === true || options.allowElementNotFound === true;
55+
}
56+
if (appError.code === 'AMBIGUOUS_MATCH') return options.delegateSemanticFailures === true;
3857
// The runner refuses to tap a hittable match whose frame is outside the app
3958
// frame (closed drawer, off-viewport carousel). The tree-based path either
4059
// prefers an on-screen candidate or raises the actionable offscreen_selector
4160
// error, so always fall back.
4261
if (appError.code === 'ELEMENT_OFFSCREEN') return true;
4362
if (appError.code !== 'COMMAND_FAILED') return false;
63+
// Transport-failure classification stays message-based deliberately: the
64+
// sniffed shapes originate at 4+ scattered throw sites (runner-transport
65+
// deadline errors, runner-contract connect errors, runner-session's invalid
66+
// response) plus raw undici "fetch failed" TypeErrors that are only wrapped
67+
// into AppError at this boundary — and isRetryableRunnerError performs the
68+
// same message sniffing for retry policy. A typed transport marker needs a
69+
// wrapping layer around all of them in one change; tracked as Tier-3 error
70+
// cleanup, not worth entangling with this fallback decision.
4471
const message = appError.message.toLowerCase();
4572
return (
4673
message.includes('fetch failed') ||

‎src/daemon/handlers/__tests__/interaction.test.ts‎

Lines changed: 110 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -657,17 +657,38 @@ test('click simple iOS id selector falls back to snapshot coordinates on transpo
657657
}
658658
});
659659

660-
test('click simple iOS id selector does not snapshot-fallback on runner element miss', async () => {
660+
test('click simple iOS id selector falls back to snapshot resolution on runner element miss', async () => {
661661
const sessionStore = makeSessionStore();
662662
const sessionName = 'ios-direct-selector-element-miss';
663663
sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' }));
664664

665-
mockDispatch.mockImplementation(async (_device, command, _positionals, _out, context) => {
665+
mockDispatch.mockImplementation(async (_device, command, positionals, _out, context) => {
666666
if (command === 'press' && (context as Record<string, unknown>)?.directElementSelector) {
667667
throw new AppError('ELEMENT_NOT_FOUND', 'element not found');
668668
}
669669
if (command === 'snapshot') {
670-
throw new Error('snapshot fallback should not run');
670+
return {
671+
nodes: attachRefs([
672+
{
673+
index: 0,
674+
type: 'Window',
675+
rect: { x: 0, y: 0, width: 390, height: 844 },
676+
},
677+
{
678+
index: 1,
679+
parentIndex: 0,
680+
type: 'XCUIElementTypeButton',
681+
identifier: 'submit',
682+
rect: { x: 20, y: 80, width: 120, height: 40 },
683+
enabled: true,
684+
hittable: true,
685+
},
686+
]),
687+
backend: 'xctest',
688+
};
689+
}
690+
if (command === 'press') {
691+
return { x: Number(positionals[0]), y: Number(positionals[1]), pressed: true };
671692
}
672693
return {};
673694
});
@@ -685,24 +706,58 @@ test('click simple iOS id selector does not snapshot-fallback on runner element
685706
contextFromFlags,
686707
});
687708

688-
expect(response?.ok).toBe(false);
689-
if (response?.ok === false) {
690-
expect(response.error.code).toBe('ELEMENT_NOT_FOUND');
709+
expect(response?.ok).toBe(true);
710+
const pressCalls = mockDispatch.mock.calls.filter((call) => call[1] === 'press');
711+
expect(pressCalls.length).toBe(2);
712+
expect(pressCalls[1]?.[2]).toEqual(['80', '100']);
713+
if (response?.ok) {
714+
expect(response.data?.selectorChain).toContain('id="submit"');
691715
}
692-
expect(mockDispatch.mock.calls.filter((call) => call[1] === 'snapshot')).toHaveLength(0);
693716
});
694717

695-
test('click simple iOS id selector does not snapshot-fallback on ambiguous runner match', async () => {
718+
test('click simple iOS id selector falls back to runtime disambiguation on ambiguous runner match', async () => {
696719
const sessionStore = makeSessionStore();
697720
const sessionName = 'ios-direct-selector-ambiguous';
698721
sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' }));
699722

700-
mockDispatch.mockImplementation(async (_device, command, _positionals, _out, context) => {
723+
mockDispatch.mockImplementation(async (_device, command, positionals, _out, context) => {
701724
if (command === 'press' && (context as Record<string, unknown>)?.directElementSelector) {
702725
throw new AppError('AMBIGUOUS_MATCH', 'Selector matched multiple elements');
703726
}
704727
if (command === 'snapshot') {
705-
throw new Error('snapshot fallback should not run');
728+
return {
729+
nodes: attachRefs([
730+
{
731+
index: 0,
732+
type: 'Window',
733+
rect: { x: 0, y: 0, width: 390, height: 844 },
734+
},
735+
// Off-screen drawer twin: runtime disambiguation prefers the
736+
// visible candidate instead of failing like the runner did.
737+
{
738+
index: 1,
739+
parentIndex: 0,
740+
type: 'XCUIElementTypeButton',
741+
identifier: 'submit',
742+
rect: { x: -300, y: 80, width: 120, height: 40 },
743+
enabled: true,
744+
hittable: true,
745+
},
746+
{
747+
index: 2,
748+
parentIndex: 0,
749+
type: 'XCUIElementTypeButton',
750+
identifier: 'submit',
751+
rect: { x: 20, y: 80, width: 120, height: 40 },
752+
enabled: true,
753+
hittable: true,
754+
},
755+
]),
756+
backend: 'xctest',
757+
};
758+
}
759+
if (command === 'press') {
760+
return { x: Number(positionals[0]), y: Number(positionals[1]), pressed: true };
706761
}
707762
return {};
708763
});
@@ -720,13 +775,53 @@ test('click simple iOS id selector does not snapshot-fallback on ambiguous runne
720775
contextFromFlags,
721776
});
722777

723-
expect(response?.ok).toBe(false);
724-
if (response?.ok === false) {
725-
expect(response.error.code).toBe('AMBIGUOUS_MATCH');
726-
}
727-
expect(mockDispatch.mock.calls.filter((call) => call[1] === 'snapshot')).toHaveLength(0);
778+
expect(response?.ok).toBe(true);
779+
const pressCalls = mockDispatch.mock.calls.filter((call) => call[1] === 'press');
780+
expect(pressCalls.length).toBe(2);
781+
expect(pressCalls[1]?.[2]).toEqual(['80', '100']);
728782
});
729783

784+
test.each([
785+
['ELEMENT_NOT_FOUND', 'element not found'],
786+
['AMBIGUOUS_MATCH', 'Selector matched multiple elements'],
787+
] as const)(
788+
'maestro-flagged click keeps runner %s error without snapshot fallback',
789+
async (code, message) => {
790+
const sessionStore = makeSessionStore();
791+
const sessionName = `ios-maestro-direct-selector-${code}`;
792+
sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' }));
793+
794+
mockDispatch.mockImplementation(async (_device, command, _positionals, _out, context) => {
795+
if (command === 'press' && (context as Record<string, unknown>)?.directElementSelector) {
796+
throw new AppError(code, message);
797+
}
798+
if (command === 'snapshot') {
799+
throw new Error('snapshot fallback should not run for maestro replay dispatches');
800+
}
801+
return {};
802+
});
803+
804+
const response = await handleInteractionCommands({
805+
req: {
806+
token: 't',
807+
session: sessionName,
808+
command: 'click',
809+
positionals: ['id="submit"'],
810+
flags: { maestro: { allowNonHittableCoordinateFallback: true } },
811+
},
812+
sessionName,
813+
sessionStore,
814+
contextFromFlags,
815+
});
816+
817+
expect(response?.ok).toBe(false);
818+
if (response?.ok === false) {
819+
expect(response.error.code).toBe(code);
820+
}
821+
expect(mockDispatch.mock.calls.filter((call) => call[1] === 'snapshot')).toHaveLength(0);
822+
},
823+
);
824+
730825
test('click simple iOS id selector waits for snapshot path after pending gesture stabilization', async () => {
731826
const sessionStore = makeSessionStore();
732827
const sessionName = 'ios-direct-selector-after-swipe';

‎src/daemon/handlers/interaction-touch.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -352,7 +352,13 @@ async function dispatchDirectIosSelectorInteraction(params: {
352352
actionFinishedAt,
353353
});
354354
} catch (error) {
355-
if (!isDirectIosSelectorFallbackError(error)) {
355+
// ADR 0011 delegation-on-error: semantic runner failures fall back to the
356+
// tree-based runtime path — except for Maestro replay dispatches, whose
357+
// runner-native error shapes must be preserved.
358+
const fallback = isDirectIosSelectorFallbackError(error, {
359+
delegateSemanticFailures: selector.allowNonHittableCoordinateFallback !== true,
360+
});
361+
if (!fallback) {
356362
return { ok: false, error: normalizeError(error) };
357363
}
358364
emitDiagnostic({

0 commit comments

Comments
 (0)