Skip to content

Commit 2964115

Browse files
committed
fix(maestro): give target-resolution misses their own typed failure reason
An out-of-range selector index over otherwise-visible matches produced a resolution with matched/visible evidence but no target, and every miss was reported with the same generic 'did not resolve to a visible element' text. assertVisible then accepted the vacuous matched/visible pair, so a flow could assert an element its selector never selected. Carry a MaestroTargetFailureReason from the resolution site through the daemon match projection to the reporter: messages now name the selector and the actual miss (no match, none visible, index out of range, no usable geometry), the typed reason rides along in error details, and observation conditions hold only for an actionable resolution.
1 parent 24b2ce6 commit 2964115

10 files changed

Lines changed: 266 additions & 9 deletions

‎packages/maestro/src/daemon-port/__tests__/daemon-runtime-port-observation.test.ts‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,46 @@ test('computes expensive target evidence only for the command policies that cons
150150
expect(relational).not.toHaveProperty('dispatchSelector');
151151
});
152152

153+
test('projects an out-of-range index miss onto target evidence with its typed failure reason', () => {
154+
const snapshot = makeSnapshot([
155+
{ index: 0, type: 'Application', rect: { x: 0, y: 0, width: 402, height: 874 } },
156+
{
157+
index: 1,
158+
parentIndex: 0,
159+
type: 'Button',
160+
kind: formatRole('Button'),
161+
identifier: 'not-now',
162+
label: 'Not Now',
163+
rect: { x: 20, y: 700, width: 120, height: 44 },
164+
},
165+
{
166+
index: 2,
167+
parentIndex: 0,
168+
type: 'Button',
169+
kind: formatRole('Button'),
170+
identifier: 'not-now',
171+
label: 'Not Now',
172+
rect: { x: 20, y: 760, width: 120, height: 44 },
173+
},
174+
]);
175+
176+
const miss = resolveTypedMaestroTarget({
177+
context: { generation: 3, env: {} },
178+
snapshot,
179+
platform: 'ios',
180+
query: { selector: { id: 'not-now', index: 2 }, purpose: 'tap', timeoutMs: 0 },
181+
});
182+
183+
expect(miss).toMatchObject({
184+
generation: 3,
185+
matched: true,
186+
visible: true,
187+
candidateCount: 2,
188+
failureReason: 'index-out-of-range',
189+
});
190+
expect(miss).not.toHaveProperty('rect');
191+
});
192+
153193
test('matches iOS Maestro ids on semantic nodes suppressed from interactive presentation', async () => {
154194
const snapshot = makeSnapshot([
155195
{ index: 0, type: 'Application', rect: { x: 0, y: 0, width: 402, height: 874 } },

‎packages/maestro/src/daemon-port/daemon-runtime-port-observation.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,7 @@ function targetMatchFromResolution(
272272
matched: resolution.evidence.matched,
273273
visible: resolution.evidence.visible,
274274
candidateCount: resolution.evidence.candidateCount,
275+
failureReason: resolution.failureReason,
275276
...(resolution.evidence.ref ? { ref: resolution.evidence.ref } : {}),
276277
...(viewport ? { viewport } : {}),
277278
};

‎packages/maestro/src/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ export {
6161
type MaestroSelector,
6262
type MaestroSinglePointerGestureInput,
6363
type MaestroSnapshotTargetQuery,
64+
type MaestroTargetFailureReason,
6465
type MaestroTargetMatch,
6566
type MaestroTargetQuery,
6667
type MaestroTargetResolution,
Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
import { expect, test } from 'vitest';
2+
import { maestroObservationMatches } from '../runtime-port-observation.ts';
3+
4+
const visibleCondition = { kind: 'visible' as const, selector: { text: 'Not Now' } };
5+
const notVisibleCondition = {
6+
kind: 'notVisible' as const,
7+
selector: { text: 'Not Now' },
8+
timeoutMs: 0,
9+
};
10+
11+
test('a visible condition requires an actionable resolution, not just matched evidence', () => {
12+
const resolved = { generation: 0, matched: true, visible: true, candidateCount: 1 };
13+
14+
expect(maestroObservationMatches(visibleCondition, resolved)).toBe(true);
15+
// An out-of-range `index` over otherwise visible matches leaves matched and
16+
// visible evidence with no selected element; the condition must not hold.
17+
expect(
18+
maestroObservationMatches(visibleCondition, {
19+
...resolved,
20+
failureReason: 'index-out-of-range',
21+
}),
22+
).toBe(false);
23+
expect(
24+
maestroObservationMatches(visibleCondition, {
25+
...resolved,
26+
matched: false,
27+
visible: false,
28+
failureReason: 'selector-did-not-match',
29+
}),
30+
).toBe(false);
31+
});
32+
33+
test('a notVisible condition is the strict complement of an actionable resolution', () => {
34+
const resolved = { generation: 0, matched: true, visible: true, candidateCount: 1 };
35+
36+
expect(maestroObservationMatches(notVisibleCondition, resolved)).toBe(false);
37+
expect(
38+
maestroObservationMatches(notVisibleCondition, {
39+
...resolved,
40+
failureReason: 'index-out-of-range',
41+
}),
42+
).toBe(true);
43+
expect(
44+
maestroObservationMatches(notVisibleCondition, {
45+
...resolved,
46+
matched: false,
47+
visible: false,
48+
failureReason: 'selector-did-not-match',
49+
}),
50+
).toBe(true);
51+
});

‎packages/maestro/src/internal/__tests__/runtime-port.test.ts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -351,6 +351,35 @@ describe('MaestroRuntimePort', () => {
351351
expect(operations.resolveGestureViewport).not.toHaveBeenCalled();
352352
});
353353

354+
test('reports an out-of-range index with its own message and typed reason', async () => {
355+
const operations = makeOperations({
356+
resolveTarget: async (_input, context) => ({
357+
generation: context.generation,
358+
matched: true,
359+
visible: true,
360+
candidateCount: 2,
361+
failureReason: 'index-out-of-range',
362+
ref: undefined,
363+
}),
364+
tapOn: async () => undefined,
365+
});
366+
const command = parseMaestroProgram('---\n- tapOn:\n text: Not Now\n index: 2\n')
367+
.commands[0]!;
368+
369+
await expect(
370+
createMaestroRuntimePort(operations).execute({
371+
command: command as Extract<typeof command, { kind: 'tapOn' }>,
372+
generation: 0,
373+
env: {},
374+
invalidateObservation: vi.fn(),
375+
}),
376+
).rejects.toMatchObject({
377+
message:
378+
'Maestro target "text":"Not Now", "index":2 matched 2 element(s); its index is out of range.',
379+
details: { reason: 'maestro-test-failure', targetFailureReason: 'index-out-of-range' },
380+
});
381+
});
382+
354383
test('reports optional tap misses to the interpreter without hiding infrastructure failures', async () => {
355384
const tapOn = vi.fn(async () => undefined);
356385
const missingOperations = makeOperations({

‎packages/maestro/src/internal/__tests__/runtime-targets-typed.test.ts‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,62 @@ test('typed childOf reports a scoped miss when only an outside child matches', (
124124
});
125125
});
126126

127+
test('reports an out-of-range index as its own typed failure instead of a visible miss', () => {
128+
const snapshot = makeSnapshot([
129+
{ index: 0, type: 'Application', rect: { x: 0, y: 0, width: 320, height: 640 } },
130+
{
131+
index: 1,
132+
parentIndex: 0,
133+
type: 'Button',
134+
label: 'Not Now',
135+
rect: { x: 16, y: 500, width: 120, height: 44 },
136+
},
137+
{
138+
index: 2,
139+
parentIndex: 0,
140+
type: 'Button',
141+
label: 'Not Now',
142+
rect: { x: 16, y: 560, width: 120, height: 44 },
143+
},
144+
]);
145+
146+
expect(
147+
resolveMaestroTargetFromSnapshot(
148+
snapshot,
149+
{ selector: { text: 'Not Now', index: 2 } },
150+
'android',
151+
),
152+
).toMatchObject({
153+
ok: false,
154+
failureReason: 'index-out-of-range',
155+
message: 'Maestro selector matched 2 visible element(s); index 2 is out of range.',
156+
evidence: { matched: true, visible: true, candidateCount: 2 },
157+
});
158+
});
159+
160+
test('reports matched-but-invisible targets with a typed no-visible-match reason', () => {
161+
const snapshot = makeSnapshot([
162+
{ index: 0, type: 'Application', rect: { x: 0, y: 0, width: 320, height: 640 } },
163+
{
164+
index: 1,
165+
parentIndex: 0,
166+
type: 'Button',
167+
label: 'Hidden',
168+
rect: { x: 16, y: 500, width: 120, height: 44 },
169+
visibleToUser: false,
170+
},
171+
]);
172+
173+
expect(
174+
resolveMaestroTargetFromSnapshot(snapshot, { selector: { text: 'Hidden' } }, 'android'),
175+
).toMatchObject({
176+
ok: false,
177+
failureReason: 'no-visible-match',
178+
message: 'Maestro selector matched 1 element(s), but none were visible.',
179+
evidence: { matched: true, visible: false, candidateCount: 1 },
180+
});
181+
});
182+
127183
test('keeps Maestro-visible app content matchable while a React Native overlay is present', () => {
128184
const snapshot = makeSnapshot([
129185
{

‎packages/maestro/src/internal/facade-runtime-port.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import { literalFromMaestroRegex } from './selector-regex.ts';
3333
import {
3434
resolveMaestroTargetFromSnapshot,
3535
hasMaestroRecursiveRelations,
36+
type MaestroTargetFailureReason,
3637
type MaestroTargetQuery as MaestroSnapshotTargetQuery,
3738
type MaestroTargetResolution,
3839
} from './runtime-targets.ts';
@@ -91,6 +92,7 @@ export type {
9192
MaestroSelector,
9293
MaestroSinglePointerGestureInput,
9394
MaestroSnapshotTargetQuery,
95+
MaestroTargetFailureReason,
9496
MaestroTargetMatch,
9597
MaestroTargetQuery,
9698
MaestroTargetResolution,

‎packages/maestro/src/internal/runtime-port-observation.ts‎

Lines changed: 30 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -44,11 +44,13 @@ export async function observeMaestroCondition(
4444

4545
export function maestroObservationMatches(
4646
condition: MaestroObservationCondition,
47-
match: Pick<MaestroTargetMatch, 'matched' | 'visible'>,
47+
match: Pick<MaestroTargetMatch, 'matched' | 'visible' | 'failureReason'>,
4848
): boolean {
49-
return condition.kind === 'visible'
50-
? match.matched && match.visible
51-
: !match.matched || !match.visible;
49+
// A typed failure reason means no element was selected (for example an
50+
// out-of-range `index` over matches that are otherwise visible), so the
51+
// observed selector holds only when a target actually resolved.
52+
const holds = match.matched && match.visible && match.failureReason === undefined;
53+
return condition.kind === 'visible' ? holds : !holds;
5254
}
5355

5456
export async function resolveMaestroTarget(
@@ -63,9 +65,10 @@ export async function resolveMaestroTarget(
6365
);
6466
const validated = validateTargetMatch(match, request.generation);
6567
if (!validated.matched || !validated.visible || !validated.rect) {
66-
throw maestroTestFailure('Maestro target did not resolve to a visible element.', {
68+
throw maestroTestFailure(maestroTargetFailureMessage(selector, validated), {
6769
selector,
6870
candidateCount: validated.candidateCount,
71+
...(validated.failureReason ? { targetFailureReason: validated.failureReason } : {}),
6972
});
7073
}
7174
return {
@@ -93,6 +96,28 @@ export function observationForTarget(target: MaestroTargetResolution): MaestroOb
9396
};
9497
}
9598

99+
function maestroTargetFailureMessage(selector: MaestroSelector, match: MaestroTargetMatch): string {
100+
const selectorText = formatMaestroSelector(selector);
101+
switch (match.failureReason) {
102+
case 'selector-did-not-match':
103+
return `Maestro target ${selectorText} did not match any element.`;
104+
case 'no-visible-match':
105+
return `Maestro target ${selectorText} matched ${match.candidateCount} element(s), but none were visible.`;
106+
case 'index-out-of-range':
107+
return `Maestro target ${selectorText} matched ${match.candidateCount} element(s); its index is out of range.`;
108+
case 'no-usable-geometry':
109+
return `Maestro target ${selectorText} did not provide usable geometry.`;
110+
default:
111+
return 'Maestro target did not resolve to a visible element.';
112+
}
113+
}
114+
115+
function formatMaestroSelector(selector: MaestroSelector): string {
116+
const entries = Object.entries(selector).filter(([, value]) => value !== undefined);
117+
if (entries.length === 0) return 'selector';
118+
return entries.map(([key, value]) => `"${key}":${JSON.stringify(value)}`).join(', ');
119+
}
120+
96121
function isRect(value: unknown): value is { x: number; y: number; width: number; height: number } {
97122
if (!value || typeof value !== 'object') return false;
98123
const rect = value as Record<string, unknown>;

‎packages/maestro/src/internal/runtime-port-types.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import type {
1313
MaestroObservationCondition,
1414
MaestroObservationEvidence,
1515
} from './engine-types.ts';
16+
import type { MaestroTargetFailureReason } from './runtime-targets.ts';
1617

1718
export type MaestroRuntimeReadContext = {
1819
readonly appId?: string;
@@ -40,6 +41,13 @@ export type MaestroTargetMatch = {
4041
readonly ref?: string;
4142
readonly dispatchSelector?: MaestroDispatchSelector;
4243
readonly surfaceSignature?: string;
44+
/**
45+
* The typed reason for a failed resolution. An out-of-range `index` or a degenerate
46+
* candidate geometry leaves a resolution matched and visible with no target, and
47+
* `MaestroTargetMatch` is observation evidence too, where `matched`/`visible` ARE the
48+
* outcome — so the absence of a dispatch target travels as its own reason.
49+
*/
50+
readonly failureReason?: MaestroTargetFailureReason;
4351
};
4452

4553
export type MaestroDispatchSelector = {

0 commit comments

Comments
 (0)