Skip to content

Commit 4a1862b

Browse files
committed
fix(limrun): harden iOS alert detection against ambiguous sheets
Review follow-up on the alert legs: a sheet must offer at least two pressable choices, launch-icon buttons never qualify, a capture whose host sheets disagree is refused instead of shadowing one with the other, and an action fires only when exactly one button states its choice. The legs' poll/confirm budgets now spend through an injected sleep seam so their tests drive the clock, a malformed timeout fails as a bounded default instead of an unbound loop, and the grant vocabulary covers the iOS variants (Allow Once / While Using App / All the Time) and Close App. User docs and CHANGELOG updated.
1 parent 7fc0b53 commit 4a1862b

7 files changed

Lines changed: 301 additions & 106 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,6 @@
22

33
## Unreleased
44

5+
- Limrun iOS sessions now serve `alert get/wait/accept/dismiss` instead of refusing with `unsupported-provider-mode`. System alerts are read from the same element-tree capture snapshots ride — a blank-label application root hosting a text-and-buttons sheet — and accepted or dismissed by tapping the button whose label states the asked-for choice; an alert whose choices leave the action undecided is reported, never guessed, and app-owned dialogs stay app UI.
56
- Limrun iOS snapshots no longer fail when the instance reports a numeric or boolean accessibility value (the system photo picker's tabs report `1`/`0`): the provider seam normalizes variant scalars to text, the way the Apple tree does. Limrun nodes now also publish `selected: true` for nodes carrying the `Selected` trait, so snapshot output and `selected:` selectors can tell a picked photo from an unpicked one.
67
- Carry provider keyboard facts through snapshots. Appium captures, including wait polls and fill pre-captures, add one sequential keyboard visibility probe with a 500 ms timeout. Probe results preserve tap-time docked-tree checks; unavailable probes leave visibility unknown.

‎packages/provider-limrun/src/alert-detection.test.ts‎

Lines changed: 60 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -64,12 +64,7 @@ describe('findLimrunAlertCandidate', () => {
6464
expect(candidate?.alert.source).toBe('permission');
6565
expect(candidate?.buttons.map((button) => button.role)).toEqual(['dismiss', 'accept']);
6666
// The tap point is the live button frame's center, rounded the way `centerOfRect` rounds.
67-
expect(candidate?.buttons[1]).toEqual({
68-
label: 'Allow',
69-
x: 201,
70-
y: 592,
71-
role: 'accept',
72-
});
67+
expect(candidate?.buttons[1]).toEqual({ label: 'Allow', x: 201, y: 592, role: 'accept' });
7368
});
7469

7570
it('finds the alert when the app screen is the first root of the capture', () => {
@@ -89,10 +84,38 @@ describe('findLimrunAlertCandidate', () => {
8984
expect(findLimrunAlertCandidate(limrunHomeScreenTree())).toBeNull();
9085
});
9186

87+
it('refuses a widget-bearing home screen: LaunchIcon buttons never qualify', () => {
88+
const home = limrunHomeScreenTree();
89+
const withWidgets = {
90+
...home,
91+
children: [
92+
{
93+
type: 'StaticText',
94+
role: 'AXStaticText',
95+
AXLabel: 'Mostly Sunny',
96+
pid: home.pid,
97+
frame: { x: 24, y: 300, width: 160, height: 20 },
98+
},
99+
...(home.children ?? []),
100+
],
101+
};
102+
expect(findLimrunAlertCandidate(withWidgets)).toBeNull();
103+
});
104+
92105
it('refuses an alert-shaped sheet with no actionable button', () => {
93106
expect(findLimrunAlertCandidate(limrunAlertTreeWithoutButtons())).toBeNull();
94107
});
95108

109+
it('refuses a one-choice sheet: no action can tell its single option from the asked-for one', () => {
110+
expect(findLimrunAlertCandidate(alertTreeWithButtons('Allow'))).toBeNull();
111+
expect(findLimrunAlertCandidate(alertTreeWithButtons('OK'))).toBeNull();
112+
});
113+
114+
it('refuses a capture whose host sheets disagree, rather than shadowing one with the other', () => {
115+
const second = { ...limrunAttAlertTree(), pid: 99 };
116+
expect(findLimrunAlertCandidate([limrunAttAlertTree(), second])).toBeNull();
117+
});
118+
96119
it('reports a permission alert from its button vocabulary and a plain one as a system dialog', () => {
97120
expect(findLimrunAlertCandidate(limrunAttAlertTree())?.alert.source).toBe('permission');
98121
expect(findLimrunAlertCandidate(alertTreeWithButtons('No', 'Yes'))?.alert.source).toBe(
@@ -118,18 +141,21 @@ describe('chooseLimrunAlertButton', () => {
118141
expect(chooseLimrunAlertButton(buttons, 'dismiss')).toBeNull();
119142
});
120143

121-
it('refuses an action that would press the wrong choice of a one-button alert', () => {
122-
const allowOnly = findLimrunAlertCandidate(alertTreeWithButtons('Allow'))!.buttons;
123-
const denyOnly = findLimrunAlertCandidate(
124-
alertTreeWithButtons('Ask App Not to Track'),
125-
)!.buttons;
144+
it('never picks a choice of the wrong role, whatever else the alert offers', () => {
145+
const allowOnly = findLimrunAlertCandidate(alertTreeWithButtons('Deny', 'Allow'))!.buttons;
126146
expect(chooseLimrunAlertButton(allowOnly, 'accept')?.label).toBe('Allow');
127-
expect(chooseLimrunAlertButton(allowOnly, 'dismiss')).toBeNull();
128-
expect(chooseLimrunAlertButton(denyOnly, 'dismiss')?.label).toBe('Ask App Not to Track');
129-
expect(chooseLimrunAlertButton(denyOnly, 'accept')).toBeNull();
147+
expect(chooseLimrunAlertButton(allowOnly, 'dismiss')?.label).toBe('Deny');
148+
const noAccept = findLimrunAlertCandidate(
149+
alertTreeWithButtons('Cancel', 'Maybe Later'),
150+
)!.buttons;
151+
expect(chooseLimrunAlertButton(noAccept, 'accept')).toBeNull();
152+
const noDismiss = findLimrunAlertCandidate(
153+
alertTreeWithButtons('Open Settings', 'Continue'),
154+
)!.buttons;
155+
expect(chooseLimrunAlertButton(noDismiss, 'dismiss')).toBeNull();
130156
});
131157

132-
it('classifies the classic iOS pair, the curly-apostrophe denial, and an open-settings prompt', () => {
158+
it('classifies the classic iOS pair, the typographic apostrophe, and iOS variant grants', () => {
133159
const classic = findLimrunAlertCandidate(alertTreeWithButtons("Don't Allow", 'OK'))!.buttons;
134160
expect(chooseLimrunAlertButton(classic, 'accept')?.label).toBe('OK');
135161
expect(chooseLimrunAlertButton(classic, 'dismiss')?.label).toBe("Don't Allow");
@@ -140,9 +166,15 @@ describe('chooseLimrunAlertButton', () => {
140166
)!.buttons;
141167
expect(chooseLimrunAlertButton(settings, 'accept')?.label).toBe('Open Settings');
142168
expect(chooseLimrunAlertButton(settings, 'dismiss')?.label).toBe('Not Now');
169+
for (const label of ['Allow Once', 'Allow All the Time', 'Allow While Using App']) {
170+
const granted = findLimrunAlertCandidate(
171+
alertTreeWithButtons('Ask App Not to Track', label),
172+
)!.buttons;
173+
expect(chooseLimrunAlertButton(granted, 'accept')?.label).toBe(label);
174+
}
143175
});
144176

145-
it('refuses the candidate when no button is enabled with a frame to tap', () => {
177+
it('refuses the candidate when no button is pressable', () => {
146178
const root = limrunAttAlertTree();
147179
const crippled = {
148180
...root,
@@ -163,12 +195,22 @@ describe('chooseLimrunAlertButton', () => {
163195
AXLabel: 'Ask App Not to Track',
164196
pid: root.pid,
165197
enabled: true,
198+
hittable: false,
199+
frame: { x: 57, y: 567.67, width: 288, height: 48 },
200+
traits: ['Button'],
201+
},
202+
{
203+
type: 'Button',
204+
role: 'AXButton',
205+
AXLabel: 'Later',
206+
pid: root.pid,
207+
enabled: true,
166208
traits: ['Button'],
167209
},
168210
],
169211
};
170-
// The only button that could deny has no frame to tap, and the framed one is disabled:
171-
// nothing is actionable, so the whole candidate is refused rather than half-served.
212+
// Disabled, unhittable, and frameless buttons are all unpressable; with nothing pressable
213+
// left, the whole candidate is refused rather than half-served.
172214
expect(findLimrunAlertCandidate(crippled)).toBeNull();
173215
});
174216
});

‎packages/provider-limrun/src/alert-detection.ts‎

Lines changed: 42 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,14 @@ import { readIosNodeChildren, readIosNodeRect, type IosTreeNode } from './snapsh
66
/**
77
* System-alert detection over a Limrun iOS element tree (issue #3349). A Limrun iOS direct
88
* session has no XCUITest runner, but its element tree does carry the system alert: on a live
9-
* instance the ATT prompt arrives as its own `AXApplication` root whose `AXLabel` is blank and
10-
* whose content nodes carry the hosting process's `pid` — not the app's. The app's own
11-
* `UIAlertController`, by contrast, rides inside the app's root, labeled with the app name and
12-
* carrying the app's pid. The rules below encode exactly that observed split; they are
13-
* conservative in both directions and every shape here is pinned to the live captures.
9+
* instance the ATT prompt arrives as its own `AXApplication` root whose `AXLabel` is blank, with
10+
* its text and buttons as direct children all sharing the hosting process's `pid` — never the
11+
* app's. The app's own `UIAlertController`, by contrast, rides inside the app's root, labeled
12+
* with the app name and carrying the app's pid. The rules below encode that observed split and
13+
* refuse everything it does not vouch for: a named root is app UI, launch-icon buttons are a
14+
* home screen, and a button without a frame or a live `pid` match across visible roots is not
15+
* pressable. What one live capture cannot exclude — an app that left its own root's label blank
16+
* while showing a dialog — is why the acting legs press only label-unambiguous buttons below.
1417
*/
1518

1619
type LimrunAlertButtonRole = 'accept' | 'dismiss' | 'neutral';
@@ -37,12 +40,15 @@ export type LimrunAlertCandidate = Readonly<{
3740
}>;
3841

3942
const ACCEPT_LABEL_PATTERN =
40-
/^(?:ok|allow|allow all|always allow|yes|continue|save|confirm|turn on|open settings)$/i;
43+
/^(?:ok|allow|allow all|allow once|allow all the time|allow while using (?:the )?app|allow notifications|always allow|yes|continue|save|confirm|turn on|open settings|while using the app|only this time)$/i;
44+
// `don.?t allow` covers the straight, missing, and typographic apostrophe in one alternate.
4145
const DISMISS_LABEL_PATTERN =
42-
/^(?:cancel|deny|don.?t allow|don’t allow|don.t allow|not now|no|dismiss|close|later|skip|ask app not to track)$/i;
46+
/^(?:cancel|deny|don.?t allow|not now|no|dismiss|close|close app|later|skip|ask app not to track)$/i;
4347
/** The subset that states a grant/deny choice; a plain Yes/No dialog is not a permission prompt. */
4448
const PERMISSION_LABEL_PATTERN =
45-
/^(?:allow.*|always allow|deny|don.?t allow|don’t allow|don.t allow|ask app not to track|turn on|open settings)$/i;
49+
/^(?:allow.*|deny|don.?t allow|ask app not to track|turn on|open settings)$/i;
50+
/** SpringBoard marks its home-screen icons with this trait; an alert never carries it. */
51+
const LAUNCH_ICON_TRAIT = 'LaunchIcon';
4652

4753
function nodeLabel(node: IosTreeNode): string {
4854
const label = node.AXLabel ?? node.label;
@@ -62,25 +68,36 @@ export function isLimrunAlertHostRoot(node: IosTreeNode): boolean {
6268

6369
/**
6470
* The actionable alert on this capture, or null. A host root qualifies only when its content is
65-
* dialog-shaped: a button that can actually be pressed (enabled, with a complete rect) and at
66-
* least one piece of text or another button beside it. That refusals both an app dialog (wrong
67-
* root) and SpringBoard's home screen (blank root, launch icons instead of a dialog).
71+
* dialog-shaped: at least two pressable choices (enabled, hittable, with a complete rect), at
72+
* least one piece of text, and no SpringBoard launch icons. One live capture cannot exclude an
73+
* app that leaves its own root's label blank while showing a dialog — the provider reports no
74+
* capture-level fact that separates it from the alert host — which is why the acting legs press
75+
* only buttons whose label states the asked-for choice, and an alert-shaped sheet offering a
76+
* single choice is refused entirely rather than acted on by position.
77+
*
78+
* The button vocabulary is English, matching the untranslated simulator locale these instances
79+
* run; a localized alert classifies as undecided and the acting legs report the ambiguity rather
80+
* than guessing.
6881
*/
6982
export function findLimrunAlertCandidate(
7083
tree: IosTreeNode | IosTreeNode[],
7184
): LimrunAlertCandidate | null {
85+
const candidates: LimrunAlertCandidate[] = [];
7286
for (const root of Array.isArray(tree) ? tree : [tree]) {
7387
if (!isLimrunAlertHostRoot(root)) continue;
7488
const candidate = readAlertCandidateFromHostRoot(root);
75-
if (candidate) return candidate;
89+
if (candidate) candidates.push(candidate);
7690
}
77-
return null;
91+
// More than one alert-shaped host sheet is a shape this reader does not understand; pressing
92+
// into it would guess, and the observation would silently shadow one with the other.
93+
return candidates.length === 1 ? candidates[0]! : null;
7894
}
7995

8096
function readAlertCandidateFromHostRoot(root: IosTreeNode): LimrunAlertCandidate | null {
8197
const descendants = collectDescendants(root);
8298
const buttons = findAlertButtons(descendants);
83-
if (buttons.length === 0) return null;
99+
// A lone choice says nothing about which action the caller asked for; refuse the sheet.
100+
if (buttons.length < 2) return null;
84101
const texts = descendants
85102
.filter((node) => isStaticTextNode(node) && !isButtonNode(node))
86103
.map(nodeLabel)
@@ -120,14 +137,17 @@ function isStaticTextNode(node: IosTreeNode): boolean {
120137
}
121138

122139
/**
123-
* Pressable buttons only: a disabled button is not a choice the alert offers right now, and a
124-
* button without a complete rect has no point the touch seam could aim at.
140+
* Pressable alert choices only: a disabled button is not a choice the alert offers right now, a
141+
* button without a complete rect has no point the touch seam could aim at, a button the provider
142+
* reports as unhittable cannot be pressed, and a `LaunchIcon` is a home-screen app icon rather
143+
* than a dialog button.
125144
*/
126145
function findAlertButtons(nodes: IosTreeNode[]): LimrunAlertButton[] {
127146
const seen = new Set<string>();
128147
const buttons: LimrunAlertButton[] = [];
129148
for (const node of nodes) {
130-
if (!isButtonNode(node) || node.enabled === false) continue;
149+
if (!isButtonNode(node) || node.enabled === false || node.hittable === false) continue;
150+
if (node.traits?.includes(LAUNCH_ICON_TRAIT)) continue;
131151
const label = nodeLabel(node);
132152
if (!label || seen.has(label.toLowerCase())) continue;
133153
const rect = readIosNodeRect(node);
@@ -160,14 +180,15 @@ function classifyAlertSource(buttons: readonly LimrunAlertButton[]): LimrunAlert
160180
/**
161181
* The button one action may press, or null when pressing any visible choice would guess.
162182
* Deliberately stricter than Android's: with no `android:id/button1`-style identity to fall back
163-
* on, only a button whose own label states the asked-for choice may be pressed — an accept must
164-
* never press a denial, and an alert whose choices are all neutral is refused for both actions
165-
* rather than resolved by position.
183+
* on, only a button whose own label states the asked-for choice may be pressed, and even then
184+
* only when exactly one choice does — an accept must never press a denial, and an alert whose
185+
* choices leave the action undecided is refused rather than resolved by tree position.
166186
*/
167187
export function chooseLimrunAlertButton(
168188
buttons: readonly LimrunAlertButton[],
169189
action: 'accept' | 'dismiss',
170190
): LimrunAlertButton | null {
171191
const wanted = action === 'accept' ? 'accept' : 'dismiss';
172-
return buttons.find((button) => button.role === wanted) ?? null;
192+
const matching = buttons.filter((button) => button.role === wanted);
193+
return matching.length === 1 ? matching[0]! : null;
173194
}
Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
import { afterEach, describe, expect, it, vi } from 'vitest';
2+
import {
3+
limrunAlertAmbiguousButtonsTree,
4+
limrunAppScreenTree,
5+
limrunAttAlertTree,
6+
} from './alert-detection.fixtures.ts';
7+
import { handleLimrunIosAlert, type LimrunIosAlertClient } from './alert.ts';
8+
import type { IosTreeNode } from './snapshot.ts';
9+
10+
// The clock is the assertion, not the wait (`docs/agents/testing.md`): only `Date` is faked, and
11+
// the legs' sleep seam advances it, so budget loops terminate without real time.
12+
const advanceClock = async (ms: number) => {
13+
vi.advanceTimersByTime(ms);
14+
};
15+
16+
function fakeClock() {
17+
vi.useFakeTimers({ toFake: ['Date'] });
18+
}
19+
20+
function clientWithTrees(trees: Array<IosTreeNode | IosTreeNode[]>): LimrunIosAlertClient {
21+
const elementTree = vi.fn(async () => {
22+
const next = trees.length > 1 ? trees.shift() : trees[0];
23+
return [next] as unknown as ReturnType<LimrunIosAlertClient['elementTree']>;
24+
});
25+
return { elementTree, tap: vi.fn(async () => {}) };
26+
}
27+
28+
afterEach(() => {
29+
vi.useRealTimers();
30+
});
31+
32+
describe('wait budget', () => {
33+
it('spends exactly the caller window through the sleep seam and reports the timeout', async () => {
34+
fakeClock();
35+
const client = clientWithTrees([limrunAppScreenTree()]);
36+
const sleep = vi.fn(advanceClock);
37+
38+
const settled = handleLimrunIosAlert(client, 'wait', { timeoutMs: 5_000, sleep }).then(
39+
() => undefined,
40+
(error: unknown) => error,
41+
);
42+
const error = await settled;
43+
44+
expect(error).toBeInstanceOf(Error);
45+
expect((error as Error).message).toBe('alert wait timed out');
46+
// One poll per interval over the whole window: never fewer (gave up early) nor real seconds.
47+
expect(sleep).toHaveBeenCalledTimes(Math.ceil(5_000 / 300));
48+
});
49+
50+
it('treats a malformed window as the bounded default rather than an unbound loop', async () => {
51+
fakeClock();
52+
const client = clientWithTrees([limrunAppScreenTree()]);
53+
const sleep = vi.fn(advanceClock);
54+
55+
await expect(
56+
handleLimrunIosAlert(client, 'wait', { timeoutMs: Number.NaN, sleep }),
57+
).rejects.toThrow('alert wait timed out');
58+
// The fallback is the contract default, spent on the driven clock — this returned, so the
59+
// loop had a bound, which a NaN comparison would not have given it.
60+
expect(sleep).toHaveBeenCalledTimes(Math.ceil(10_000 / 300));
61+
});
62+
63+
it('surfaces a failed capture as itself, never as a timeout', async () => {
64+
const elementTree = vi
65+
.fn()
66+
.mockRejectedValue(
67+
new Error('websocket closed'),
68+
) as unknown as LimrunIosAlertClient['elementTree'];
69+
const client = { elementTree, tap: vi.fn(async () => {}) } satisfies LimrunIosAlertClient;
70+
71+
await expect(handleLimrunIosAlert(client, 'wait', { sleep: async () => {} })).rejects.toThrow(
72+
'websocket closed',
73+
);
74+
});
75+
});
76+
77+
describe('acting budgets', () => {
78+
it('spends the confirm budget through the sleep seam and reports a stuck alert', async () => {
79+
fakeClock();
80+
const client = clientWithTrees([limrunAttAlertTree(), limrunAttAlertTree()]);
81+
const sleep = vi.fn(advanceClock);
82+
83+
await expect(handleLimrunIosAlert(client, 'dismiss', { sleep })).rejects.toThrow(
84+
'alert dismiss did not dismiss the visible alert',
85+
);
86+
expect(client.tap).toHaveBeenCalledTimes(1);
87+
expect(sleep).toHaveBeenCalled();
88+
});
89+
90+
it('refuses an ambiguous alert without tapping', async () => {
91+
const client = clientWithTrees([limrunAlertAmbiguousButtonsTree()]);
92+
93+
await expect(handleLimrunIosAlert(client, 'accept', { sleep: async () => {} })).rejects.toThrow(
94+
'no button that clearly matches the action',
95+
);
96+
expect(client.tap).not.toHaveBeenCalled();
97+
});
98+
});
99+
100+
describe('button vocabulary', () => {
101+
function alertWith(left: string, right: string): IosTreeNode {
102+
const root = limrunAttAlertTree();
103+
return {
104+
...root,
105+
children: [
106+
...(root.children ?? []).slice(0, 2),
107+
...[left, right].map((label, index) => ({
108+
type: 'Button',
109+
role: 'AXButton',
110+
AXLabel: label,
111+
pid: root.pid,
112+
enabled: true,
113+
frame: { x: 57, y: 511.67 + index * 56, width: 288, height: 48 },
114+
traits: ['Button', 'Scrollable'],
115+
})),
116+
],
117+
};
118+
}
119+
120+
it('accepts the iOS variant grants, not just the bare Allow', async () => {
121+
for (const label of ['Allow While Using App', 'Allow Once', 'Allow All the Time']) {
122+
const client = clientWithTrees([
123+
alertWith('Ask App Not to Track', label),
124+
limrunAppScreenTree(),
125+
]);
126+
const result = await handleLimrunIosAlert(client, 'accept', { sleep: async () => {} });
127+
expect(result.kind === 'alertHandled' && result.button).toBe(label);
128+
}
129+
});
130+
131+
it('dismisses Close App, the ANRB denial', async () => {
132+
const client = clientWithTrees([alertWith('Close App', 'Wait'), limrunAppScreenTree()]);
133+
const result = await handleLimrunIosAlert(client, 'dismiss', { sleep: async () => {} });
134+
expect(result.kind === 'alertHandled' && result.button).toBe('Close App');
135+
});
136+
});

0 commit comments

Comments
 (0)