Skip to content

Commit b75c500

Browse files
thymikeeclaude
andcommitted
fix(webdriver): resolve Android bar membership on the lazy snapshot path
The Appium source parser is evaluated eagerly through the scroll-frame path, so importing the system-chrome contract there grew the provider's eager closure. The membership pass now runs in the Android snapshot parser the interactor already imports on demand. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 1737033 commit b75c500

5 files changed

Lines changed: 113 additions & 104 deletions

File tree

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
import assert from 'node:assert/strict';
2+
import { withoutSettleChrome } from '@agent-device/capture-kit/snapshot-chrome';
3+
import { attachRefs } from '@agent-device/kernel/snapshot';
4+
import { test } from 'vitest';
5+
import { parseWebDriverAndroidSnapshotNodes } from './webdriver-android-source.ts';
6+
import { parseWebDriverSourceFacts } from './webdriver-source.ts';
7+
8+
test('Android WebDriver carries package identity and system-bar membership per same-package subtree', () => {
9+
const source = `<hierarchy>
10+
<android.widget.FrameLayout package="com.android.systemui" resource-id="com.android.systemui:id/status_bar">
11+
<android.widget.TextView package="com.android.systemui" text="Clock" />
12+
<android.widget.Button package="com.example" text="Foreign" />
13+
<android.widget.TextView text="Unknown owner" />
14+
</android.widget.FrameLayout>
15+
<android.widget.FrameLayout package="com.android.systemui" resource-id="com.android.systemui:id/navigation_bar_frame">
16+
<android.widget.Button package="com.android.systemui" text="Back" />
17+
</android.widget.FrameLayout>
18+
<android.widget.FrameLayout package="com.android.systemui" resource-id="com.android.systemui:id/split_shade_status_bar">
19+
<android.widget.TextView package="com.android.systemui" text="Shade clock" />
20+
</android.widget.FrameLayout>
21+
<android.widget.Button package="com.android.systemui" resource-id="com.android.systemui:id/status_bar_notification_row" text="Popup" />
22+
<android.widget.Button package="android" text="Dialog" />
23+
</hierarchy>`;
24+
const nodes = parseWebDriverAndroidSnapshotNodes(source);
25+
assert.deepEqual(
26+
nodes.map((node) => node.systemBar),
27+
[
28+
'status',
29+
'status',
30+
undefined,
31+
undefined,
32+
'navigation',
33+
'navigation',
34+
undefined,
35+
undefined,
36+
undefined,
37+
undefined,
38+
],
39+
);
40+
assert.equal(nodes.find((node) => node.label === 'Clock')?.bundleId, 'com.android.systemui');
41+
assert.equal(nodes.find((node) => node.label === 'Foreign')?.bundleId, 'com.example');
42+
assert.equal(nodes.find((node) => node.label === 'Dialog')?.bundleId, 'android');
43+
assert.equal(nodes.find((node) => node.label === 'Unknown owner')?.bundleId, undefined);
44+
assert.ok(
45+
parseWebDriverSourceFacts(source, 'ios').nodes.every(
46+
(node) => !('systemBar' in node) && !('bundleId' in node),
47+
),
48+
);
49+
});
50+
51+
test('Android Appium popup ids containing status_bar stay actionable through chrome classification', () => {
52+
const source = `<hierarchy><android.widget.Button package="com.android.systemui" resource-id="com.android.systemui:id/status_bar_notification_row" text="Open notification" bounds="[0,0][100,40]" /></hierarchy>`;
53+
const nodes = attachRefs(parseWebDriverAndroidSnapshotNodes(source));
54+
assert.deepEqual(
55+
withoutSettleChrome(nodes, 'com.example').map((node) => node.label),
56+
['Open notification'],
57+
);
58+
});
59+
60+
test('Android native wrappers retain unknown ownership barriers for membership and settle', () => {
61+
const nodes = parseWebDriverAndroidSnapshotNodes(
62+
`<hierarchy>
63+
<android.widget.FrameLayout package="com.android.systemui" resource-id="com.android.systemui:id/status_bar">
64+
<android.widget.FrameLayout>
65+
<android.widget.Button package="com.android.systemui" text="Unknown subtree action" bounds="[0,0][100,40]" />
66+
</android.widget.FrameLayout>
67+
</android.widget.FrameLayout>
68+
</hierarchy>`,
69+
);
70+
assert.equal(nodes[0]?.systemBar, 'status');
71+
assert.equal(nodes.find((node) => node.label === 'Unknown subtree action')?.systemBar, undefined);
72+
assert.deepEqual(
73+
withoutSettleChrome(attachRefs(nodes), 'com.example').map((node) => node.label),
74+
[undefined, 'Unknown subtree action'],
75+
);
76+
});
77+
78+
test('Appium launcher navbuttons are navigation chrome while the surrounding taskbar stays content', () => {
79+
const nodes = attachRefs(
80+
parseWebDriverAndroidSnapshotNodes(
81+
`<hierarchy>
82+
<android.widget.FrameLayout package="com.google.android.apps.nexuslauncher" resource-id="com.google.android.apps.nexuslauncher:id/taskbar_container">
83+
<android.widget.FrameLayout package="com.google.android.apps.nexuslauncher" resource-id="com.google.android.apps.nexuslauncher:id/navbuttons_view">
84+
<android.widget.Button package="com.google.android.apps.nexuslauncher" text="Back" />
85+
</android.widget.FrameLayout>
86+
<android.widget.Button package="com.google.android.apps.nexuslauncher" text="Dock app" />
87+
</android.widget.FrameLayout>
88+
</hierarchy>`,
89+
),
90+
);
91+
assert.equal(nodes.find((node) => node.label === 'Back')?.systemBar, 'navigation');
92+
assert.equal(nodes.find((node) => node.label === 'Dock app')?.systemBar, undefined);
93+
assert.deepEqual(
94+
withoutSettleChrome(nodes, 'com.example').map((node) => node.label),
95+
[undefined, 'Dock app'],
96+
);
97+
});
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
import { androidSystemBarMembership } from '@agent-device/contracts/android-system-chrome';
2+
import type { RawSnapshotNode } from '@agent-device/kernel/snapshot';
3+
import { parseWebDriverSourceFacts } from './webdriver-source.ts';
4+
5+
/** Android page-source snapshot nodes, with persistent system-bar membership per same-package subtree. */
6+
export function parseWebDriverAndroidSnapshotNodes(source: string): RawSnapshotNode[] {
7+
const { nodes } = parseWebDriverSourceFacts(source, 'android');
8+
for (const node of nodes) {
9+
const parent = node.parentIndex === undefined ? undefined : nodes[node.parentIndex];
10+
const systemBar = androidSystemBarMembership(node.bundleId, node.identifier, parent);
11+
if (systemBar) node.systemBar = systemBar;
12+
}
13+
return nodes;
14+
}

‎packages/provider-webdriver/src/webdriver-interactor.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -328,11 +328,11 @@ class WebDriverInteractor implements Interactor {
328328
const { acquireWebDriverIosSnapshot } = await import('./webdriver-ios-snapshot.ts');
329329
return acquireWebDriverIosSnapshot(source, this.targetId);
330330
}
331-
const { parseWebDriverSourceFacts } = await import('./webdriver-source.ts');
331+
const { parseWebDriverAndroidSnapshotNodes } = await import('./webdriver-android-source.ts');
332332
return {
333333
backend: 'android' as const,
334334
producer: 'appium-source' as const,
335-
nodes: parseWebDriverSourceFacts(source, 'android').nodes,
335+
nodes: parseWebDriverAndroidSnapshotNodes(source),
336336
};
337337
}
338338

‎packages/provider-webdriver/src/webdriver-source.test.ts‎

Lines changed: 0 additions & 94 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ import {
55
scrollFrameFromAndroidWebDriverSource,
66
scrollFrameFromIosWebDriverSource,
77
} from './webdriver-scroll-frame.ts';
8-
import { withoutSettleChrome } from '@agent-device/capture-kit/snapshot-chrome';
98
import { parseWebDriverSourceFacts } from './webdriver-source.ts';
109

1110
test('Android WebDriver facts keep platform defaults and derive hittability from geometry', () => {
@@ -329,96 +328,3 @@ test('iOS WebDriver facts keep the tag as the type whatever a class attribute sa
329328

330329
assert.equal(node!.type, 'XCUIElementTypeButton');
331330
});
332-
333-
test('Android WebDriver carries package identity and system-bar membership per same-package subtree', () => {
334-
const source = `<hierarchy>
335-
<android.widget.FrameLayout package="com.android.systemui" resource-id="com.android.systemui:id/status_bar">
336-
<android.widget.TextView package="com.android.systemui" text="Clock" />
337-
<android.widget.Button package="com.example" text="Foreign" />
338-
<android.widget.TextView text="Unknown owner" />
339-
</android.widget.FrameLayout>
340-
<android.widget.FrameLayout package="com.android.systemui" resource-id="com.android.systemui:id/navigation_bar_frame">
341-
<android.widget.Button package="com.android.systemui" text="Back" />
342-
</android.widget.FrameLayout>
343-
<android.widget.FrameLayout package="com.android.systemui" resource-id="com.android.systemui:id/split_shade_status_bar">
344-
<android.widget.TextView package="com.android.systemui" text="Shade clock" />
345-
</android.widget.FrameLayout>
346-
<android.widget.Button package="com.android.systemui" resource-id="com.android.systemui:id/status_bar_notification_row" text="Popup" />
347-
<android.widget.Button package="android" text="Dialog" />
348-
</hierarchy>`;
349-
const nodes = parseWebDriverSourceFacts(source, 'android').nodes;
350-
assert.deepEqual(
351-
nodes.map((node) => node.systemBar),
352-
[
353-
'status',
354-
'status',
355-
undefined,
356-
undefined,
357-
'navigation',
358-
'navigation',
359-
undefined,
360-
undefined,
361-
undefined,
362-
undefined,
363-
],
364-
);
365-
assert.equal(nodes.find((node) => node.label === 'Clock')?.bundleId, 'com.android.systemui');
366-
assert.equal(nodes.find((node) => node.label === 'Foreign')?.bundleId, 'com.example');
367-
assert.equal(nodes.find((node) => node.label === 'Dialog')?.bundleId, 'android');
368-
assert.equal(nodes.find((node) => node.label === 'Unknown owner')?.bundleId, undefined);
369-
assert.ok(
370-
parseWebDriverSourceFacts(source, 'ios').nodes.every(
371-
(node) => !('systemBar' in node) && !('bundleId' in node),
372-
),
373-
);
374-
});
375-
376-
test('Android Appium popup ids containing status_bar stay actionable through chrome classification', () => {
377-
const source = `<hierarchy><android.widget.Button package="com.android.systemui" resource-id="com.android.systemui:id/status_bar_notification_row" text="Open notification" bounds="[0,0][100,40]" /></hierarchy>`;
378-
const nodes = attachRefs(parseWebDriverSourceFacts(source, 'android').nodes);
379-
assert.deepEqual(
380-
withoutSettleChrome(nodes, 'com.example').map((node) => node.label),
381-
['Open notification'],
382-
);
383-
});
384-
385-
test('Android native wrappers retain unknown ownership barriers for membership and settle', () => {
386-
const nodes = parseWebDriverSourceFacts(
387-
`<hierarchy>
388-
<android.widget.FrameLayout package="com.android.systemui" resource-id="com.android.systemui:id/status_bar">
389-
<android.widget.FrameLayout>
390-
<android.widget.Button package="com.android.systemui" text="Unknown subtree action" bounds="[0,0][100,40]" />
391-
</android.widget.FrameLayout>
392-
</android.widget.FrameLayout>
393-
</hierarchy>`,
394-
'android',
395-
).nodes;
396-
assert.equal(nodes[0]?.systemBar, 'status');
397-
assert.equal(nodes.find((node) => node.label === 'Unknown subtree action')?.systemBar, undefined);
398-
assert.deepEqual(
399-
withoutSettleChrome(attachRefs(nodes), 'com.example').map((node) => node.label),
400-
[undefined, 'Unknown subtree action'],
401-
);
402-
});
403-
404-
test('Appium launcher navbuttons are navigation chrome while the surrounding taskbar stays content', () => {
405-
const nodes = attachRefs(
406-
parseWebDriverSourceFacts(
407-
`<hierarchy>
408-
<android.widget.FrameLayout package="com.google.android.apps.nexuslauncher" resource-id="com.google.android.apps.nexuslauncher:id/taskbar_container">
409-
<android.widget.FrameLayout package="com.google.android.apps.nexuslauncher" resource-id="com.google.android.apps.nexuslauncher:id/navbuttons_view">
410-
<android.widget.Button package="com.google.android.apps.nexuslauncher" text="Back" />
411-
</android.widget.FrameLayout>
412-
<android.widget.Button package="com.google.android.apps.nexuslauncher" text="Dock app" />
413-
</android.widget.FrameLayout>
414-
</hierarchy>`,
415-
'android',
416-
).nodes,
417-
);
418-
assert.equal(nodes.find((node) => node.label === 'Back')?.systemBar, 'navigation');
419-
assert.equal(nodes.find((node) => node.label === 'Dock app')?.systemBar, undefined);
420-
assert.deepEqual(
421-
withoutSettleChrome(nodes, 'com.example').map((node) => node.label),
422-
[undefined, 'Dock app'],
423-
);
424-
});

‎packages/provider-webdriver/src/webdriver-source.ts‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
import { androidSystemBarMembership } from '@agent-device/contracts/android-system-chrome';
21
import type { RawSnapshotNode } from '@agent-device/kernel/snapshot';
32
import { parseBounds } from '@agent-device/kernel/bounds';
43
import { AppError } from '@agent-device/kernel/errors';
@@ -97,13 +96,6 @@ export function parseWebDriverSourceFacts(
9796
for (const root of roots) {
9897
appendSourceNodes(nodes, root, undefined, 0, sourceRoots, platform);
9998
}
100-
if (platform === 'android') {
101-
for (const node of nodes) {
102-
const parent = node.parentIndex === undefined ? undefined : nodes[node.parentIndex];
103-
const systemBar = androidSystemBarMembership(node.bundleId, node.identifier, parent);
104-
if (systemBar) node.systemBar = systemBar;
105-
}
106-
}
10799
return { nodes, roots: sourceRoots };
108100
}
109101

0 commit comments

Comments
 (0)