Skip to content

Commit 2bab15a

Browse files
committed
fix: preserve focused Android TV nodes
1 parent 8fb663a commit 2bab15a

6 files changed

Lines changed: 77 additions & 31 deletions

File tree

‎src/commands/interaction/runtime/selector-capture-policy.test.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,19 @@ import { test } from 'vitest';
33
import { parseSelectorChain } from '../../../utils/selectors-parse.ts';
44
import { deriveSelectorCapturePolicy } from './selector-capture-policy.ts';
55

6-
test('selector capture policy requests interactive snapshots for focused predicates', () => {
6+
test('selector capture policy reads full snapshots for focused predicates', () => {
77
assert.deepEqual(deriveSelectorCapturePolicy({ predicate: 'focused' }), {
88
includeRects: false,
9-
interactiveOnly: true,
9+
interactiveOnly: false,
1010
});
1111
});
1212

13-
test('selector capture policy requests interactive snapshots for focus selectors', () => {
13+
test('selector capture policy reads full snapshots for focus selectors', () => {
1414
assert.deepEqual(
1515
deriveSelectorCapturePolicy({ selectorChain: parseSelectorChain('focused=true') }),
1616
{
1717
includeRects: false,
18-
interactiveOnly: true,
18+
interactiveOnly: false,
1919
},
2020
);
2121
});

‎src/commands/interaction/runtime/selector-capture-policy.ts‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
import type { SelectorChain } from '../../../utils/selectors-parse.ts';
21
import type { IsPredicate } from '../../../utils/selector-is-predicates.ts';
2+
import type { SelectorChain } from '../../../utils/selectors-parse.ts';
33

44
export type SelectorCapturePolicyInput = {
55
predicate?: IsPredicate;
@@ -17,16 +17,10 @@ export function deriveSelectorCapturePolicy(
1717
const includeRects = predicateNeedsRects(input.predicate);
1818
return {
1919
includeRects,
20-
interactiveOnly:
21-
input.predicate === 'focused' ||
22-
(input.selectorChain ? selectorChainReadsFocus(input.selectorChain) : false),
20+
interactiveOnly: false,
2321
};
2422
}
2523

26-
function selectorChainReadsFocus(chain: SelectorChain): boolean {
27-
return chain.selectors.some((selector) => selector.terms.some((term) => term.key === 'focused'));
28-
}
29-
3024
function predicateNeedsRects(predicate: IsPredicate | undefined): boolean {
3125
return predicate === 'visible' || predicate === 'hidden';
3226
}

‎src/commands/interaction/runtime/selector-read.test.ts‎

Lines changed: 46 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,7 @@ test('runtime visibility predicates request snapshot rects', async () => {
168168
assert.equal(captureOptions?.includeRects, true);
169169
});
170170

171-
test('runtime focused predicate requests an interactive snapshot', async () => {
171+
test('runtime focused predicate requests a full snapshot', async () => {
172172
const snapshot = makeSnapshotState([
173173
{
174174
index: 0,
@@ -199,10 +199,49 @@ test('runtime focused predicate requests an interactive snapshot', async () => {
199199
});
200200

201201
assert.equal(result.pass, true);
202-
assert.equal(captureOptions?.interactiveOnly, true);
202+
assert.equal(captureOptions?.interactiveOnly, false);
203203
});
204204

205-
test('runtime focused selector waits against an interactive snapshot', async () => {
205+
test('runtime focused predicate reads focused Android TV nodes from the full tree', async () => {
206+
const fullSnapshot = makeSnapshotState(
207+
[
208+
{
209+
index: 0,
210+
depth: 0,
211+
type: 'TextView',
212+
label: 'Featured',
213+
focused: true,
214+
hittable: false,
215+
},
216+
],
217+
{ backend: 'android' },
218+
);
219+
const interactiveSnapshot = makeSnapshotState([], { backend: 'android' });
220+
let captureOptions: BackendSnapshotOptions | undefined;
221+
const device = createAgentDevice({
222+
backend: {
223+
platform: 'android',
224+
captureSnapshot: async (_context, options) => {
225+
captureOptions = options;
226+
return { snapshot: options?.interactiveOnly ? interactiveSnapshot : fullSnapshot };
227+
},
228+
} satisfies AgentDeviceBackend,
229+
artifacts: createLocalArtifactAdapter(),
230+
sessions: createMemorySessionStore([{ name: 'default', snapshot: interactiveSnapshot }]),
231+
policy: localCommandPolicy(),
232+
});
233+
234+
const result = await device.selectors.is({
235+
session: 'default',
236+
predicate: 'focused',
237+
selector: 'label=Featured',
238+
});
239+
240+
assert.equal(result.pass, true);
241+
assert.equal(captureOptions?.interactiveOnly, false);
242+
});
243+
244+
test('runtime focused selector waits against a full snapshot', async () => {
206245
const snapshot = makeSnapshotState([
207246
{
208247
index: 0,
@@ -231,8 +270,10 @@ test('runtime focused selector waits against an interactive snapshot', async ()
231270
target: { kind: 'selector', selector: 'focused=true', timeoutMs: 1000 },
232271
});
233272

234-
assert.deepEqual(result, { kind: 'selector', selector: 'focused=true', waitedMs: 0 });
235-
assert.equal(captureOptions?.interactiveOnly, true);
273+
assert.equal(result.kind, 'selector');
274+
assert.equal(result.selector, 'focused=true');
275+
assert.equal(result.waitedMs >= 0, true);
276+
assert.equal(captureOptions?.interactiveOnly, false);
236277
});
237278

238279
test('runtime is validates selector predicates', async () => {

‎src/core/tv-remote.ts‎

Lines changed: 7 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -6,60 +6,53 @@ type TvRemoteButtonDefinition = {
66
aliases: readonly string[];
77
androidKeyevent: string;
88
appleRemoteButton: string;
9-
durationMode: {
10-
android: TvRemoteDurationMode;
11-
apple: TvRemoteDurationMode;
12-
};
139
};
1410

11+
const TV_REMOTE_DURATION_MODE = {
12+
android: 'longpress',
13+
apple: 'exact',
14+
} as const satisfies Record<string, TvRemoteDurationMode>;
15+
1516
const TV_REMOTE_BUTTON_DEFINITIONS = {
1617
up: {
1718
aliases: [],
1819
androidKeyevent: 'KEYCODE_DPAD_UP',
1920
appleRemoteButton: 'up',
20-
durationMode: { android: 'longpress', apple: 'exact' },
2121
},
2222
down: {
2323
aliases: [],
2424
androidKeyevent: 'KEYCODE_DPAD_DOWN',
2525
appleRemoteButton: 'down',
26-
durationMode: { android: 'longpress', apple: 'exact' },
2726
},
2827
left: {
2928
aliases: [],
3029
androidKeyevent: 'KEYCODE_DPAD_LEFT',
3130
appleRemoteButton: 'left',
32-
durationMode: { android: 'longpress', apple: 'exact' },
3331
},
3432
right: {
3533
aliases: [],
3634
androidKeyevent: 'KEYCODE_DPAD_RIGHT',
3735
appleRemoteButton: 'right',
38-
durationMode: { android: 'longpress', apple: 'exact' },
3936
},
4037
select: {
4138
aliases: ['ok', 'center', 'enter'],
4239
androidKeyevent: 'KEYCODE_DPAD_CENTER',
4340
appleRemoteButton: 'select',
44-
durationMode: { android: 'longpress', apple: 'exact' },
4541
},
4642
menu: {
4743
aliases: [],
4844
androidKeyevent: 'KEYCODE_MENU',
4945
appleRemoteButton: 'menu',
50-
durationMode: { android: 'longpress', apple: 'exact' },
5146
},
5247
home: {
5348
aliases: [],
5449
androidKeyevent: 'KEYCODE_HOME',
5550
appleRemoteButton: 'home',
56-
durationMode: { android: 'longpress', apple: 'exact' },
5751
},
5852
back: {
5953
aliases: [],
6054
androidKeyevent: 'KEYCODE_BACK',
6155
appleRemoteButton: 'menu',
62-
durationMode: { android: 'longpress', apple: 'exact' },
6356
},
6457
} as const satisfies Record<string, TvRemoteButtonDefinition>;
6558

@@ -98,9 +91,9 @@ export function toAndroidTvRemoteKeyevent(button: TvRemoteButton): string {
9891
}
9992

10093
export function tvRemoteDurationMode(
101-
platform: keyof TvRemoteButtonDefinition['durationMode'],
94+
platform: keyof typeof TV_REMOTE_DURATION_MODE,
10295
): TvRemoteDurationMode {
103-
return TV_REMOTE_BUTTON_DEFINITIONS.select.durationMode[platform];
96+
return TV_REMOTE_DURATION_MODE[platform];
10497
}
10598

10699
const TV_REMOTE_BUTTON_ALIAS_MAP = new Map<string, TvRemoteButton>(

‎src/platforms/android/__tests__/ui-hierarchy.test.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,20 @@ test('parseUiHierarchy excludes Android nodes that are not visible to the user',
205205
);
206206
});
207207

208+
test('parseUiHierarchy keeps focused non-clickable Android TV nodes in interactive snapshots', () => {
209+
const xml = `<hierarchy>
210+
<node class="android.widget.FrameLayout" bounds="[0,0][960,540]" enabled="true" visible-to-user="true">
211+
<node class="android.widget.TextView" text="Featured" bounds="[80,80][360,160]" clickable="false" focusable="true" focused="true" enabled="true" visible-to-user="true"/>
212+
</node>
213+
</hierarchy>`;
214+
215+
const result = parseUiHierarchy(xml, 800, { interactiveOnly: true });
216+
const focused = result.nodes.find((node) => node.label === 'Featured');
217+
218+
assert.equal(focused?.focused, true);
219+
assert.equal(focused?.hittable, false);
220+
});
221+
208222
test('parseUiHierarchy prunes Android nodes that are not visible to the user in raw snapshots', () => {
209223
const xml = `<hierarchy>
210224
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" enabled="true" visible-to-user="true">

‎src/platforms/android/ui-hierarchy.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,7 @@ function appendAndroidSnapshotNode(
173173
bundleId: node.packageName ?? undefined,
174174
rect: node.rect,
175175
enabled: node.enabled,
176+
focused: node.focused,
176177
visibleToUser: node.visibleToUser,
177178
hittable: node.hittable,
178179
depth,
@@ -422,6 +423,7 @@ export type AndroidUiHierarchy = {
422423
enabled?: boolean;
423424
visibleToUser?: boolean;
424425
drawingOrder?: number;
426+
focused?: boolean;
425427
hittable?: boolean;
426428
depth: number;
427429
parentIndex?: number;
@@ -490,6 +492,7 @@ export function parseUiHierarchyTree(xml: string): AndroidUiHierarchy {
490492
packageName: attrs.packageName,
491493
rect: attrs.rect,
492494
enabled: attrs.enabled,
495+
focused: attrs.focused,
493496
visibleToUser: attrs.visibleToUser,
494497
drawingOrder: attrs.drawingOrder,
495498
hittable: attrs.clickable ?? attrs.focusable,
@@ -735,6 +738,7 @@ function shouldIncludeInteractiveAndroidNode(
735738
ancestorCollection: boolean,
736739
): boolean {
737740
if (hasNonPositiveRect(node)) return false;
741+
if (node.focused) return true;
738742
if (node.hittable) return true;
739743
if (isScrollableType(info.type) && descendantHittable) return true;
740744
return shouldIncludeInteractiveProxyNode(

0 commit comments

Comments
 (0)