|
| 1 | +import assert from 'node:assert/strict'; |
| 2 | +import { test } from 'vitest'; |
| 3 | +import type { Point, RawSnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot'; |
| 4 | +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; |
| 5 | +import { ref, selector } from './selector-read-utils.ts'; |
| 6 | +import { createInteractionDevice } from './__tests__/test-utils/index.ts'; |
| 7 | + |
| 8 | +// #2589: the keyboard is its own system surface, so neither `occlusion` nor `offscreen` refuses a |
| 9 | +// tap whose point belongs to it. These cover the consequences the shared classifier cannot express: |
| 10 | +// which paths refuse, which disclose, and which stay silent. |
| 11 | + |
| 12 | +const TAB_BAR_RECT = { x: 148, y: 791, width: 104, height: 83 }; |
| 13 | +const KEYBOARD_RECT = { x: 0, y: 583, width: 402, height: 291 }; |
| 14 | +const SPACE_KEY_RECT = { x: 40, y: 730, width: 240, height: 60 }; |
| 15 | + |
| 16 | +function keyboardNode(overrides: Partial<RawSnapshotNode> = {}): RawSnapshotNode { |
| 17 | + return { |
| 18 | + index: 2, |
| 19 | + depth: 1, |
| 20 | + parentIndex: 0, |
| 21 | + type: 'Keyboard', |
| 22 | + rect: KEYBOARD_RECT, |
| 23 | + hittable: false, |
| 24 | + ...overrides, |
| 25 | + }; |
| 26 | +} |
| 27 | + |
| 28 | +function keyboardTree(params: { tabRect?: RawSnapshotNode['rect'] } = {}): SnapshotState { |
| 29 | + return makeSnapshotState([ |
| 30 | + { index: 0, depth: 0, type: 'Application', rect: { x: 0, y: 0, width: 402, height: 874 } }, |
| 31 | + { |
| 32 | + index: 1, |
| 33 | + depth: 2, |
| 34 | + parentIndex: 0, |
| 35 | + type: 'Button', |
| 36 | + label: 'Form', |
| 37 | + rect: params.tabRect ?? TAB_BAR_RECT, |
| 38 | + hittable: true, |
| 39 | + }, |
| 40 | + keyboardNode(), |
| 41 | + { |
| 42 | + index: 3, |
| 43 | + depth: 2, |
| 44 | + parentIndex: 2, |
| 45 | + type: 'Key', |
| 46 | + label: 'space', |
| 47 | + rect: SPACE_KEY_RECT, |
| 48 | + hittable: true, |
| 49 | + }, |
| 50 | + ]); |
| 51 | +} |
| 52 | + |
| 53 | +function tappedDevice(snapshot: SnapshotState, calls: Point[]) { |
| 54 | + return createInteractionDevice(snapshot, { |
| 55 | + tap: async (_context, point) => { |
| 56 | + calls.push(point); |
| 57 | + return { ok: true }; |
| 58 | + }, |
| 59 | + }); |
| 60 | +} |
| 61 | + |
| 62 | +test('press @ref refuses a target whose center sits behind the visible keyboard', async () => { |
| 63 | + const calls: Point[] = []; |
| 64 | + const device = tappedDevice(keyboardTree(), calls); |
| 65 | + |
| 66 | + await assert.rejects( |
| 67 | + () => device.interactions.click(ref('@e2'), { session: 'default' }), |
| 68 | + (error: unknown) => { |
| 69 | + assert.ok(error instanceof Error); |
| 70 | + assert.equal((error as { code?: string }).code, 'COMMAND_FAILED'); |
| 71 | + assert.match(error.message, /Ref @e2 is behind the visible keyboard/); |
| 72 | + const details = (error as { details?: Record<string, unknown> }).details; |
| 73 | + assert.equal(details?.reason, 'tap_keyboard_occludes_target'); |
| 74 | + assert.equal(details?.ref, '@e2'); |
| 75 | + assert.deepEqual(details?.rect, TAB_BAR_RECT); |
| 76 | + assert.deepEqual(details?.keyboardFrame, { x: 0, y: 583, width: 402, height: 291 }); |
| 77 | + assert.match(String(details?.hint), /End editing first/); |
| 78 | + return true; |
| 79 | + }, |
| 80 | + ); |
| 81 | + assert.deepEqual(calls, []); |
| 82 | +}); |
| 83 | + |
| 84 | +test('fill refuses with the verb matching the action', async () => { |
| 85 | + const device = tappedDevice(keyboardTree(), []); |
| 86 | + await assert.rejects( |
| 87 | + () => device.interactions.fill(ref('@e2'), 'hello', { session: 'default' }), |
| 88 | + /cannot be filled safely/, |
| 89 | + ); |
| 90 | +}); |
| 91 | + |
| 92 | +test('press selector refuses the same node the ref path refuses', async () => { |
| 93 | + const calls: Point[] = []; |
| 94 | + const device = tappedDevice(keyboardTree(), calls); |
| 95 | + |
| 96 | + await assert.rejects( |
| 97 | + () => device.interactions.press(selector('label=Form'), { session: 'default' }), |
| 98 | + (error: unknown) => { |
| 99 | + assert.ok(error instanceof Error); |
| 100 | + assert.match(error.message, /Selector label=Form is behind the visible keyboard/); |
| 101 | + const details = (error as { details?: Record<string, unknown> }).details; |
| 102 | + assert.equal(details?.reason, 'tap_keyboard_occludes_target'); |
| 103 | + return true; |
| 104 | + }, |
| 105 | + ); |
| 106 | + assert.deepEqual(calls, []); |
| 107 | +}); |
| 108 | + |
| 109 | +test('a keyboard key is a legitimate target: pressing one still taps', async () => { |
| 110 | + const calls: Point[] = []; |
| 111 | + const device = tappedDevice(keyboardTree(), calls); |
| 112 | + |
| 113 | + await device.interactions.click(ref('@e4'), { session: 'default' }); |
| 114 | + |
| 115 | + assert.deepEqual(calls, [{ x: 160, y: 760 }]); |
| 116 | +}); |
| 117 | + |
| 118 | +test('an element whose center stays above the key plane still taps', async () => { |
| 119 | + const calls: Point[] = []; |
| 120 | + const peeking = { x: 148, y: 470, width: 104, height: 110 }; |
| 121 | + const device = tappedDevice(keyboardTree({ tabRect: peeking }), calls); |
| 122 | + |
| 123 | + await device.interactions.click(ref('@e2'), { session: 'default' }); |
| 124 | + |
| 125 | + assert.deepEqual(calls, [{ x: 200, y: 525 }]); |
| 126 | +}); |
| 127 | + |
| 128 | +test('no keyboard in the tree means nothing to refuse', async () => { |
| 129 | + const calls: Point[] = []; |
| 130 | + const withoutKeyboard = makeSnapshotState( |
| 131 | + keyboardTree().nodes.filter((node) => node.type !== 'Keyboard' && node.type !== 'Key'), |
| 132 | + ); |
| 133 | + const device = tappedDevice(withoutKeyboard, calls); |
| 134 | + |
| 135 | + await device.interactions.click(ref('@e2'), { session: 'default' }); |
| 136 | + |
| 137 | + assert.deepEqual(calls, [{ x: 200, y: 833 }]); |
| 138 | +}); |
| 139 | + |
| 140 | +test('an app-drawn keypad in the upper half is not the system keyboard', async () => { |
| 141 | + const calls: Point[] = []; |
| 142 | + const upperKeypad = makeSnapshotState([ |
| 143 | + ...keyboardTree().nodes.slice(0, 1), |
| 144 | + keyboardTree().nodes[1]!, |
| 145 | + keyboardNode({ rect: { x: 0, y: 60, width: 402, height: 220 } }), |
| 146 | + keyboardTree().nodes[3]!, |
| 147 | + ]); |
| 148 | + const device = tappedDevice(upperKeypad, calls); |
| 149 | + |
| 150 | + await device.interactions.click(ref('@e2'), { session: 'default' }); |
| 151 | + |
| 152 | + assert.deepEqual(calls, [{ x: 200, y: 833 }]); |
| 153 | +}); |
| 154 | + |
| 155 | +test('a coordinate behind the keyboard taps anyway and discloses the reason', async () => { |
| 156 | + const calls: Point[] = []; |
| 157 | + const device = tappedDevice(keyboardTree(), calls); |
| 158 | + |
| 159 | + const result = await device.interactions.press( |
| 160 | + { kind: 'point', x: 200, y: 810 }, |
| 161 | + { |
| 162 | + session: 'default', |
| 163 | + }, |
| 164 | + ); |
| 165 | + |
| 166 | + assert.deepEqual(calls, [{ x: 200, y: 810 }]); |
| 167 | + assert.match(result.warning ?? '', /behind the visible keyboard/); |
| 168 | + assert.match(result.warning ?? '', /tap_keyboard_occludes_target/); |
| 169 | +}); |
| 170 | + |
| 171 | +test('a coordinate on a reported key is the keyboard the caller asked for', async () => { |
| 172 | + const calls: Point[] = []; |
| 173 | + const device = tappedDevice(keyboardTree(), calls); |
| 174 | + |
| 175 | + const result = await device.interactions.press( |
| 176 | + { kind: 'point', x: 160, y: 760 }, |
| 177 | + { |
| 178 | + session: 'default', |
| 179 | + }, |
| 180 | + ); |
| 181 | + |
| 182 | + assert.deepEqual(calls, [{ x: 160, y: 760 }]); |
| 183 | + assert.equal(result.warning, undefined); |
| 184 | +}); |
| 185 | + |
| 186 | +test('the out-of-viewport disclosure still wins on a stale-tree coordinate', async () => { |
| 187 | + const device = tappedDevice(keyboardTree(), []); |
| 188 | + |
| 189 | + const result = await device.interactions.press( |
| 190 | + { kind: 'point', x: 900, y: 810 }, |
| 191 | + { |
| 192 | + session: 'default', |
| 193 | + }, |
| 194 | + ); |
| 195 | + |
| 196 | + assert.match(result.warning ?? '', /outside the last-known viewport/); |
| 197 | +}); |
0 commit comments