|
| 1 | +import assert from 'node:assert/strict'; |
| 2 | +import { test } from 'vitest'; |
| 3 | +import type { Point, Rect, SnapshotState } from '@agent-device/kernel/snapshot'; |
| 4 | +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; |
| 5 | +import { keyboardCoveredTabBarSnapshot } from '../../../../test/integration/interaction-contract/fixtures.ts'; |
| 6 | +import { ref, selector } from './selector-read-utils.ts'; |
| 7 | +import { createInteractionDevice } from './__tests__/test-utils/index.ts'; |
| 8 | + |
| 9 | +// #2589: the keyboard is its own system surface, so neither `occlusion` nor `offscreen` refuses a |
| 10 | +// tap whose point belongs to it. These cover the consequences the shared classifier cannot express: |
| 11 | +// which paths refuse, which disclose, and which stay silent. The tree itself is the contract |
| 12 | +// fixtures' #2589 shape, so the refusal and the disclosure are measured on the same geometry the |
| 13 | +// ADR 0011 cells claim. |
| 14 | + |
| 15 | +const TAB_BAR_RECT: Rect = { x: 148, y: 791, width: 104, height: 83 }; |
| 16 | + |
| 17 | +/** The contract fixture with its app-owned button moved, so the variants stay one tree, not copies. */ |
| 18 | +function keyboardTree(params: { tabRect?: Rect } = {}): SnapshotState { |
| 19 | + const tabRect = params.tabRect; |
| 20 | + if (!tabRect) return keyboardCoveredTabBarSnapshot(); |
| 21 | + return makeSnapshotState( |
| 22 | + keyboardCoveredTabBarSnapshot().nodes.map((node) => |
| 23 | + node.index === 1 ? { ...node, rect: tabRect } : node, |
| 24 | + ), |
| 25 | + ); |
| 26 | +} |
| 27 | + |
| 28 | +/** |
| 29 | + * Keyboard-owned rects hauled above the docking budget: the tree stops describing a keyboard the |
| 30 | + * bottom of the screen belongs to, which is what the app-drawn-keypad and aim cases measure against. |
| 31 | + */ |
| 32 | +function liftKeyboardOffBottomEdge(nodes: SnapshotState['nodes']): SnapshotState['nodes'] { |
| 33 | + return nodes.map((node) => |
| 34 | + node.rect && (node.type === 'Keyboard' || node.type === 'Key') |
| 35 | + ? { ...node, rect: { ...node.rect, y: node.rect.y - 560 } } |
| 36 | + : node, |
| 37 | + ); |
| 38 | +} |
| 39 | + |
| 40 | +function tappedDevice(snapshot: SnapshotState, calls: Point[]) { |
| 41 | + return createInteractionDevice(snapshot, { |
| 42 | + tap: async (_context, point) => { |
| 43 | + calls.push(point); |
| 44 | + return { ok: true }; |
| 45 | + }, |
| 46 | + }); |
| 47 | +} |
| 48 | + |
| 49 | +test('press @ref refuses a target whose center sits behind the visible keyboard', async () => { |
| 50 | + const calls: Point[] = []; |
| 51 | + const device = tappedDevice(keyboardTree(), calls); |
| 52 | + |
| 53 | + await assert.rejects( |
| 54 | + () => device.interactions.click(ref('@e2'), { session: 'default' }), |
| 55 | + (error: unknown) => { |
| 56 | + assert.ok(error instanceof Error); |
| 57 | + assert.equal((error as { code?: string }).code, 'COMMAND_FAILED'); |
| 58 | + assert.match(error.message, /Ref @e2 is behind the visible keyboard/); |
| 59 | + const details = (error as { details?: Record<string, unknown> }).details; |
| 60 | + assert.equal(details?.reason, 'tap_keyboard_occludes_target'); |
| 61 | + assert.equal(details?.ref, '@e2'); |
| 62 | + assert.deepEqual(details?.rect, TAB_BAR_RECT); |
| 63 | + assert.deepEqual(details?.keyboardFrame, { x: 0, y: 583, width: 402, height: 291 }); |
| 64 | + assert.match(String(details?.hint), /End editing first/); |
| 65 | + return true; |
| 66 | + }, |
| 67 | + ); |
| 68 | + assert.deepEqual(calls, []); |
| 69 | +}); |
| 70 | + |
| 71 | +test('fill refuses with the verb matching the action', async () => { |
| 72 | + const device = tappedDevice(keyboardTree(), []); |
| 73 | + await assert.rejects( |
| 74 | + () => device.interactions.fill(ref('@e2'), 'hello', { session: 'default' }), |
| 75 | + /cannot be filled safely/, |
| 76 | + ); |
| 77 | +}); |
| 78 | + |
| 79 | +test('press selector refuses the same node the ref path refuses', async () => { |
| 80 | + const calls: Point[] = []; |
| 81 | + const device = tappedDevice(keyboardTree(), calls); |
| 82 | + |
| 83 | + await assert.rejects( |
| 84 | + () => device.interactions.press(selector('label=Form'), { session: 'default' }), |
| 85 | + (error: unknown) => { |
| 86 | + assert.ok(error instanceof Error); |
| 87 | + assert.match(error.message, /Selector label=Form is behind the visible keyboard/); |
| 88 | + const details = (error as { details?: Record<string, unknown> }).details; |
| 89 | + assert.equal(details?.reason, 'tap_keyboard_occludes_target'); |
| 90 | + return true; |
| 91 | + }, |
| 92 | + ); |
| 93 | + assert.deepEqual(calls, []); |
| 94 | +}); |
| 95 | + |
| 96 | +test('a keyboard key is a legitimate target: pressing one still taps', async () => { |
| 97 | + const calls: Point[] = []; |
| 98 | + const device = tappedDevice(keyboardTree(), calls); |
| 99 | + |
| 100 | + await device.interactions.click(ref('@e6'), { session: 'default' }); |
| 101 | + |
| 102 | + assert.deepEqual(calls, [{ x: 202, y: 779 }]); |
| 103 | +}); |
| 104 | + |
| 105 | +test('an element whose center stays above the key plane still taps', async () => { |
| 106 | + const calls: Point[] = []; |
| 107 | + const peeking = { x: 148, y: 470, width: 104, height: 110 }; |
| 108 | + const device = tappedDevice(keyboardTree({ tabRect: peeking }), calls); |
| 109 | + |
| 110 | + await device.interactions.click(ref('@e2'), { session: 'default' }); |
| 111 | + |
| 112 | + assert.deepEqual(calls, [{ x: 200, y: 525 }]); |
| 113 | +}); |
| 114 | + |
| 115 | +test('no keyboard in the tree means nothing to refuse', async () => { |
| 116 | + const calls: Point[] = []; |
| 117 | + const withoutKeyboard = makeSnapshotState( |
| 118 | + keyboardTree().nodes.filter((node) => node.type !== 'Keyboard' && node.type !== 'Key'), |
| 119 | + ); |
| 120 | + const device = tappedDevice(withoutKeyboard, calls); |
| 121 | + |
| 122 | + await device.interactions.click(ref('@e2'), { session: 'default' }); |
| 123 | + |
| 124 | + assert.deepEqual(calls, [{ x: 200, y: 833 }]); |
| 125 | +}); |
| 126 | + |
| 127 | +test('an app-drawn keypad that stops short of the bottom edge is not the system keyboard', async () => { |
| 128 | + const calls: Point[] = []; |
| 129 | + const appOwnedKeypad = makeSnapshotState(liftKeyboardOffBottomEdge(keyboardTree().nodes)); |
| 130 | + const device = tappedDevice(appOwnedKeypad, calls); |
| 131 | + |
| 132 | + await device.interactions.click(ref('@e2'), { session: 'default' }); |
| 133 | + |
| 134 | + assert.deepEqual(calls, [{ x: 200, y: 833 }]); |
| 135 | +}); |
| 136 | + |
| 137 | +test('a coordinate behind the keyboard taps anyway and discloses the reason', async () => { |
| 138 | + const calls: Point[] = []; |
| 139 | + const device = tappedDevice(keyboardTree(), calls); |
| 140 | + |
| 141 | + const result = await device.interactions.press( |
| 142 | + { kind: 'point', x: 200, y: 810 }, |
| 143 | + { |
| 144 | + session: 'default', |
| 145 | + }, |
| 146 | + ); |
| 147 | + |
| 148 | + assert.deepEqual(calls, [{ x: 200, y: 810 }]); |
| 149 | + assert.match(result.warning ?? '', /behind the visible keyboard/); |
| 150 | + assert.match(result.warning ?? '', /tap_keyboard_occludes_target/); |
| 151 | +}); |
| 152 | + |
| 153 | +test('the guard reads the point the interaction dispatches, not the rect center', async () => { |
| 154 | + // The Form button's own center sits 3 pt above the key plane, and its interactive child owns that |
| 155 | + // upper region, so the point the tap dispatches is pushed down into the keyboard's band. |
| 156 | + const aimShifted = makeSnapshotState([ |
| 157 | + ...keyboardCoveredTabBarSnapshot().nodes, |
| 158 | + { |
| 159 | + index: 5, |
| 160 | + depth: 3, |
| 161 | + parentIndex: 1, |
| 162 | + type: 'Button', |
| 163 | + label: 'Send', |
| 164 | + rect: { x: 148, y: 500, width: 104, height: 83 }, |
| 165 | + hittable: true, |
| 166 | + }, |
| 167 | + ]).nodes.map((node) => |
| 168 | + node.index === 1 ? { ...node, rect: { x: 148, y: 500, width: 104, height: 160 } } : node, |
| 169 | + ); |
| 170 | + |
| 171 | + const calls: Point[] = []; |
| 172 | + await assert.rejects( |
| 173 | + () => |
| 174 | + tappedDevice(makeSnapshotState(aimShifted), calls).interactions.click(ref('@e2'), { |
| 175 | + session: 'default', |
| 176 | + }), |
| 177 | + /Ref @e2 is behind the visible keyboard/, |
| 178 | + ); |
| 179 | + assert.deepEqual(calls, []); |
| 180 | + |
| 181 | + // The same tree with its keyboard hauled off the bottom edge shows where that tap was aiming: below |
| 182 | + // the key plane, which is what makes this the dispatched point's refusal rather than the center's. |
| 183 | + const aim: Point[] = []; |
| 184 | + await tappedDevice( |
| 185 | + makeSnapshotState(liftKeyboardOffBottomEdge(aimShifted)), |
| 186 | + aim, |
| 187 | + ).interactions.click(ref('@e2'), { session: 'default' }); |
| 188 | + assert.equal(aim.length, 1); |
| 189 | + assert.ok((aim[0]?.y ?? 0) > 583, `expected the dispatched point below 583, got ${aim[0]?.y}`); |
| 190 | +}); |
| 191 | + |
| 192 | +test('a coordinate on a reported key is the keyboard the caller asked for', async () => { |
| 193 | + const calls: Point[] = []; |
| 194 | + const device = tappedDevice(keyboardTree(), calls); |
| 195 | + |
| 196 | + const result = await device.interactions.press( |
| 197 | + { kind: 'point', x: 160, y: 760 }, |
| 198 | + { |
| 199 | + session: 'default', |
| 200 | + }, |
| 201 | + ); |
| 202 | + |
| 203 | + assert.deepEqual(calls, [{ x: 160, y: 760 }]); |
| 204 | + assert.equal(result.warning, undefined); |
| 205 | +}); |
| 206 | + |
| 207 | +test('the out-of-viewport disclosure still wins on a stale-tree coordinate', async () => { |
| 208 | + const device = tappedDevice(keyboardTree(), []); |
| 209 | + |
| 210 | + const result = await device.interactions.press( |
| 211 | + { kind: 'point', x: 900, y: 810 }, |
| 212 | + { |
| 213 | + session: 'default', |
| 214 | + }, |
| 215 | + ); |
| 216 | + |
| 217 | + assert.match(result.warning ?? '', /outside the last-known viewport/); |
| 218 | +}); |
0 commit comments