|
1 | 1 | import assert from 'node:assert/strict'; |
2 | 2 | import { test } from 'vitest'; |
3 | 3 | import type { Point } from '@agent-device/kernel/snapshot'; |
| 4 | +import { attachSnapshotNodeInteractionBounds } from '@agent-device/kernel/rect-center'; |
| 5 | +import { buildSnapshotState } from '@agent-device/capture-kit/snapshot-state'; |
4 | 6 | import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; |
5 | 7 | import { |
6 | 8 | buildUiHierarchySnapshot, |
@@ -120,12 +122,114 @@ test('runtime press refuses a selector match with a zero-area rect and dispatche |
120 | 122 | assert.equal((error as { code?: string }).code, 'COMMAND_FAILED'); |
121 | 123 | assert.equal(details?.reason, 'target_bounds_invalid'); |
122 | 124 | assert.equal(details?.dispatched, 'no'); |
| 125 | + // No scroll container holds the icon, so no scroll is offered as the recovery. |
| 126 | + assert.equal(details?.scrollContainerRef, undefined); |
| 127 | + assert.equal(details?.hint, undefined); |
123 | 128 | return true; |
124 | 129 | }, |
125 | 130 | ); |
126 | 131 | assert.deepEqual(taps, []); |
127 | 132 | }); |
128 | 133 |
|
| 134 | +/** |
| 135 | + * A Flutter screen on iOS at the top of its content (#4451), as the Simulator AX bridge presents it: |
| 136 | + * the scroll view and its content are siblings, and the content below the fold keeps a 0x0 rect. |
| 137 | + */ |
| 138 | +function scrolledOutButtonSnapshot(label = 'View all') { |
| 139 | + return makeSnapshotState([ |
| 140 | + { |
| 141 | + index: 0, |
| 142 | + depth: 0, |
| 143 | + type: 'Application', |
| 144 | + rect: { x: 0, y: 0, width: 402, height: 874 }, |
| 145 | + hittable: false, |
| 146 | + }, |
| 147 | + { |
| 148 | + index: 1, |
| 149 | + depth: 1, |
| 150 | + parentIndex: 0, |
| 151 | + type: 'ScrollView', |
| 152 | + role: 'FlutterSemanticsScrollView', |
| 153 | + rect: { x: 0, y: 118, width: 402, height: 756 }, |
| 154 | + hittable: true, |
| 155 | + }, |
| 156 | + { |
| 157 | + index: 2, |
| 158 | + depth: 1, |
| 159 | + parentIndex: 0, |
| 160 | + type: 'StaticText', |
| 161 | + role: 'FlutterSemanticsObject', |
| 162 | + label: 'Tall block', |
| 163 | + rect: { x: 157, y: 546, width: 86, height: 27 }, |
| 164 | + hittable: true, |
| 165 | + }, |
| 166 | + { |
| 167 | + index: 3, |
| 168 | + depth: 1, |
| 169 | + parentIndex: 0, |
| 170 | + type: 'Button', |
| 171 | + role: 'FlutterSemanticsObject', |
| 172 | + label, |
| 173 | + rect: { x: 0, y: 0, width: 0, height: 0 }, |
| 174 | + hittable: false, |
| 175 | + }, |
| 176 | + ]); |
| 177 | +} |
| 178 | + |
| 179 | +test('runtime press on a zero-area target beside a scroll container names the scroll that reveals it', async () => { |
| 180 | + const taps: Point[] = []; |
| 181 | + const device = createInteractionDevice(scrolledOutButtonSnapshot(), { |
| 182 | + tap: async (_context, point) => { |
| 183 | + taps.push(point); |
| 184 | + return { ok: true }; |
| 185 | + }, |
| 186 | + }); |
| 187 | + |
| 188 | + await assert.rejects( |
| 189 | + () => device.interactions.press(selector('label="View all"'), { session: 'default' }), |
| 190 | + (error: unknown) => { |
| 191 | + const details = (error as { details?: Record<string, unknown> }).details; |
| 192 | + assert.equal(details?.reason, 'target_bounds_invalid'); |
| 193 | + assert.equal(details?.dispatched, 'no'); |
| 194 | + assert.equal(details?.scrollContainerRef, 'e2'); |
| 195 | + assert.match(String(details?.hint), /scroll <direction> --until 'label="View all"'/); |
| 196 | + return true; |
| 197 | + }, |
| 198 | + ); |
| 199 | + await assert.rejects( |
| 200 | + () => device.interactions.press(ref('@e4'), { session: 'default' }), |
| 201 | + (error: unknown) => { |
| 202 | + const details = (error as { details?: Record<string, unknown> }).details; |
| 203 | + assert.equal(details?.reason, 'target_bounds_invalid'); |
| 204 | + assert.equal(details?.scrollContainerRef, 'e2'); |
| 205 | + // A scroll expires the ref frame, so the ref recovery steers to a selector instead. |
| 206 | + assert.match(String(details?.hint), /retry with a selector/); |
| 207 | + assert.match(String(details?.hint), /re-run snapshot -i/); |
| 208 | + assert.doesNotMatch(String(details?.hint), /--until/); |
| 209 | + return true; |
| 210 | + }, |
| 211 | + ); |
| 212 | + assert.deepEqual(taps, []); |
| 213 | +}); |
| 214 | + |
| 215 | +test('the scroll a zero-area target is pointed at survives the shell', async () => { |
| 216 | + // The hint is copied into a shell: double quotes would expand the `$`, and a bare single-quote |
| 217 | + // wrap would end at the apostrophe. |
| 218 | + const device = createInteractionDevice(scrolledOutButtonSnapshot("Today's $5 deal"), { |
| 219 | + tap: async () => ({ ok: true }), |
| 220 | + }); |
| 221 | + |
| 222 | + await assert.rejects( |
| 223 | + () => device.interactions.press(selector('label="Today\'s $5 deal"'), { session: 'default' }), |
| 224 | + (error: unknown) => { |
| 225 | + const details = (error as { details?: Record<string, unknown> }).details; |
| 226 | + assert.equal(details?.reason, 'target_bounds_invalid'); |
| 227 | + assert.match(String(details?.hint), /--until 'label="Today'\\''s \$5 deal"'/); |
| 228 | + return true; |
| 229 | + }, |
| 230 | + ); |
| 231 | +}); |
| 232 | + |
129 | 233 | test('runtime press on a Compose button dispatches its center only once its role node is folded', async () => { |
130 | 234 | const tree = parseUiHierarchyTree(`<hierarchy> |
131 | 235 | <node class="android.view.View" resource-id="compose-container" bounds="[53,1856][1027,1982]" |
@@ -166,3 +270,98 @@ test('runtime press on a Compose button dispatches its center only once its role |
166 | 270 | dispatched: 'no', |
167 | 271 | }); |
168 | 272 | }); |
| 273 | + |
| 274 | +test('runtime press offers no scroll for a zero-area target listed before the only scroll container', async () => { |
| 275 | + const state = scrolledOutButtonSnapshot(); |
| 276 | + const [application, scrollView, , button] = state.nodes; |
| 277 | + const device = createInteractionDevice( |
| 278 | + makeSnapshotState([application!, { ...button!, index: 1 }, { ...scrollView!, index: 2 }]), |
| 279 | + { tap: async () => ({ ok: true }) }, |
| 280 | + ); |
| 281 | + |
| 282 | + await assert.rejects( |
| 283 | + () => device.interactions.press(selector('label="View all"'), { session: 'default' }), |
| 284 | + (error: unknown) => { |
| 285 | + const details = (error as { details?: Record<string, unknown> }).details; |
| 286 | + assert.equal(details?.reason, 'target_bounds_invalid'); |
| 287 | + assert.equal(details?.scrollContainerRef, undefined); |
| 288 | + assert.equal(details?.hint, undefined); |
| 289 | + return true; |
| 290 | + }, |
| 291 | + ); |
| 292 | +}); |
| 293 | + |
| 294 | +test('runtime press offers no scroll for a measured target whose input bounds exclude it', async () => { |
| 295 | + const root = { x: 0, y: 0, width: 1080, height: 2400 }; |
| 296 | + // A row with a real frame, wholly under the navigation bar the input bounds stop at. |
| 297 | + const row = attachSnapshotNodeInteractionBounds( |
| 298 | + { |
| 299 | + index: 2, |
| 300 | + depth: 2, |
| 301 | + parentIndex: 1, |
| 302 | + type: 'Button', |
| 303 | + identifier: 'row', |
| 304 | + label: 'Row', |
| 305 | + hittable: true, |
| 306 | + rect: { x: 0, y: 2305, width: 1080, height: 95 }, |
| 307 | + }, |
| 308 | + { ...root, height: 2274 }, |
| 309 | + ); |
| 310 | + const snapshot = buildSnapshotState( |
| 311 | + { |
| 312 | + backend: 'android', |
| 313 | + producer: 'android-uiautomator', |
| 314 | + nodes: [ |
| 315 | + { index: 0, depth: 0, type: 'Application', rect: root }, |
| 316 | + { index: 1, depth: 1, parentIndex: 0, type: 'ScrollView', rect: root, hittable: true }, |
| 317 | + row, |
| 318 | + ], |
| 319 | + }, |
| 320 | + { snapshotRaw: true }, |
| 321 | + ); |
| 322 | + const device = createInteractionDevice(snapshot, { |
| 323 | + platform: 'android', |
| 324 | + tap: async () => ({ ok: true }), |
| 325 | + }); |
| 326 | + |
| 327 | + await assert.rejects( |
| 328 | + () => device.interactions.press(selector('id=row'), { session: 'default' }), |
| 329 | + (error: unknown) => { |
| 330 | + const details = (error as { details?: Record<string, unknown> }).details; |
| 331 | + assert.equal(details?.reason, 'target_bounds_invalid'); |
| 332 | + assert.equal(details?.scrollContainerRef, undefined); |
| 333 | + assert.equal(details?.hint, undefined); |
| 334 | + return true; |
| 335 | + }, |
| 336 | + ); |
| 337 | +}); |
| 338 | + |
| 339 | +test('runtime press names the scroll view, not a scroll bar listed after it, as the scroll container', async () => { |
| 340 | + const state = scrolledOutButtonSnapshot(); |
| 341 | + const [application, scrollView, block, button] = state.nodes; |
| 342 | + const device = createInteractionDevice( |
| 343 | + makeSnapshotState([ |
| 344 | + application!, |
| 345 | + scrollView!, |
| 346 | + { |
| 347 | + ...block!, |
| 348 | + type: 'Other', |
| 349 | + role: 'AXScrollBar', |
| 350 | + label: 'Vertical scroll bar, 2 pages', |
| 351 | + rect: { x: 396, y: 118, width: 6, height: 756 }, |
| 352 | + }, |
| 353 | + button!, |
| 354 | + ]), |
| 355 | + { tap: async () => ({ ok: true }) }, |
| 356 | + ); |
| 357 | + |
| 358 | + await assert.rejects( |
| 359 | + () => device.interactions.press(selector('label="View all"'), { session: 'default' }), |
| 360 | + (error: unknown) => { |
| 361 | + const details = (error as { details?: Record<string, unknown> }).details; |
| 362 | + assert.equal(details?.reason, 'target_bounds_invalid'); |
| 363 | + assert.equal(details?.scrollContainerRef, 'e2'); |
| 364 | + return true; |
| 365 | + }, |
| 366 | + ); |
| 367 | +}); |
0 commit comments