Repository navigation
Expand file tree
/
Copy pathviewport.ts
More file actions
407 lines (380 loc) · 16.1 KB
/
Copy pathviewport.ts
File metadata and controls
407 lines (380 loc) · 16.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
import type {Element as AppiumElement, Rect, Size, StringRecord} from '@appium/types';
import type {WebviewsMapping} from 'appium-android-driver';
import type {Chromedriver} from 'appium-chromedriver';
import {errors} from 'appium/driver.js';
import {util} from 'appium/support.js';
import type {AndroidUiautomator2Driver} from '../driver.js';
import type {RelativeRect} from './types.js';
declare module '../driver.js' {
interface AndroidUiautomator2Driver {
getStatusBarHeight(...args: Parameters<typeof getStatusBarHeight>): ReturnType<typeof getStatusBarHeight>;
getDevicePixelRatio(...args: Parameters<typeof getDevicePixelRatio>): ReturnType<typeof getDevicePixelRatio>;
getViewPortRect(...args: Parameters<typeof getViewPortRect>): ReturnType<typeof getViewPortRect>;
mobileViewPortRect(...args: Parameters<typeof mobileViewPortRect>): ReturnType<typeof mobileViewPortRect>;
mobileViewportElementRect(
...args: Parameters<typeof mobileViewportElementRect>
): ReturnType<typeof mobileViewportElementRect>;
}
}
/**
* Gets the status bar height in pixels.
* @returns The status bar height in pixels.
*/
export async function getStatusBarHeight(this: AndroidUiautomator2Driver): Promise<number> {
const {statusBar} = (await this.uiautomator2.jwproxy.command(`/appium/device/system_bars`, 'GET', {})) as {
statusBar: number;
};
return statusBar;
}
/**
* Gets the device pixel ratio.
* @returns The device pixel ratio as a string.
*/
export async function getDevicePixelRatio(this: AndroidUiautomator2Driver): Promise<string> {
return String(await this.uiautomator2.jwproxy.command('/appium/device/pixel_ratio', 'GET', {}));
}
/**
* Gets the viewport rectangle coordinates.
* @returns The viewport rectangle (left, top, width, height), accounting for status bar height.
*/
export async function getViewPortRect(this: AndroidUiautomator2Driver): Promise<RelativeRect> {
const windowSize = await this.getWindowSize();
const statusBarHeight = await this.getStatusBarHeight();
// android returns the upscaled window size, so to get the true size of the
// rect we have to downscale
return {
left: 0,
top: statusBarHeight,
width: windowSize.width,
height: windowSize.height - statusBarHeight,
};
}
/**
* Returns the viewport coordinates.
* @returns The viewport rectangle (left, top, width, height).
*/
export async function mobileViewPortRect(this: AndroidUiautomator2Driver): Promise<RelativeRect> {
return await this.getViewPortRect();
}
/**
* Clips `rect` to the area covered by `bounds`, so that it never extends
* outside it. An element that is partially (or entirely) scrolled out of its
* container, or that otherwise overflows it, would otherwise translate to
* coordinates outside of what's actually on screen at that container.
*
* @returns The clipped rectangle. Its width/height are 0 if `rect` doesn't
* overlap `bounds` at all.
*/
export function clampRectToBounds(rect: Rect, bounds: Rect): Rect {
const left = Math.min(Math.max(rect.x, bounds.x), bounds.x + bounds.width);
const top = Math.min(Math.max(rect.y, bounds.y), bounds.y + bounds.height);
const right = Math.min(Math.max(rect.x + rect.width, bounds.x), bounds.x + bounds.width);
const bottom = Math.min(Math.max(rect.y + rect.height, bounds.y), bounds.y + bounds.height);
return {
x: left,
y: top,
width: Math.max(0, right - left),
height: Math.max(0, bottom - top),
};
}
/**
* Gets the rectangle of an element located in the current web view context,
* translated into native device screen coordinates. `getElementRect` is
* relative to the element's own (possibly nested-frame) document and ignores
* scroll/zoom, so this corrects for scroll offset, ancestor frame offsets,
* and the visual viewport (pinch/input zoom) before applying the device
* pixel ratio and the hosting WebView's on-screen bounds (see
* `getWebviewGeometryContext`). The result is clamped to those bounds.
*
* @param elementId - ID of an element found in the current web view context.
* @returns The element rectangle (x, y, width, height) in native screen coordinates.
* @throws {errors.InvalidContextError} If the current context is not a web view.
* @throws {errors.ElementNotInteractableError} If the element is not visible/is entirely
* offscreen, i.e. outside the WebView's own on-screen bounds, once clamped.
*/
export async function mobileViewportElementRect(this: AndroidUiautomator2Driver, elementId: string): Promise<Rect> {
if (!this.isWebContext()) {
throw new errors.InvalidContextError(
'The current context must be a web view in order to translate an element rect into native screen coordinates',
);
}
const [webRect, geometry, webViewRect] = await Promise.all([
this.getElementRect(elementId),
getWebviewGeometryContext(this),
getNativeWebViewRect(this),
]);
// getElementRect's x/y are relative to the current frame's own document; convert
// to layout-viewport-relative coordinates of the top-level page
const layoutX = webRect.x - geometry.scrollX + geometry.frameOffsetX;
const layoutY = webRect.y - geometry.scrollY + geometry.frameOffsetY;
// then account for the page's visual viewport (pinch zoom / auto input zoom),
// which the layout viewport and devicePixelRatio alone don't reflect
const visualX = (layoutX - geometry.visualOffsetLeft) * geometry.visualScale;
const visualY = (layoutY - geometry.visualOffsetTop) * geometry.visualScale;
const scale = geometry.visualScale * geometry.pixelRatio;
const translated = {
x: Math.round(webViewRect.x + visualX * geometry.pixelRatio),
y: Math.round(webViewRect.y + visualY * geometry.pixelRatio),
width: Math.round(webRect.width * scale),
height: Math.round(webRect.height * scale),
};
const clamped = clampRectToBounds(translated, webViewRect);
if (clamped.width === 0 || clamped.height === 0) {
throw new errors.ElementNotInteractableError(
`The element is not visible: its translated rectangle does not overlap the WebView's on-screen bounds`,
);
}
return clamped;
}
/**
* Gets the window rectangle (W3C endpoint).
* @returns The window rectangle (x, y, width, height).
*/
export async function getWindowRect(this: AndroidUiautomator2Driver): Promise<Rect> {
const {width, height} = await this.getWindowSize();
return {
width,
height,
x: 0,
y: 0,
};
}
/**
* Gets the display density.
* @returns The display density value.
*/
export async function getDisplayDensity(this: AndroidUiautomator2Driver): Promise<number> {
return (await this.uiautomator2.jwproxy.command('/appium/device/display_density', 'GET', {})) as number;
}
/**
* Gets the window size.
* @returns The window size (width, height).
*/
export async function getWindowSize(this: AndroidUiautomator2Driver): Promise<Size> {
return (await this.uiautomator2.jwproxy.command('/window/current/size', 'GET', {})) as Size;
}
export type WebviewsMappingWithRect = WebviewsMapping & {
/** The webview's on-screen bounding rectangle, in native device screen coordinates, if it could be determined. */
rect?: Rect;
};
/**
* Enriches a `mobile: getContexts` mapping with each webview's on-screen `rect`. Prefers
* CDP-reported bounds; falls back to a native view hierarchy scan only when there's exactly
* one (otherwise ambiguous) webview with no CDP data.
*/
export async function enrichWebviewsMappingWithRects(
driver: AndroidUiautomator2Driver,
mapping: WebviewsMapping[],
): Promise<WebviewsMappingWithRect[]> {
let hasCdpRect = false;
for (const m of mapping as WebviewsMappingWithRect[]) {
for (const page of m.pages ?? []) {
const rect = parseWebviewRectFromPage(page as StringRecord);
if (rect) {
m.rect = rect;
hasCdpRect = true;
break;
}
}
}
if (!hasCdpRect && mapping.length === 1) {
try {
(mapping[0] as WebviewsMappingWithRect).rect = await getNativeWebViewRectFromViewHierarchy(driver);
} catch {
// no visible native WebView element found either; leave rect unattached
}
}
return mapping as WebviewsMappingWithRect[];
}
// broad match so custom/vendor WebView subclasses (hybrid frameworks, etc.) are still found
const NATIVE_WEBVIEW_CLASS_SELECTOR = "//*[contains(@class,'WebView')]";
interface CdpPageDescription {
screenX?: number;
screenY?: number;
width?: number;
height?: number;
visible?: boolean;
empty?: boolean;
}
/**
* Parses a CDP `/json/list` page entry's `description` into its self-reported on-screen
* rect — authoritative regardless of which native view class hosts the WebView.
* @returns `null` if not reported, or reported as non-visible/zero-area.
*/
function parseWebviewRectFromPage(page: StringRecord): Rect | null {
const raw = page.description;
if (typeof raw !== 'string' || !raw) {
return null;
}
let parsed: CdpPageDescription;
try {
parsed = JSON.parse(raw);
} catch {
return null;
}
const {screenX: x, screenY: y, width, height, visible, empty} = parsed;
if (visible === false || empty === true) {
return null;
}
if (
typeof x === 'number' &&
typeof y === 'number' &&
typeof width === 'number' &&
typeof height === 'number' &&
width > 0 &&
height > 0
) {
return {x, y, width, height};
}
return null;
}
/**
* Finds the on-screen bounding rectangle of the currently active web view,
* as self-reported by Chromium's own WebView embedding layer (see
* `parseWebviewRectFromPage`).
*
* @returns The rectangle, or `null` if this data isn't available (e.g. the
* page didn't report a `description`, or the CDP lookup failed).
*/
async function getWebviewRectFromCdp(driver: AndroidUiautomator2Driver): Promise<Rect | null> {
let mapping: WebviewsMapping[];
try {
mapping = await driver.mobileGetContexts();
} catch {
return null;
}
const found = mapping.find((m) => m.webviewName === driver.curContext);
for (const page of found?.pages ?? []) {
const rect = parseWebviewRectFromPage(page as StringRecord);
if (rect) {
return rect;
}
}
// reuse mobileGetContexts' own native-hierarchy fallback rect instead of scanning again
return (found as WebviewsMappingWithRect | undefined)?.rect ?? null;
}
/**
* Finds the on-screen bounding rectangle of the native Android WebView that
* hosts web content, in native device screen coordinates, via an XPath scan
* of the native view hierarchy for a class name containing "WebView".
*
* There is no reliable way to correlate a native view-hierarchy node with the
* specific Chromium instance backing the currently active CDP target, so this
* only succeeds when exactly one non-degenerate (visible, non-zero-area)
* WebView node is found; it throws rather than guessing if more than one
* candidate remains, since silently picking the wrong one (e.g. the largest)
* would translate coordinates using an unrelated WebView's origin and bounds.
* This call bypasses the active web view context, since it must query the
* native view hierarchy rather than the DOM. Used only as a fallback when the
* CDP-reported bounds (see `getWebviewRectFromCdp`) aren't available.
*
* Uses `doFindElementOrEls` directly (skipping `findElOrEls`'s implicit wait), since this
* is a one-shot best-effort lookup rather than a wait-for-element request.
*/
async function getNativeWebViewRectFromViewHierarchy(driver: AndroidUiautomator2Driver): Promise<Rect> {
const webViewElements = (await driver.doFindElementOrEls({
strategy: 'xpath',
selector: NATIVE_WEBVIEW_CLASS_SELECTOR,
context: '',
multiple: true,
})) as AppiumElement[];
if (!webViewElements.length) {
throw new errors.NoSuchElementError('Could not find a native WebView element on screen');
}
// fetched independently per element: one going stale (e.g. the view hierarchy
// changed between the find and this lookup) must not fail the others
const settled = await Promise.allSettled(
webViewElements.map((el) => {
const elementId = util.unwrapElement(el);
return driver.uiautomator2.jwproxy.command(`/element/${elementId}/rect`, 'GET') as Promise<Rect>;
}),
);
const rects = settled
.filter((result): result is PromiseFulfilledResult<Rect> => result.status === 'fulfilled')
.map((result) => result.value)
.filter((rect) => rect.width > 0 && rect.height > 0);
if (!rects.length) {
throw new errors.NoSuchElementError(
'Could not determine the bounds of any visible native WebView element on screen',
);
}
if (rects.length > 1) {
throw new errors.UnknownError(
'Found multiple visible native WebView elements on screen and could not determine which one hosts the ' +
'current web view context, since its on-screen bounds were not available via CDP. Ensure only one ' +
'WebView is visible on screen at a time, or check that the CDP devtools socket for this web view is reachable.',
);
}
return rects[0];
}
/**
* Finds the on-screen bounding rectangle of the currently active web view, in
* native device screen coordinates. Prefers the bounds Chromium itself
* reports over CDP; falls back to scanning the native view hierarchy only if
* that data isn't available.
*/
async function getNativeWebViewRect(driver: AndroidUiautomator2Driver): Promise<Rect> {
return (await getWebviewRectFromCdp(driver)) ?? (await getNativeWebViewRectFromViewHierarchy(driver));
}
interface WebviewGeometryContext {
/** Ratio between native device pixels and CSS pixels, per `window.devicePixelRatio`. */
pixelRatio: number;
/** The current frame's own horizontal scroll offset, in its own document's CSS pixels. */
scrollX: number;
/** The current frame's own vertical scroll offset, in its own document's CSS pixels. */
scrollY: number;
/** Sum of each ancestor frame's on-screen horizontal offset within its own parent, up to the first cross-origin ancestor. */
frameOffsetX: number;
/** Sum of each ancestor frame's on-screen vertical offset within its own parent, up to the first cross-origin ancestor. */
frameOffsetY: number;
/** The page's visual viewport scale, i.e. the current pinch zoom / auto input zoom level. `1` if unzoomed or unsupported. */
visualScale: number;
/** The visual viewport's horizontal pan offset from the layout viewport, in CSS pixels. */
visualOffsetLeft: number;
/** The visual viewport's vertical pan offset from the layout viewport, in CSS pixels. */
visualOffsetTop: number;
}
// Walks up window.parent to accumulate same-origin ancestor frame offsets (stopping at the
// first cross-origin one, which can't be measured from script), then reads devicePixelRatio,
// the current frame's own scroll, and the outermost reachable window's visual viewport state.
const GET_WEBVIEW_GEOMETRY_CONTEXT_SCRIPT = `
var win = window;
var frameOffsetX = 0;
var frameOffsetY = 0;
try {
while (win !== win.parent) {
var frameRect = win.frameElement.getBoundingClientRect();
frameOffsetX += frameRect.left;
frameOffsetY += frameRect.top;
win = win.parent;
}
} catch (e) {
// cross-origin ancestor frame; its geometry can't be measured from here
}
var visualViewport = win.visualViewport;
return {
pixelRatio: window.devicePixelRatio,
scrollX: window.scrollX,
scrollY: window.scrollY,
frameOffsetX: frameOffsetX,
frameOffsetY: frameOffsetY,
visualScale: visualViewport ? visualViewport.scale : 1,
visualOffsetLeft: visualViewport ? visualViewport.offsetLeft : 0,
visualOffsetTop: visualViewport ? visualViewport.offsetTop : 0,
};
`;
/**
* Reads the geometry needed to translate an element's `getElementRect` rectangle
* (which is relative to its own, possibly nested, document, and unaffected by pinch
* zoom) into a position within the top-level page's currently visible viewport.
* Read directly from Chromium itself, rather than assumed from the OS-level display
* density or a generic viewport rectangle, since none of this can be assumed to stay
* fixed across zoom, scroll, or frame nesting.
*/
async function getWebviewGeometryContext(driver: AndroidUiautomator2Driver): Promise<WebviewGeometryContext> {
const chromedriver = driver.chromedriver as Chromedriver;
return (await chromedriver.jwproxy.command('/execute/sync', 'POST', {
script: GET_WEBVIEW_GEOMETRY_CONTEXT_SCRIPT,
args: [],
})) as WebviewGeometryContext;
}