Skip to content

Commit 4e48ff1

Browse files
thymikeeclaude
andcommitted
fix(mobile): measure observations in the viewport agent-device publishes
agent-device 0.21.21 publishes the viewport a snapshot's rects are measured in (callstack/agent-device#3182). The engine prefers it and infers one from the tree only for a snapshot that publishes none. The benchmark capture keeps the field for the next recapture. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 449fa93 commit 4e48ff1

7 files changed

Lines changed: 50 additions & 11 deletions

File tree

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@e2e-dev/mobile": patch
3+
---
4+
5+
The mobile engine measures an observation in the viewport agent-device publishes with each snapshot (0.21.21), and infers it from the tree's largest window or rect extent only for a snapshot that publishes none. A tree that doesn't reach the screen's edges no longer shrinks the viewport, so a system panel above the bottom of the screen is no longer taken for the navigation bar.

‎apps/mobile-benchmark/tests-capture/snapshots.e2e.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ function slug(name: string): string {
4040

4141
interface Snapshot {
4242
readonly nodes: readonly { readonly type?: string; readonly label?: string; readonly value?: string }[];
43+
readonly viewport?: { readonly width: number; readonly height: number };
4344
readonly truncated?: boolean;
4445
readonly appName?: string;
4546
readonly appBundleId?: string;
@@ -74,6 +75,7 @@ function write(platform: Platform, name: string, snap: Snapshot): void {
7475
mkdirSync(path.join(OUT, platform), { recursive: true });
7576
const kept = {
7677
nodes: snap.nodes,
78+
...(snap.viewport === undefined ? {} : { viewport: snap.viewport }),
7779
...(snap.truncated === undefined ? {} : { truncated: snap.truncated }),
7880
...(snap.appName === undefined ? {} : { appName: snap.appName }),
7981
...(snap.appBundleId === undefined ? {} : { appBundleId: snap.appBundleId }),

‎packages/mobile/src/nodes.ts‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -382,11 +382,16 @@ function isSystemBar(node: RawNode, screen: ViewportSize | undefined, displayOnl
382382

383383
/**
384384
* Projects one snapshot. `mintId` is called once per node in document order,
385-
* so the surface's id space stays unique across observations. Android's
386-
* system bars are left out with their children, see `isSystemBar`.
385+
* so the surface's id space stays unique across observations. `viewport` is
386+
* the one agent-device published with the snapshot; a producer that measured
387+
* none leaves it to `viewportOf`. Android's system bars are left out with
388+
* their children, see `isSystemBar`.
387389
*/
388-
export function projectSnapshot(raw: readonly RawNode[], options: { readonly mintId: () => string }): ProjectedSnapshot {
389-
const viewport = viewportOf(raw);
390+
export function projectSnapshot(
391+
raw: readonly RawNode[],
392+
options: { readonly mintId: () => string; readonly viewport?: ViewportSize | undefined },
393+
): ProjectedSnapshot {
394+
const viewport = options.viewport ?? viewportOf(raw);
390395
const parents = parentPositions(raw);
391396
const children = new Map<number, number[]>();
392397
const roots: number[] = [];

‎packages/mobile/src/surface.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ export interface OpenAppOptions {
130130
* is the band the capture's producer measured: `visible` with its frame,
131131
* `absent`, or that it could not look.
132132
*/
133-
type RawSnapshot = Partial<Pick<CaptureSnapshotResult, 'nodes' | 'truncated' | 'appName' | 'appBundleId' | 'snapshotQuality' | 'keyboard'>>;
133+
type RawSnapshot = Partial<Pick<CaptureSnapshotResult, 'nodes' | 'viewport' | 'truncated' | 'appName' | 'appBundleId' | 'snapshotQuality' | 'keyboard'>>;
134134

135135
/** Platform element types that are the soft keyboard or one of its keys, as agent-device names them. */
136136
const KEYBOARD_TYPES: ReadonlySet<string> = new Set(['keyboard', 'key']);
@@ -936,6 +936,7 @@ export class AgentDeviceSurface {
936936
this.idCounter += 1;
937937
return `n${this.idCounter}`;
938938
},
939+
viewport: raw.viewport,
939940
});
940941
this.latestIndex = projected.index;
941942
if (projected.viewport !== undefined) this.knownViewport = projected.viewport;
@@ -1360,7 +1361,7 @@ export class AgentDeviceSurface {
13601361
private async dismissKeyboardByGesture(signal: AbortSignal): Promise<void> {
13611362
let raw = await this.dismissKeyboardTip(await this.snapshot(signal, false), signal);
13621363
if (!keyboardShowing(raw)) return;
1363-
const projected = projectSnapshot(raw.nodes ?? [], { mintId: () => 'keyboard' });
1364+
const projected = projectSnapshot(raw.nodes ?? [], { mintId: () => 'keyboard', viewport: raw.viewport });
13641365
const viewport = await this.viewportFor(projected, signal);
13651366
const centre = { x: viewport.width / 2, y: viewport.height / 2 };
13661367
const drags: readonly Point[] = [

‎packages/mobile/tests/unit/captured-snapshots.test.ts‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@
1010
import { existsSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
1111
import path from 'node:path';
1212
import { describe, expect, it } from 'vitest';
13-
import type { SemanticNode } from 'e2e/engine';
14-
import { projectSnapshot, screenRoot, viewportOf, type RawNode } from '../../src/nodes.ts';
13+
import type { SemanticNode, ViewportSize } from 'e2e/engine';
14+
import { projectSnapshot, screenRoot, type RawNode } from '../../src/nodes.ts';
1515

1616
const ROOT = path.join(import.meta.dirname, '../fixtures/snapshots');
1717
const UPDATE = process.env['E2E_GOLDEN_UPDATE'] === '1';
@@ -37,10 +37,10 @@ function render(node: SemanticNode, depth = 0, out: string[] = []): string[] {
3737

3838
/** The tree `observe` would report for a captured snapshot. */
3939
function treeOf(file: string): string {
40-
const raw = JSON.parse(readFileSync(file, 'utf8')) as { nodes: RawNode[]; truncated?: boolean };
40+
const raw = JSON.parse(readFileSync(file, 'utf8')) as { nodes: RawNode[]; viewport?: ViewportSize; truncated?: boolean };
4141
let counter = 0;
42-
const projected = projectSnapshot(raw.nodes, { mintId: () => `n${(counter += 1)}` });
43-
const root = screenRoot(projected.roots, viewportOf(raw.nodes) ?? { width: 0, height: 0 });
42+
const projected = projectSnapshot(raw.nodes, { mintId: () => `n${(counter += 1)}`, viewport: raw.viewport });
43+
const root = screenRoot(projected.roots, projected.viewport ?? { width: 0, height: 0 });
4444
return `${raw.truncated === true ? '# truncated\n' : ''}${render(root).join('\n')}\n`;
4545
}
4646

‎packages/mobile/tests/unit/engine.test.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -680,6 +680,17 @@ describe('observation', () => {
680680
}
681681
});
682682

683+
it('measures the screen in the viewport the snapshot publishes, and in its tree when it publishes none', async () => {
684+
const h = harness();
685+
// The application node reads 390x844; a published viewport outranks it.
686+
h.fake.respond('capture.snapshot', () => ({ ...SETTINGS_SNAPSHOT, viewport: { width: 402, height: 874 } }));
687+
await openAttempt(h);
688+
expect((await h.engine.observe!(operation())).viewport).toEqual({ width: 402, height: 874 });
689+
h.fake.respond('capture.snapshot', () => SETTINGS_SNAPSHOT);
690+
expect((await h.engine.observe!(operation())).viewport).toEqual({ width: 390, height: 844 });
691+
expect(h.fake.methods()).not.toContain('capture.screenshot');
692+
});
693+
683694
it('is an empty screen before any app is open when no app is pinned, and a fault when one is', async () => {
684695
const free = harness({}, false);
685696
free.fake.respond('capture.snapshot', () => {

‎packages/mobile/tests/unit/nodes.test.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,21 @@ describe('snapshot projection', () => {
8585
expect(viewportOf([{ type: 'button' }])).toBeUndefined();
8686
});
8787

88+
it('measures system bars against a published viewport, where the tree alone would end the screen at its lowest window', () => {
89+
const systemui = 'com.android.systemui';
90+
// A panel above the bottom of a 2400 px screen, at the bottom of a tree that reaches only 1800 px.
91+
const nodes: RawNode[] = [
92+
{ ref: '@e1', depth: 0, type: 'android.widget.FrameLayout', bundleId: 'dev.e2e.benchmark', rect: { x: 0, y: 0, width: 1080, height: 1800 } },
93+
{ ref: '@e2', depth: 0, type: 'android.widget.FrameLayout', bundleId: systemui, rect: { x: 0, y: 1700, width: 1080, height: 100 } },
94+
];
95+
const published = projectSnapshot(nodes, { mintId: () => 'n', viewport: { width: 1080, height: 2400 } });
96+
expect(published.viewport).toEqual({ width: 1080, height: 2400 });
97+
expect(published.roots).toHaveLength(2);
98+
const inferred = projectSnapshot(nodes, { mintId: () => 'n' });
99+
expect(inferred.viewport).toEqual({ width: 1080, height: 1800 });
100+
expect(inferred.roots).toHaveLength(1);
101+
});
102+
88103
it('leaves the Android status bar and navigation bar out, with their children, and keeps the app window', () => {
89104
const systemui = 'com.android.systemui';
90105
const projected = project([

0 commit comments

Comments
 (0)