Skip to content

Commit 4747131

Browse files
committed
feat(snapshot): publish the viewport a snapshot's rects are measured in (#3182)
Carry a SnapshotViewportSize from each producer through the state, response, serialization, digest view, and client reader, published once at response level and only through the kernel guard that makes a zero-size viewport unrepresentable. Android: the snapshot helper reports the DisplayMetrics extent beside the density it already declares; the host converts the pair to a viewport at the helper-result boundary and keeps only the response fact. Apple: the host publishes the viewport evidence the presentation fold already resolves, with no new runner wire key.
1 parent fae04c3 commit 4747131

24 files changed

Lines changed: 329 additions & 24 deletions

File tree

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
package com.callstack.agentdevice.snapshothelper;
2+
3+
/**
4+
* The screen the captured bounds were measured on, in physical pixels, in the current rotation.
5+
*
6+
* Node bounds reach the host through {@code AccessibilityNodeInfo.getBoundsInScreen()}, which is
7+
* absolute screen space, so this is the box those numbers are measured in (#3182). Two consequences
8+
* follow from reading the display instead of the tree: a capture of an empty screen still reports a
9+
* viewport, because the box is a property of the device and no node has to survive on screen for it to
10+
* be known; and the box does not shrink when the foreground app is letterboxed or a keyboard is up,
11+
* because a rect's coordinates answer to the screen rather than to the app's window.
12+
*
13+
* The caller hands in the extent of the same {@code DisplayMetrics} whose {@code density} this helper
14+
* publishes, so size and density cannot describe two different configurations — it is the one the
15+
* framework laid the captured nodes out with, a {@code wm size} or {@code wm density} override
16+
* included, and it rotates with the screen. {@code getRealSize} is deliberately not consulted: it is
17+
* deprecated and rotates inconsistently across the devices this helper runs on.
18+
*
19+
* A display that answers with nothing usable publishes no fact at all. Absence is the only way to say
20+
* "unknown", so the host never sees a zero.
21+
*/
22+
final class DisplaySizeReader {
23+
private DisplaySizeReader() {}
24+
25+
/** The screen's pixel size in the current rotation. */
26+
static final class Size {
27+
final int width;
28+
final int height;
29+
30+
Size(int width, int height) {
31+
this.width = width;
32+
this.height = height;
33+
}
34+
}
35+
36+
/**
37+
* Rejects a display that answered with no usable extent. Plain parameters because a
38+
* {@code DisplayMetrics} is a device type this refusal has to be testable without, and a zero
39+
* reaching the host would claim a screen with no width — the failure this gate exists to prevent.
40+
*/
41+
static Size resolve(int widthPixels, int heightPixels) {
42+
if (widthPixels <= 0 || heightPixels <= 0) return null;
43+
return new Size(widthPixels, heightPixels);
44+
}
45+
}

‎android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import android.content.res.Resources;
66
import android.os.Bundle;
77
import android.util.Base64;
8+
import android.util.DisplayMetrics;
89
import java.io.BufferedReader;
910
import java.io.File;
1011
import java.io.FileOutputStream;
@@ -124,8 +125,19 @@ private static void putCaptureMetadata(
124125
// Physical pixels per dp of the display the bounds above were measured on, from the same
125126
// configuration the framework lays out with (a `wm density` override included), so a host that
126127
// works in dp has the factor beside the pixels instead of a second adb round trip.
127-
result.putString(
128-
"pixelDensity", Float.toString(Resources.getSystem().getDisplayMetrics().density));
128+
DisplayMetrics metrics = Resources.getSystem().getDisplayMetrics();
129+
result.putString("pixelDensity", Float.toString(metrics.density));
130+
// The same display's extent, so the host can publish the box the bounds are measured in without
131+
// reading the tree for it. One metrics read backs both facts, so a rotation landing between the
132+
// density and the size cannot make them describe two configurations. Published by absence: a
133+
// display this helper could not address adds no keys, which the host reads as unknown rather
134+
// than as a screen of no size (#3182).
135+
DisplaySizeReader.Size display =
136+
DisplaySizeReader.resolve(metrics.widthPixels, metrics.heightPixels);
137+
if (display != null) {
138+
result.putString("displayWidth", Integer.toString(display.width));
139+
result.putString("displayHeight", Integer.toString(display.height));
140+
}
129141
}
130142

131143
private void runOneShotViewport(Bundle result) {
Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
package com.callstack.agentdevice.snapshothelper;
2+
3+
public final class DisplaySizeReaderTest {
4+
private DisplaySizeReaderTest() {}
5+
6+
static void run() {
7+
assertReportsAUsableDisplayExtent();
8+
assertRefusesADisplayThatAnsweredWithNoExtent();
9+
}
10+
11+
private static void assertReportsAUsableDisplayExtent() {
12+
DisplaySizeReader.Size portrait = DisplaySizeReader.resolve(1080, 2400);
13+
assertSize(portrait, 1080, 2400, "portrait phone");
14+
15+
// The rotation matters: the caller hands in whichever extent the metrics currently carry, and a
16+
// landscape capture publishes the box its bounds are actually measured in.
17+
DisplaySizeReader.Size landscape = DisplaySizeReader.resolve(2400, 1080);
18+
assertSize(landscape, 2400, 1080, "landscape phone");
19+
20+
// One pixel wide is still a box a rect can be measured in; the gate is zero, not plausibility.
21+
DisplaySizeReader.Size minimal = DisplaySizeReader.resolve(1, 1);
22+
assertSize(minimal, 1, 1, "smallest usable display");
23+
}
24+
25+
private static void assertRefusesADisplayThatAnsweredWithNoExtent() {
26+
// A refused read is what the host reads as "unknown". Answering 0 instead would tell a consumer
27+
// this screen has no width, which is a claim no producer ever measured.
28+
assertNull(DisplaySizeReader.resolve(0, 2400), "zero width");
29+
assertNull(DisplaySizeReader.resolve(1080, 0), "zero height");
30+
assertNull(DisplaySizeReader.resolve(0, 0), "both dimensions zero");
31+
assertNull(DisplaySizeReader.resolve(-1, 2400), "negative width");
32+
assertNull(DisplaySizeReader.resolve(1080, -1), "negative height");
33+
}
34+
35+
private static void assertSize(DisplaySizeReader.Size size, int width, int height, String label) {
36+
if (size == null) {
37+
throw new AssertionError("expected a size for " + label);
38+
}
39+
if (size.width != width || size.height != height) {
40+
throw new AssertionError(
41+
"wrong size for " + label + ": got " + size.width + "x" + size.height);
42+
}
43+
}
44+
45+
private static void assertNull(DisplaySizeReader.Size size, String label) {
46+
if (size != null) {
47+
throw new AssertionError("expected no size for " + label + " but got " + size.width + "x" + size.height);
48+
}
49+
}
50+
}

‎android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/SnapshotHelperTestSuite.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,5 +8,6 @@ public static void main(String[] args) throws Exception {
88
AccessibilityCaptureStabilizerTest.run();
99
BoundedUiAutomationConnectionTest.run();
1010
GestureViewportReaderTest.run();
11+
DisplaySizeReaderTest.run();
1112
}
1213
}

‎packages/capture-kit/src/snapshot-state.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import {
1515
type SnapshotKeyboardBandFact,
1616
snapshotStateProvenance,
1717
type SnapshotState,
18+
type SnapshotViewportSize,
1819
} from '@agent-device/kernel/snapshot';
1920
import {
2021
annotateCoveredSnapshotNodes,
@@ -51,6 +52,8 @@ export function buildSnapshotState(
5152
targetActivation?: IosTargetActivation;
5253
/** The keyboard band the producer measured, carried to the state the tap guards read (#2660). */
5354
keyboard?: SnapshotKeyboardBandFact;
55+
/** The box the producer measured the rects in (#3182), carried to the response and the find path. */
56+
viewport?: SnapshotViewportSize;
5457
} & SnapshotCaptureProvenance,
5558
flags:
5659
| (Pick<CommandFlags, 'snapshotDepth' | 'snapshotInteractiveOnly' | 'snapshotRaw'> &
@@ -89,6 +92,7 @@ export function buildSnapshotState(
8992
...(data.systemSurface ? { iosSystemSurfaceBundleId: data.systemSurface.bundleId } : {}),
9093
...(data.targetActivation ? { targetActivation: data.targetActivation } : {}),
9194
...(data.keyboard ? { keyboard: data.keyboard } : {}),
95+
...(data.viewport ? { viewport: data.viewport } : {}),
9296
presentationKey: buildSnapshotPresentationKey(snapshotPresentationOptionsFromFlags(flags)),
9397
// Only broad Android snapshots become freshness baselines. If the user asked for a scoped
9498
// or filtered view, preserve that output contract but avoid pretending it is safe for

‎packages/contracts/src/client-capture.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import type {
88
SnapshotKeyboardBandFact,
99
SnapshotNode,
1010
SnapshotUnchanged,
11+
SnapshotViewportSize,
1112
SnapshotVisibility,
1213
} from '@agent-device/kernel/snapshot';
1314
import type { ScreenshotResultData } from './snapshot-types.ts';
@@ -64,6 +65,19 @@ export type CaptureSnapshotResult = {
6465
* the producer measured no band and the tap guard derived one from the tree.
6566
*/
6667
keyboard?: SnapshotKeyboardBandFact;
68+
/**
69+
* The box the node rects are measured in, as the producer measured it (#3182). Same coordinate
70+
* space and orientation as the rects beside it, so a consumer scales and clips against the screen
71+
* it is being shown instead of inferring it from the largest rect on screen.
72+
*
73+
* Which surface it names is the producer's answer, not always the physical panel: the app window
74+
* for iOS (so iPad Split View and a foldable panel do not inflate it), the screen the bounds were
75+
* measured on for Android and Apple TV, the browser viewport on web. Absent means the producer
76+
* measured no box — a provider handing over a bare tree, or a desktop capture with no single
77+
* window behind it — and never a zero. This is the full screen size only; content-safe gesture
78+
* bounds stay with #1821.
79+
*/
80+
viewport?: SnapshotViewportSize;
6781
/**
6882
* Screenshot captured automatically when the semantic snapshot was sparse.
6983
* Remote clients receive a materialized local path through the daemon artifact channel.

‎packages/contracts/src/snapshot-types.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import type {
44
SnapshotOptions,
55
SnapshotQualityVerdict,
66
ScreenshotOverlayRef,
7+
SnapshotViewportSize,
78
} from '@agent-device/kernel/snapshot';
89
import type { DeviceRotation } from './device-rotation.ts';
910
import type { SnapshotDiagnosticsSummary } from './snapshot-diagnostics.ts';
@@ -28,6 +29,11 @@ export type BackendSnapshotResult = {
2829
appName?: string;
2930
appBundleId?: string;
3031
snapshotDiagnostics?: SnapshotDiagnosticsSummary;
32+
/**
33+
* The box the node rects are measured in, as the producer measured it (#3182). Absent means the
34+
* producer measured no box; see {@link SnapshotViewportSize} for what each producer measures.
35+
*/
36+
viewport?: SnapshotViewportSize;
3137
analysis?: { rawNodeCount: number; maxDepth: number };
3238
androidSnapshot?: AndroidSnapshotBackendMetadata;
3339
freshness?: {

‎packages/kernel/src/rect.ts‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import type { Rect } from './snapshot.ts';
1+
import type { Rect, SnapshotViewportSize } from './snapshot.ts';
22

33
/**
44
* CoreGraphics' `CGRectInfinite`, spelled in the four doubles Apple builds it from. This is what a
@@ -40,6 +40,31 @@ export function isPositiveFiniteRect(rect: Rect | undefined): rect is Rect {
4040
return !isCGRectInfinite(rect);
4141
}
4242

43+
/**
44+
* The ONE construction path for the viewport a snapshot response publishes (#3182): a producer hands
45+
* over the box it read, and a box this guard refuses yields `undefined` — the absence that means
46+
* unknown — instead of a size with a zero in it. A producer that published a zero would be
47+
* answering "this screen has no width", which no producer measured; a reader that has to tell the
48+
* two apart can only do it when one of them is unrepresentable.
49+
*/
50+
export function snapshotViewportSizeFrom(box: Rect | undefined): SnapshotViewportSize | undefined {
51+
if (!isPositiveFiniteRect(box)) return undefined;
52+
return { width: box.width, height: box.height };
53+
}
54+
55+
/**
56+
* Re-read of a viewport this repo already published, out of an untyped wire payload: only a
57+
* `{ width, height }` pair the guard accepts survives, and anything else is the absence that means
58+
* unknown. A reader that trusted the payload could otherwise hand a consumer the `width: 0` a broken
59+
* producer wrote, which is the claim {@link snapshotViewportSizeFrom} exists to make unrepresentable.
60+
*/
61+
export function readSnapshotViewportSize(value: unknown): SnapshotViewportSize | undefined {
62+
if (typeof value !== 'object' || value === null || Array.isArray(value)) return undefined;
63+
const { width, height } = value as Record<string, unknown>;
64+
if (typeof width !== 'number' || typeof height !== 'number') return undefined;
65+
return isPositiveFiniteRect({ x: 0, y: 0, width, height }) ? { width, height } : undefined;
66+
}
67+
4368
export function rectContains(container: Rect, nested: Rect): boolean {
4469
return (
4570
nested.x >= container.x &&

‎packages/kernel/src/snapshot.ts‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -338,6 +338,28 @@ export type SnapshotKeyboardBandFact =
338338
*/
339339
| { kind: 'unmeasurable'; reason: string };
340340

341+
/**
342+
* The box a capture's node rects are measured in, published beside them so a reader never has to
343+
* infer it from the tree (#3182). Two dimensions and no origin: the field answers "how big is the
344+
* surface these numbers describe", which is the only question a consumer that places points on the
345+
* tree cannot answer from the tree itself when the tree is empty or sparse.
346+
*
347+
* It is the box of the surface the producer read, which is not always the physical panel. iOS
348+
* reports the app window in the app's orientation space (ADR 0004), which is smaller than the panel
349+
* under iPad Split View and is never the foldable panel `fold` reports (ADR 0025); Android and the
350+
* Apple TV runner report the screen the bounds were measured on; web reports the browser viewport
351+
* the box model is relative to. A consumer that needs a *gesture* band inside those bounds still
352+
* reads `keyboard` and the app window, which is #1821's remaining scope.
353+
*
354+
* Absent means the producer measured no box. Absence is never `0`: `snapshotViewportSizeFrom` from
355+
* `@agent-device/kernel/rect` is the sole construction path and refuses a box
356+
* `isPositiveFiniteRect` refuses, so a zero-size viewport is not a value this type can hold.
357+
*/
358+
export type SnapshotViewportSize = {
359+
width: number;
360+
height: number;
361+
};
362+
341363
export type SnapshotNode = RawSnapshotNode & {
342364
ref: string;
343365
/**
@@ -642,6 +664,12 @@ export type SnapshotState = {
642664
* needs no geometry to be plausible (#2660). Absent means the guard measures the tree as before.
643665
*/
644666
keyboard?: SnapshotKeyboardBandFact;
667+
/**
668+
* The box these rects are measured in, as the producer measured it (#3182). The session keeps it so
669+
* a consumer of the stored tree — the find-path viewport, a later diff — reads the same box the
670+
* response published instead of inferring one from the largest rect still on screen.
671+
*/
672+
viewport?: SnapshotViewportSize;
645673
/**
646674
* iOS: this capture's own command found the session app out of foreground and the runner
647675
* activated it before answering, so an earlier observation in the session described whatever held

‎packages/platform-android/src/snapshot-capture.ts‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,10 @@ import type {
99
RawSnapshotNode,
1010
SnapshotBackend,
1111
SnapshotQualityVerdict,
12+
SnapshotViewportSize,
1213
} from '@agent-device/kernel/snapshot';
14+
import { snapshotViewportSizeFrom } from '@agent-device/kernel/rect';
15+
import type { AndroidSnapshotHelperMetadata } from './snapshot-helper-types.ts';
1316
import type { AndroidSnapshotAnalysis } from './ui-hierarchy.ts';
1417

1518
const androidCaptureEvidence = Symbol('androidSnapshotCaptureEvidence');
@@ -25,6 +28,12 @@ type AndroidSnapshotCaptureData = Readonly<{
2528
analysis: AndroidSnapshotAnalysis;
2629
androidSnapshot: AndroidSnapshotBackendMetadata;
2730
quality?: SnapshotQualityVerdict;
31+
/**
32+
* The screen the captured bounds are measured in (#3182): the helper's own display read, so a
33+
* capture of an empty screen still reports it. Derived, never stored, so this carrier cannot drift
34+
* from the `displayWidth`/`displayHeight` pair it came from.
35+
*/
36+
viewport?: SnapshotViewportSize;
2837
}>;
2938

3039
/** Opaque acquisition envelope: exact Android facts cannot be detached from the captured nodes. */
@@ -45,6 +54,23 @@ export function createAndroidSnapshotCapture(
4554
return data as AndroidSnapshotCapture;
4655
}
4756

57+
/**
58+
* The one place Android's published viewport is built: from the display pair the helper transport
59+
* carries (#3182), through the shared guard, so a zero the platform answered with never becomes a
60+
* viewport a reader trusts. The pair lives only on the helper's own metadata; the backend metadata
61+
* the response publishes keeps the converted fact instead of a second copy of the raw read.
62+
*/
63+
export function androidSnapshotViewportFromHelperMetadata(
64+
metadata: AndroidSnapshotHelperMetadata,
65+
): SnapshotViewportSize | undefined {
66+
return snapshotViewportSizeFrom({
67+
x: 0,
68+
y: 0,
69+
width: metadata.displayWidth ?? Number.NaN,
70+
height: metadata.displayHeight ?? Number.NaN,
71+
});
72+
}
73+
4874
/** The sole adapter from Android acquisition into daemon snapshot publication. */
4975
export function androidSnapshotPublicationInput(
5076
capture: AndroidSnapshotCapture,

0 commit comments

Comments
 (0)