Skip to content

Commit 5b2c4cd

Browse files
committed
fix(snapshot): retain default views and surface transition evidence
1 parent bba6172 commit 5b2c4cd

9 files changed

Lines changed: 70 additions & 47 deletions

File tree

‎packages/capture-kit/src/snapshot-baseline.test.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,8 @@ test.each([
5757
{ interactiveOnly: true, scope: 'Composer' },
5858
'frame',
5959
],
60-
['interactive frame, candidate with no recorded view', INTERACTIVE, undefined, 'candidate'],
60+
['interactive frame, candidate with default full view', INTERACTIVE, undefined, 'frame'],
61+
['full frame, candidate with default full view', FULL, undefined, 'candidate'],
6162
] as const)(
6263
'%s: the %s answers',
6364
async (_case, framePresentation, candidatePresentation, expected) => {

‎packages/capture-kit/src/snapshot-baseline.ts‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,12 @@ import type { SnapshotState } from '@agent-device/contracts/snapshot-model';
22
import type { SurfaceScopedNodes } from '@agent-device/contracts/interaction';
33
import type { SnapshotPresentation } from '@agent-device/kernel/snapshot';
44

5-
/** The view a post-action read captures in when its baseline recorded none. */
6-
export const DEFAULT_POST_ACTION_PRESENTATION: SnapshotPresentation = { interactiveOnly: true };
5+
/** Reproduce a populated baseline's view; without one, capture the interactive view. */
6+
export function postActionPresentation(
7+
baseline: SurfaceScopedNodes | undefined,
8+
): SnapshotPresentation {
9+
return baseline?.presentation ?? (baseline?.nodes.length ? {} : { interactiveOnly: true });
10+
}
711

812
/**
913
* Mints the one baseline value a post-action comparison carries: the captured nodes together with the
@@ -61,7 +65,7 @@ export async function postActionBaseline(params: {
6165
(baseline) =>
6266
baseline !== undefined &&
6367
baseline.nodes.length > 0 &&
64-
isSameSnapshotPresentation(baseline.presentation, view),
68+
(!frame?.nodes.length || isSameSnapshotPresentation(baseline.presentation, view)),
6569
);
6670
return candidate ?? frame ?? { nodes: [] };
6771
}

‎packages/command-runtime/src/commands/interaction/runtime/post-action-observation.ts‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,12 @@ import type {
66
SettleParams,
77
} from '@agent-device/contracts/interaction';
88
import { isSameSnapshotPresentation } from '@agent-device/kernel/snapshot';
9-
import { summarizePostActionEvidence } from './post-action-surface.ts';
109
import {
11-
DEFAULT_POST_ACTION_PRESENTATION,
10+
resolvePostActionSurfaceChange,
11+
summarizePostActionEvidence,
12+
} from './post-action-surface.ts';
13+
import {
14+
postActionPresentation,
1215
surfaceScopedNodes,
1316
} from '@agent-device/capture-kit/snapshot-baseline';
1417
import { captureSelectorSnapshot } from './selector-read-shared.ts';
@@ -85,14 +88,14 @@ async function observeAfterInteraction(
8588
...params.settle,
8689
resolved,
8790
});
88-
// The evidence compares one view: a pre-action tree read in another view than the settle
89-
// capture makes no changedFromBefore claim (#4486).
91+
// A surface transition is evidence in every view; node digests compare only within one view.
9092
const baseline =
9193
outcome.settledCapture &&
92-
isSameSnapshotPresentation(
93-
resolved.preAction?.presentation,
94-
outcome.settledCapture.presentation,
95-
)
94+
(resolvePostActionSurfaceChange(resolved.preAction, outcome.settledCapture) ||
95+
isSameSnapshotPresentation(
96+
resolved.preAction?.presentation,
97+
outcome.settledCapture.presentation,
98+
))
9699
? resolved.preAction
97100
: undefined;
98101
const evidence = params.verify ? settleEvidence(outcome.settledCapture, baseline) : undefined;
@@ -119,7 +122,7 @@ async function captureVerifyEvidence(
119122
const capture = await captureSelectorSnapshot(runtime, options, {
120123
updateSession: true,
121124
includeRects: true,
122-
presentation: resolved.preAction?.presentation ?? DEFAULT_POST_ACTION_PRESENTATION,
125+
presentation: postActionPresentation(resolved.preAction),
123126
});
124127
return summarizePostActionEvidence(surfaceScopedNodes(capture.snapshot), resolved.preAction);
125128
} catch {

‎packages/command-runtime/src/commands/interaction/runtime/post-action-surface.test.ts‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -117,12 +117,13 @@ test('press --settle attaches no diff across a sheet-to-app surface change and d
117117
assert.match(settle.hint ?? '', /different surfaces/);
118118
});
119119

120-
test('press --settle --verify reports one app-to-sheet surface change on both payloads', async () => {
121-
const before = buttonSnapshot();
122-
const sheet = webSignInSheetSnapshot(WEB_SIGN_IN_SHEET_LABELS);
120+
test('press --settle --verify reports a surface change even when the selector and frame views differ', async () => {
121+
const before = { ...buttonSnapshot(), presentation: { interactiveOnly: true } };
122+
const frame = { ...before, presentation: {} };
123+
const sheet = { ...webSignInSheetSnapshot(WEB_SIGN_IN_SHEET_LABELS), presentation: {} };
123124
let captures = 0;
124125
const device = createSettleDevice({
125-
stored: before,
126+
stored: frame,
126127
captureSnapshot: () => {
127128
captures += 1;
128129
return { snapshot: captures === 1 ? before : sheet };

‎packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts‎

Lines changed: 26 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,11 @@ function dialogNodes(labels: readonly string[], wrapper: boolean) {
4444
}
4545

4646
/** The screen as each view reads it: the full tree keeps a wrapper the interactive view omits. */
47-
function screen(labels: readonly string[], presentation: SnapshotPresentation): SnapshotState {
48-
const full = presentation.interactiveOnly !== true;
47+
function screen(
48+
labels: readonly string[],
49+
presentation: SnapshotPresentation | undefined,
50+
): SnapshotState {
51+
const full = presentation?.interactiveOnly !== true;
4952
return { ...makeSnapshotState(dialogNodes(labels, full)), presentation };
5053
}
5154

@@ -102,14 +105,17 @@ function createDevice(params: {
102105
const removed = (lines: Array<{ kind: string; text: string }> | undefined) =>
103106
(lines ?? []).filter((line) => line.kind === 'removed').map((line) => line.text);
104107

105-
test('a ref press after a full snapshot settles on the full tree and reports no false removals', async () => {
106-
const { device, views } = createDevice({ frame: screen(DIALOG, FULL) });
108+
test.each([FULL, undefined])(
109+
'a ref press after a full snapshot (%j) settles without false removals',
110+
async (presentation) => {
111+
const { device, views } = createDevice({ frame: screen(DIALOG, presentation) });
107112

108-
const result = await device.interactions.press(ref('@e3'), { session: 'default', settle: {} });
113+
const result = await device.interactions.press(ref('@e3'), { session: 'default', settle: {} });
109114

110-
assert.ok(views.length > 0 && views.every((view) => view.interactiveOnly !== true));
111-
assert.deepEqual(result.settle?.diff?.summary, { additions: 0, removals: 0, unchanged: 4 });
112-
});
115+
assert.ok(views.length > 0 && views.every((view) => view.interactiveOnly !== true));
116+
assert.deepEqual(result.settle?.diff?.summary, { additions: 0, removals: 0, unchanged: 4 });
117+
},
118+
);
113119

114120
test('a ref press after snapshot -i settles on the interactive view and lists what left', async () => {
115121
const { device, views } = createDevice({
@@ -155,14 +161,20 @@ test('a selector press resolved on the full tree still diffs in the ref frame vi
155161
assert.deepEqual(removed(result.settle?.diff?.lines), []);
156162
});
157163

158-
test('verify after a full snapshot reads the full tree, so an unchanged screen reads unchanged', async () => {
159-
const { device, views } = createDevice({ frame: screen(DIALOG, FULL) });
164+
test.each([FULL, undefined])(
165+
'verify after a full snapshot (%j) reads unchanged content in the full view',
166+
async (presentation) => {
167+
const { device, views } = createDevice({ frame: screen(DIALOG, presentation) });
160168

161-
const result = await device.interactions.press(ref('@e3'), { session: 'default', verify: true });
169+
const result = await device.interactions.press(ref('@e3'), {
170+
session: 'default',
171+
verify: true,
172+
});
162173

163-
assert.deepEqual(views, [FULL]);
164-
assert.equal(result.evidence?.changedFromBefore, false);
165-
});
174+
assert.deepEqual(views, [FULL]);
175+
assert.equal(result.evidence?.changedFromBefore, false);
176+
},
177+
);
166178

167179
test('diff snapshot -i after a full snapshot starts a baseline instead of listing the view change', async () => {
168180
const { device } = createDevice({ frame: screen(DIALOG, FULL) });

‎packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import type {
44
SnapshotPresentation,
55
SnapshotQualityVerdict,
66
} from '@agent-device/kernel/snapshot';
7-
import { DEFAULT_POST_ACTION_PRESENTATION } from '@agent-device/capture-kit/snapshot-baseline';
7+
import { postActionPresentation } from '@agent-device/capture-kit/snapshot-baseline';
88
import type { SnapshotState } from '@agent-device/contracts/snapshot-model';
99
import { isViewportRootNode } from '@agent-device/contracts/snapshot';
1010
import { snapshotProducerFacts } from '@agent-device/contracts/snapshot-producers';
@@ -159,12 +159,14 @@ export async function runStableCaptureLoop(
159159
runtime,
160160
options,
161161
deadlineMs - now(runtime),
162-
params.broadTransitionBaseline?.presentation ?? {
163-
...DEFAULT_POST_ACTION_PRESENTATION,
164-
depth: options.depth,
165-
scope: options.scope,
166-
raw: options.raw,
167-
},
162+
params.broadTransitionBaseline
163+
? postActionPresentation(params.broadTransitionBaseline)
164+
: {
165+
...postActionPresentation(undefined),
166+
depth: options.depth,
167+
scope: options.scope,
168+
raw: options.raw,
169+
},
168170
preferredBackend,
169171
);
170172
if (read.kind !== 'captured') return unfinished(read);

‎packages/contracts/src/interaction.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,7 @@ export type SurfaceScopedNodes = {
163163
presentedSurface?: PresentedSurface;
164164
/**
165165
* The presentation the capture read (`-i`, `-s`, `-d`, `--raw`, `--actions`). A post-action read
166-
* captures in it, so the diff compares one view (#4486). Absent when the producer recorded none.
166+
* captures in it, so the diff compares one view (#4486). Absent means the default full tree.
167167
*/
168168
presentation?: SnapshotPresentation;
169169
};

‎packages/kernel/src/snapshot.ts‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -789,18 +789,14 @@ export function buildSnapshotPresentationKey(flags: SnapshotOptions | undefined)
789789
/**
790790
* Whether two captures read one presentation, so a diff between them compares like with like
791791
* (#4486): a full tree holds nodes an interactive read omits, and a scoped, depth-limited, raw or
792-
* custom-action read holds a different node set than a plain one. A capture whose producer
793-
* recorded no presentation compares with any.
792+
* custom-action read holds a different node set than a plain one. An absent presentation is
793+
* the default full tree, as on SnapshotState.
794794
*/
795795
export function isSameSnapshotPresentation(
796796
a: SnapshotPresentation | undefined,
797797
b: SnapshotPresentation | undefined,
798798
): boolean {
799-
return (
800-
a === undefined ||
801-
b === undefined ||
802-
buildSnapshotPresentationKey(a) === buildSnapshotPresentationKey(b)
803-
);
799+
return buildSnapshotPresentationKey(a) === buildSnapshotPresentationKey(b);
804800
}
805801

806802
export function centerOfRect(rect: Rect): Point {

‎website/docs/docs/commands.md‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -499,7 +499,11 @@ agent-device gesture tap 80 300 200 450 --count 20 --pause-ms 100 # repeated two
499499
500500
If rotation stops repeated iOS presses, inspect the screen before continuing with the remaining presses.
501501
502-
A successful `click` or `press` means the action was sent to where the command found the target. It does not confirm that the element received it or that the screen changed. When the next step depends on it, add `--verify` (reports `changedFromBefore`), add `--settle` (returns the diff once the UI goes quiet), or follow with `wait <selector>`. `--settle` and `--verify` compare trees read in one view: the view of the refs you last read (`snapshot -i`, a full `snapshot`, or a scoped one), interactive when you have none. After a full `snapshot`, a settle reads the full tree and issues full-tree refs; after `snapshot -i`, it reads the interactive view. This holds for `press`, `click`, `fill`, `longpress`, `scroll` and `back`, on refs, selectors and coordinates. Before retrying a failed interaction, see [Retry after a failed command](#retry-after-a-failed-command).
502+
A successful `click` or `press` means the action was sent to where the command found the target. It does not confirm that the element received it or that the screen changed. When the next step depends on it, add `--verify` (reports `changedFromBefore`), add `--settle` (returns the diff once the UI goes quiet), or follow with `wait <selector>`. Before retrying a failed interaction, see [Retry after a failed command](#retry-after-a-failed-command).
503+
504+
`--settle` compares trees in the view of the refs you last read. After a full `snapshot`, it reads the full tree and issues full-tree refs; after `snapshot -i`, it reads the interactive view. It also preserves `-d`, `-s`, `--raw` and `--actions`. This holds for `press`, `click`, `fill`, `longpress`, `scroll` and `back`, including ref, selector and coordinate targets.
505+
506+
Standalone `--verify` captures in the pre-action tree's view, which can differ from the last ref frame when a selector resolves a fresh tree. With `--settle --verify`, verification reuses the settled capture and reports `changedFromBefore` only when the two views match or a known surface transition occurred. Without a populated baseline, post-action captures use the interactive view.
503507
504508
A `click` or `press` whose target the daemon resolves to a point (`@ref` or selector) re-checks that target's position before dispatching when the session's previous action animated the screen, and dispatches where it came to rest. When that check cannot confirm a stable position — the target was still moving, or a re-check could not measure it — the command refuses with `target_still_moving` and `dispatched: no`: nothing was dispatched, the point the command resolved is in `error.details.rect`, and where the target was last seen is in `error.details.movedToRect` when that last look measured one. Run `wait <selector>` on the target and retry the same command, or add `--settle` so the action waits for the UI to go quiet. A refused `@ref` press needs a fresh `snapshot -i` and a selector, because the wait retires the ref. Two dispatch paths hand off before the check: a press at raw coordinates has no resolved target to re-check, and a dispatch the backend performs on the element itself — `click @ref` on a backend with element dispatch, or a Maestro non-hittable coordinate fallback — goes to that backend rather than through a resolved point. The re-check stops when the request is canceled or reaches its deadline, and nothing is dispatched (`dispatched: no`): a canceled request fails with `details.reason: request_canceled`, and a request out of time with `details.reason: daemon_request_timeout`.
505509

0 commit comments

Comments
 (0)