Skip to content

Commit 1f15eb0

Browse files
thymikeeclaude
andcommitted
fix(settle): compare post-action trees in one view across every route
A settle diff compared whatever pre-action tree it found with an interactive settle capture, so a full snapshot, a selector fallback or a full tree a selector read stored showed up as false removals (#4486). One rule now owns every post-action comparison: two trees are compared only when they were read in the same presentation. capture-kit's postActionBaseline picks the baseline in the agent's view, the presentation of the ref frame it last read, and SurfaceScopedNodes carries that presentation. The settle loop captures in it for press, click, fill, longpress, scroll and back, on refs, selectors and coordinates. --verify captures in the pre-action tree's view, and diff snapshot starts a new baseline when the stored tree was read in another view. Fixes #4486 Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 776276e commit 1f15eb0

14 files changed

Lines changed: 355 additions & 202 deletions

File tree

‎packages/capture-kit/src/snapshot-baseline.test.ts‎

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { expect, test } from 'vitest';
22
import type { SnapshotState } from '@agent-device/contracts/snapshot-model';
3-
import { surfaceScopedNodes } from './snapshot-baseline.ts';
3+
import type { SurfaceScopedNodes } from '@agent-device/contracts/interaction';
4+
import { postActionBaseline, surfaceScopedNodes } from './snapshot-baseline.ts';
45

56
const nodes = [{ index: 0, type: 'Application', ref: 'e1' }];
67

@@ -33,6 +34,54 @@ test('a session with no stored snapshot still mints an empty baseline', () => {
3334
expect(surfaceScopedNodes(undefined)).toEqual({ nodes: [] });
3435
});
3536

37+
test('the mint carries the presentation the capture read', () => {
38+
expect(surfaceScopedNodes(snapshot({ presentation: { interactiveOnly: true } }))).toEqual({
39+
nodes,
40+
presentation: { interactiveOnly: true },
41+
});
42+
});
43+
44+
const INTERACTIVE = { interactiveOnly: true };
45+
const FULL = {};
46+
const frame = (presentation: SurfaceScopedNodes['presentation']) => tree('frame', presentation);
47+
48+
// #4486: a settle diff compares one view, and the agent's view is its ref frame's presentation.
49+
test.each([
50+
['interactive frame, interactive candidate', INTERACTIVE, INTERACTIVE, 'candidate'],
51+
['interactive frame, full candidate', INTERACTIVE, FULL, 'frame'],
52+
['full frame, interactive candidate', FULL, INTERACTIVE, 'frame'],
53+
['full frame, full candidate', FULL, FULL, 'candidate'],
54+
[
55+
'interactive frame, scoped interactive candidate',
56+
INTERACTIVE,
57+
{ interactiveOnly: true, scope: 'Composer' },
58+
'frame',
59+
],
60+
['interactive frame, candidate with no recorded view', INTERACTIVE, undefined, 'candidate'],
61+
] as const)('%s: the %s answers', (_case, framePresentation, candidatePresentation, expected) => {
62+
const baseline = postActionBaseline({
63+
frame: frame(framePresentation),
64+
candidates: [tree('candidate', candidatePresentation)],
65+
});
66+
expect(baseline.nodes[0]?.label).toBe(expected);
67+
});
68+
69+
test('without a frame the first non-empty candidate answers, in its own view', () => {
70+
const baseline = postActionBaseline({
71+
frame: undefined,
72+
candidates: [undefined, { nodes: [] }, tree('candidate', FULL)],
73+
});
74+
expect(baseline).toEqual(tree('candidate', FULL));
75+
expect(postActionBaseline({ frame: undefined, candidates: [] })).toEqual({ nodes: [] });
76+
});
77+
78+
function tree(label: string, presentation: SurfaceScopedNodes['presentation']): SurfaceScopedNodes {
79+
return {
80+
nodes: [{ index: 0, type: 'Button', label, ref: 'e1' }],
81+
...(presentation ? { presentation } : {}),
82+
};
83+
}
84+
3685
function snapshot(fields: Partial<SnapshotState>): SnapshotState {
3786
return { nodes, createdAt: 0, presentationKey: '', ...fields } as SnapshotState;
3887
}

‎packages/capture-kit/src/snapshot-baseline.ts‎

Lines changed: 42 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,17 @@
11
import type { SnapshotState } from '@agent-device/contracts/snapshot-model';
22
import type { SurfaceScopedNodes } from '@agent-device/contracts/interaction';
3+
import {
4+
isSameSnapshotPresentation,
5+
type SnapshotPresentation,
6+
} from '@agent-device/kernel/snapshot';
7+
8+
/** The view a post-action read captures in when its baseline recorded none. */
9+
export const DEFAULT_POST_ACTION_PRESENTATION: SnapshotPresentation = { interactiveOnly: true };
310

411
/**
512
* Mints the one baseline value a post-action comparison carries: the captured nodes together with the
6-
* surface they describe and the presentation the capture read (#2438, #3565).
13+
* surface they describe, the presentation the capture read (#2438, #3565), and the view it was read
14+
* in (#4486).
715
*
816
* Both settle routes take their baseline from here — the element-targeted interaction route and the
917
* generic daemon route (`scroll`/`back --settle`) — so a fact a capture established cannot be carried
@@ -16,7 +24,10 @@ import type { SurfaceScopedNodes } from '@agent-device/contracts/interaction';
1624
*/
1725
export function surfaceScopedNodes(
1826
snapshot:
19-
| Pick<SnapshotState, 'nodes' | 'iosSystemSurfaceBundleId' | 'presentedSurface'>
27+
| Pick<
28+
SnapshotState,
29+
'nodes' | 'iosSystemSurfaceBundleId' | 'presentedSurface' | 'presentation'
30+
>
2031
| undefined,
2132
): SurfaceScopedNodes {
2233
return {
@@ -25,5 +36,34 @@ export function surfaceScopedNodes(
2536
? { surfaceBundleId: snapshot.iosSystemSurfaceBundleId }
2637
: {}),
2738
...(snapshot?.presentedSurface ? { presentedSurface: snapshot.presentedSurface } : {}),
39+
...(snapshot?.presentation ? { presentation: snapshot.presentation } : {}),
2840
};
2941
}
42+
43+
/**
44+
* The pre-action tree a settle diff runs against, and through its `presentation` the view the settle
45+
* captures in (#4486). Two trees compare only within one presentation, and the agent's view is the
46+
* presentation of the ref frame it last read: `snapshot -i`, a full `snapshot`, a scoped one, or the
47+
* refs a settle issued. So the first candidate read in that view answers. A candidate read in another
48+
* view, such as a selector resolution that fell back to the full tree or a full tree a selector read
49+
* stored, is skipped, and the frame itself answers when no candidate qualifies. Without a frame, the
50+
* first candidate answers.
51+
*
52+
* Every settle route picks its baseline here, so none can compare an interactive read with a full one.
53+
*/
54+
export function postActionBaseline(params: {
55+
/** The ref frame, as the caller admits it as a baseline. */
56+
frame: SurfaceScopedNodes | undefined;
57+
/** Pre-action trees in the caller's order of preference. */
58+
candidates: readonly (SurfaceScopedNodes | undefined)[];
59+
}): SurfaceScopedNodes {
60+
const { frame } = params;
61+
const view = frame && frame.nodes.length > 0 ? frame.presentation : undefined;
62+
const candidate = params.candidates.find(
63+
(baseline) =>
64+
baseline !== undefined &&
65+
baseline.nodes.length > 0 &&
66+
isSameSnapshotPresentation(baseline.presentation, view),
67+
);
68+
return candidate ?? frame ?? { nodes: [] };
69+
}

‎packages/command-runtime/src/commands/capture/runtime/snapshot.ts‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import {
88
type SnapshotDiagnosticsSummary,
99
} from '@agent-device/contracts/capture';
1010
import { AppError } from '@agent-device/kernel/errors';
11+
import { isSameSnapshotPresentation } from '@agent-device/kernel/snapshot';
1112
import { normalizeType } from '@agent-device/contracts/snapshot';
1213
import { snapshotProducerFacts } from '@agent-device/contracts/snapshot-producers';
1314
import type {
@@ -135,7 +136,13 @@ export const diffSnapshotCommand: RuntimeCommand<
135136
}
136137
const capture = await captureRuntimeSnapshot(runtime, options);
137138
const flattenForDiff = options.interactiveOnly === true;
138-
const previousSnapshot = capture.session?.snapshot;
139+
// A diff compares one view: the stored tree, else the ref frame, read in the requested
140+
// presentation. With neither, the capture initializes the baseline (#4486).
141+
const previousSnapshot = [capture.session?.snapshot, capture.session?.refFrameSnapshot].find(
142+
(stored) =>
143+
stored !== undefined &&
144+
isSameSnapshotPresentation(stored.presentation, capture.snapshot.presentation),
145+
);
139146
const nextSession = nextSnapshotSession(options.session, capture);
140147

141148
if (!previousSnapshot) {

‎packages/command-runtime/src/commands/interaction/runtime/interaction-snapshot-capture.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,7 @@ import { toBackendContext } from '../../runtime-common.ts';
66
/**
77
* A leaf shared by every interaction runtime module that needs a fresh (or session-cached) tree:
88
* `resolution.ts`'s ref/selector resolution, `selector-target-resolution.ts`'s acting selector lookup,
9-
* `moving-target-gate.ts`'s re-observation, `gestures.ts`'s viewport resolution, and
10-
* `post-action-observation.ts`'s `--verify` baseline.
9+
* `moving-target-gate.ts`'s re-observation, and `gestures.ts`'s viewport resolution.
1110
* Deliberately self-contained — it imports no sibling of this directory, so those siblings can
1211
* import it without ever risking a cycle back.
1312
*/

‎packages/command-runtime/src/commands/interaction/runtime/post-action-observation.ts‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,13 @@ import type {
55
SettleObservation,
66
SettleParams,
77
} from '@agent-device/contracts/interaction';
8-
import { captureInteractionSnapshot } from './interaction-snapshot-capture.ts';
8+
import { isSameSnapshotPresentation } from '@agent-device/kernel/snapshot';
99
import { summarizePostActionEvidence } from './post-action-surface.ts';
10-
import { surfaceScopedNodes } from '@agent-device/capture-kit/snapshot-baseline';
10+
import {
11+
DEFAULT_POST_ACTION_PRESENTATION,
12+
surfaceScopedNodes,
13+
} from '@agent-device/capture-kit/snapshot-baseline';
14+
import { captureSelectorSnapshot } from './selector-read-shared.ts';
1115
import { settleAfterInteraction, settleEvidence } from './settle.ts';
1216

1317
type ObservedResult<T extends object> = T & {
@@ -81,9 +85,17 @@ async function observeAfterInteraction(
8185
...params.settle,
8286
resolved,
8387
});
84-
const evidence = params.verify
85-
? settleEvidence(outcome.settledCapture, resolved.preAction)
86-
: undefined;
88+
// The evidence compares one view: a pre-action tree read in another view than the settle
89+
// capture makes no changedFromBefore claim (#4486).
90+
const baseline =
91+
outcome.settledCapture &&
92+
isSameSnapshotPresentation(
93+
resolved.preAction?.presentation,
94+
outcome.settledCapture.presentation,
95+
)
96+
? resolved.preAction
97+
: undefined;
98+
const evidence = params.verify ? settleEvidence(outcome.settledCapture, baseline) : undefined;
8799
return { settle: outcome.observation, ...(evidence ? { evidence } : {}) };
88100
}
89101
if (!params.verify) return {};
@@ -92,8 +104,8 @@ async function observeAfterInteraction(
92104
}
93105

94106
/**
95-
* Post-action side of `--verify` (#1047): one interactive-only capture through
96-
* the same capture helper the resolution path already uses, digested and then
107+
* Post-action side of `--verify` (#1047): one capture in the pre-action tree's presentation
108+
* (interactive-only by default), so the comparison reads one view (#4486), digested and then
97109
* discarded. The node tree itself is never attached to the result, only the
98110
* cheap summary. A missing baseline yields no `changedFromBefore` claim —
99111
* no baseline, no comparison.
@@ -104,7 +116,11 @@ async function captureVerifyEvidence(
104116
resolved: ResolvedInteractionTarget,
105117
): Promise<InteractionEvidence | undefined> {
106118
try {
107-
const capture = await captureInteractionSnapshot(runtime, options, true);
119+
const capture = await captureSelectorSnapshot(runtime, options, {
120+
updateSession: true,
121+
includeRects: true,
122+
presentation: resolved.preAction?.presentation ?? DEFAULT_POST_ACTION_PRESENTATION,
123+
});
108124
return summarizePostActionEvidence(surfaceScopedNodes(capture.snapshot), resolved.preAction);
109125
} catch {
110126
return undefined;

‎packages/command-runtime/src/commands/interaction/runtime/selector-read-shared.ts‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import {
1515
type SnapshotCommandOptionFields,
1616
type SnapshotNode,
1717
type SnapshotPreferredBackend,
18+
type SnapshotPresentation,
1819
} from '@agent-device/kernel/snapshot';
1920
import type { SnapshotState } from '@agent-device/contracts/snapshot-model';
2021
import {
@@ -72,6 +73,11 @@ export async function captureSelectorSnapshot(
7273
interactiveOnly?: boolean;
7374
includeHiddenContentHints?: boolean;
7475
preferredBackend?: SnapshotPreferredBackend;
76+
/**
77+
* The whole presentation to capture in, overriding `interactiveOnly`, `scope` and the command's
78+
* depth and raw flags: a post-action read reproduces its baseline's view (#4486).
79+
*/
80+
presentation?: SnapshotPresentation;
7581
} = {
7682
updateSession: true,
7783
},
@@ -82,11 +88,22 @@ export async function captureSelectorSnapshot(
8288
}
8389
const sessionName = options.session ?? 'default';
8490
const session = await runtime.sessions.get(sessionName);
91+
const view = captureOptions.presentation;
8592
const result = await captureSnapshot(toBackendContext(runtime, options), {
86-
interactiveOnly: captureOptions.interactiveOnly ?? false,
87-
depth: options.depth,
88-
scope: captureOptions.scope ?? options.scope,
89-
raw: options.raw,
93+
...(view
94+
? {
95+
interactiveOnly: view.interactiveOnly === true,
96+
depth: view.depth,
97+
scope: view.scope,
98+
raw: view.raw,
99+
...(view.customActions === true ? { customActions: true } : {}),
100+
}
101+
: {
102+
interactiveOnly: captureOptions.interactiveOnly ?? false,
103+
depth: options.depth,
104+
scope: captureOptions.scope ?? options.scope,
105+
raw: options.raw,
106+
}),
90107
includeRects: captureOptions.includeRects,
91108
...(captureOptions.preferredBackend
92109
? { preferredBackend: captureOptions.preferredBackend }

0 commit comments

Comments
 (0)