Skip to content

Commit 3c785c3

Browse files
committed
refactor(interaction): hand find's leaf the baseline surfaceScopedNodes minted
PreresolvedInteractionTarget carries one SurfaceScopedNodes minted from find's own capture instead of mirroring nodes, surface and presentation field by field, so the adopting leaf's settle baseline is that value as-is. PresentedSurface becomes a plain union, and the fact travels only inside SnapshotState: the top-level BackendSnapshotResult field had no production writer. Moves the moving-target gate onto the capture-kit baseline mint.
1 parent a41dbf8 commit 3c785c3

18 files changed

Lines changed: 72 additions & 121 deletions

File tree

‎apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/PresentedSurface.swift‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,7 @@ import Foundation
55
/// confirmation on it: a capture whose evidence cannot answer says `unknown` instead of staying silent,
66
/// since silence is also what a producer that never classified publishes.
77
///
8-
/// Mirrors the TypeScript `PresentedSurface` in `@agent-device/kernel/snapshot`. The rule that arrives at
9-
/// one of these three values is `SnapshotPresentedSurface` beside this type.
8+
/// Mirrors the TypeScript `PresentedSurface` in `@agent-device/kernel/snapshot`.
109
public enum PresentedSurface: String, Codable, Equatable {
1110
/// A presentation covers the content this capture describes.
1211
case presented

‎packages/capture-kit/src/snapshot-baseline.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,11 @@ import type { SurfaceScopedNodes } from '@agent-device/contracts/interaction';
1414
* One value, never two channels: every boundary that carries a baseline carries the type this returns,
1515
* so nodes cannot arrive without the surface they describe.
1616
*/
17-
export function surfaceScopedNodes(snapshot: SnapshotState | undefined): SurfaceScopedNodes {
17+
export function surfaceScopedNodes(
18+
snapshot:
19+
| Pick<SnapshotState, 'nodes' | 'iosSystemSurfaceBundleId' | 'presentedSurface'>
20+
| undefined,
21+
): SurfaceScopedNodes {
1822
return {
1923
nodes: snapshot?.nodes ?? [],
2024
...(snapshot?.iosSystemSurfaceBundleId

‎packages/contracts/src/backend.ts‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ import type {
2323
import type {
2424
Point,
2525
Rect,
26-
PresentedSurface,
2726
SnapshotKeyboardBandFact,
2827
SnapshotNode,
2928
SnapshotOptions,
@@ -59,12 +58,6 @@ export type BackendSnapshotResult = {
5958
* the band from `nodes`.
6059
*/
6160
keyboard?: SnapshotKeyboardBandFact;
62-
/**
63-
* iOS: whether this capture read a modal presentation over the app (#3565), classified by the
64-
* producer on its acquired tree. A backend that publishes nothing classified nothing; a consumer
65-
* treats that as `'unknown'`, never as absence.
66-
*/
67-
presentedSurface?: PresentedSurface;
6861
/**
6962
* The box these rects are measured in, as the producer measured it (#3182). Absent means the
7063
* producer measured no box — never a zero — since that absence is the only answer a producer with

‎packages/contracts/src/interaction.ts‎

Lines changed: 5 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -124,21 +124,14 @@ export type PreresolvedInteractionTarget = {
124124
ref: string;
125125
/** The node the caller resolved, after the caller's own promotion. */
126126
node: SnapshotNode;
127-
/** The tree `node` came from — the guards read its siblings for occlusion/viewport. */
128-
nodes: SnapshotNode[];
129-
/**
130-
* The in-place iOS system surface `nodes` describes (#2438), absent for ordinary app content.
131-
* Travels with the tree so the adopting consumer's post-action comparison knows which surface
132-
* its baseline came from.
133-
*/
134-
iosSystemSurfaceBundleId?: string;
135127
/**
136-
* Whether the producer that captured `nodes` read a modal presentation over the app (#3565),
137-
* carried so the adopting leaf's settle baseline keeps the fact its own capture established.
128+
* The tree `node` came from, minted by `surfaceScopedNodes` from the caller's capture. The guards
129+
* read its siblings for occlusion/viewport, and the adopting leaf's post-action baseline is this
130+
* value as-is, so the surface (#2438) and presentation (#3565) facts travel with the nodes.
138131
*/
139-
presentedSurface?: PresentedSurface;
132+
tree: SurfaceScopedNodes;
140133
/**
141-
* The keyboard band the capture of `nodes` measured, carried so the adopting consumer's keyboard
134+
* The keyboard band the capture of `tree` measured, carried so the adopting consumer's keyboard
142135
* guard reads the same band the tree was captured with instead of re-deriving one (#2660).
143136
*/
144137
keyboard?: SnapshotKeyboardBandFact;

‎packages/daemon/src/execution/interaction/internal/__tests__/find.test.ts‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,33 @@ test('handleFindCommands click prefers on-screen duplicate text matches', async
161161
expect(invokeCalls[0]!.positionals?.[0]).toBe('@e3');
162162
});
163163

164+
// #3565: the leaf adopts find's tree as its settle baseline, so the presentation find's capture
165+
// read must arrive on that tree rather than be re-derived from the nodes.
166+
test('handleFindCommands click hands the leaf the baseline its own capture minted', async () => {
167+
const nodes = [
168+
{ index: 0, ref: 'e1', type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } },
169+
{
170+
index: 1,
171+
ref: 'e2',
172+
type: 'Button',
173+
label: 'Close',
174+
hittable: true,
175+
rect: { x: 20, y: 80, width: 80, height: 44 },
176+
parentIndex: 0,
177+
},
178+
];
179+
mockDispatch.mockImplementation(async (_device, command) =>
180+
command === 'snapshot' ? { nodes, presentedSurface: 'presented' } : {},
181+
);
182+
183+
const { response, invokeCalls } = await runFindClickScenario({ positionals: ['Close', 'click'] });
184+
185+
expect(response.ok).toBe(true);
186+
const carried = invokeCalls[0]!.internal?.findResolvedTarget;
187+
expect(carried?.tree.presentedSurface).toBe('presented');
188+
expect(carried?.tree.nodes).toContain(carried?.node);
189+
});
190+
164191
test('handleFindCommands click tries query-scoped full retry before failing sparse verdict', async () => {
165192
const session = makeSession('default');
166193
session.snapshot = {

‎packages/daemon/src/execution/interaction/internal/__tests__/interaction-touch-fixtures.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ export function findResolvedTarget(
140140
preresolved: ReturnType<typeof makeFindPreresolvedTree>,
141141
ref = '@e2',
142142
) {
143-
return { ref, node: preresolved.node, nodes: preresolved.nodes };
143+
return { ref, node: preresolved.node, tree: { nodes: preresolved.nodes } };
144144
}
145145

146146
/** The coordinates the leaf handed to the bound point-tap operation. */

‎packages/daemon/src/execution/interaction/internal/find.ts‎

Lines changed: 9 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
1-
import type { PreresolvedInteractionTarget } from '@agent-device/contracts/interaction';
1+
import type {
2+
PreresolvedInteractionTarget,
3+
SurfaceScopedNodes,
4+
} from '@agent-device/contracts/interaction';
5+
import { surfaceScopedNodes } from '@agent-device/capture-kit/snapshot-baseline';
26
import {
37
isReadOnlyFindAction,
48
checkFindArgs,
@@ -10,7 +14,6 @@ import { SELECTOR_PIPELINE_POLICIES } from '@agent-device/selectors/selector-pip
1014
import { coveredInteractionDetails } from '@agent-device/selectors/interaction-error';
1115
import {
1216
centerOfRect,
13-
type PresentedSurface,
1417
type SnapshotKeyboardBandFact,
1518
type SnapshotState,
1619
} from '@agent-device/kernel/snapshot';
@@ -65,13 +68,10 @@ type ResolvedMatch = {
6568
node: SnapshotState['nodes'][number];
6669
resolvedNode: SnapshotState['nodes'][number];
6770
ref: string;
68-
nodes: SnapshotState['nodes'];
69-
/** The in-place iOS system surface the target capture described (#2438), if any. */
70-
iosSystemSurfaceBundleId?: string;
71+
/** The target capture's baseline: its nodes with the surface and presentation they describe. */
72+
tree: SurfaceScopedNodes;
7173
/** The keyboard band that capture's producer measured, when it measured one (#2660). */
7274
keyboard?: SnapshotKeyboardBandFact;
73-
/** Whether that capture read a modal presentation over the app (#3565). */
74-
presentedSurface?: PresentedSurface;
7575
actionFlags: Record<string, unknown>;
7676
/**
7777
* Set when find's row refuses this match as covered. Only the focus/type
@@ -211,14 +211,8 @@ export async function handleFindCommands(params: FindRouteInput): Promise<Daemon
211211
node,
212212
resolvedNode,
213213
ref,
214-
nodes,
215-
...(snapshotResult.iosSystemSurfaceBundleId
216-
? { iosSystemSurfaceBundleId: snapshotResult.iosSystemSurfaceBundleId }
217-
: {}),
214+
tree: surfaceScopedNodes(snapshotResult),
218215
...(snapshotResult.keyboard ? { keyboard: snapshotResult.keyboard } : {}),
219-
...(snapshotResult.presentedSurface
220-
? { presentedSurface: snapshotResult.presentedSurface }
221-
: {}),
222216
actionFlags,
223217
...(target.kind === 'occluded' ? { occludedNode: target.node } : {}),
224218
};
@@ -273,17 +267,9 @@ function preresolvedTarget(match: ResolvedMatch): PreresolvedInteractionTarget {
273267
return {
274268
ref: match.ref,
275269
node: match.resolvedNode,
276-
nodes: match.nodes,
277-
// #2438: the leaf's post-action verify/settle compares against this tree, so it must know
278-
// whether the tree describes the app or an in-place system surface served over it.
279-
...(match.iosSystemSurfaceBundleId
280-
? { iosSystemSurfaceBundleId: match.iosSystemSurfaceBundleId }
281-
: {}),
270+
tree: match.tree,
282271
// #2660: the leaf's keyboard guard measures the band this capture measured, not one it re-derives.
283272
...(match.keyboard ? { keyboard: match.keyboard } : {}),
284-
// #3565: the leaf's settle baseline is armed on the presentation this capture read, not a shape
285-
// the leaf reconstructs from nodes it cannot see the projection of.
286-
...(match.presentedSurface ? { presentedSurface: match.presentedSurface } : {}),
287273
};
288274
}
289275

‎packages/kernel/src/presented-surface.test.ts‎

Lines changed: 3 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import assert from 'node:assert/strict';
22
import { test } from 'vitest';
3-
import { PRESENTED_SURFACE_STATES } from './snapshot.ts';
3+
import type { PresentedSurface } from './snapshot.ts';
44
import { readPresentedSurfaceFact } from './record.ts';
55

66
// The presented-surface fact (#3565) arrives from two producers — the Apple runner's capture payload
@@ -50,7 +50,8 @@ test('a payload with no own fact key is silent rather than inherited', () => {
5050
});
5151

5252
test('each declared state reads back as itself', () => {
53-
for (const state of PRESENTED_SURFACE_STATES) {
53+
const states: PresentedSurface[] = ['presented', 'absent', 'unknown'];
54+
for (const state of states) {
5455
assert.equal(readPresentedSurfaceFact(carrying(state)), state);
5556
}
5657
});
@@ -75,25 +76,3 @@ test('a published value naming an inherited object member is not a state', () =>
7576
assert.equal(readPresentedSurfaceFact(carrying(payload)), 'unknown', payload);
7677
}
7778
});
78-
79-
// The reader decides from a table it owns, and the exported vocabulary is frozen rather than
80-
// `as const` alone: a consumer that mutated the array could otherwise move the boundary between a
81-
// trusted state and `unknown` for every other module in the process.
82-
test('the exported vocabulary cannot be widened or narrowed at runtime', () => {
83-
assert.ok(Object.isFrozen(PRESENTED_SURFACE_STATES));
84-
85-
for (const mutate of [
86-
() => (PRESENTED_SURFACE_STATES as unknown as string[]).push('not-presented'),
87-
() => ((PRESENTED_SURFACE_STATES as unknown as string[])[0] = 'absent'),
88-
() => (PRESENTED_SURFACE_STATES as unknown as string[]).pop(),
89-
]) {
90-
try {
91-
mutate();
92-
} catch {
93-
// A strict-mode assignment or a frozen push throws; either way the array is unchanged.
94-
}
95-
}
96-
97-
assert.deepEqual([...PRESENTED_SURFACE_STATES], ['presented', 'absent', 'unknown']);
98-
assert.equal(readPresentedSurfaceFact(carrying('not-presented')), 'unknown');
99-
});

‎packages/kernel/src/record.ts‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -162,12 +162,6 @@ function readKeyboardFrameFact(
162162
* The declared states, each keyed by its own name so a payload string is a state only when this table
163163
* has that exact key. Keyed by `PresentedSurface`, so a state added to the vocabulary without a row
164164
* here fails to compile instead of silently reading as `unknown`.
165-
*
166-
* A local table rather than `PRESENTED_SURFACE_STATES`: this module is the shared wire reader every
167-
* response seam imports, and a load-time edge to `kernel/snapshot.ts` from here makes that module
168-
* eager for every consumer of `kernel/record` — which ADR 0019's loading-shape probe measures per
169-
* entry (`scripts/__tests__/eager-closure-budgets.test.ts`). The type is the tie that keeps the two
170-
* in agreement, and `presented-surface.test.ts` reads both against the same payloads.
171165
*/
172166
const PRESENTED_SURFACE_BY_NAME: Readonly<Record<PresentedSurface, PresentedSurface>> = {
173167
presented: 'presented',

‎packages/kernel/src/snapshot.ts‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -399,9 +399,7 @@ export type SnapshotKeyboardBandFact =
399399
* a tree from a producer that reports no UIKit class facts, an unexplained extra container, a
400400
* truncated capture — answers `'unknown'`, never `'absent'`.
401401
*/
402-
export const PRESENTED_SURFACE_STATES = Object.freeze(['presented', 'absent', 'unknown'] as const);
403-
404-
export type PresentedSurface = (typeof PRESENTED_SURFACE_STATES)[number];
402+
export type PresentedSurface = 'presented' | 'absent' | 'unknown';
405403

406404
/**
407405
* The box a capture's node rects are measured in, published beside them so a reader never has to

0 commit comments

Comments
 (0)