Skip to content

Commit 9571d9c

Browse files
thymikeethymikee
committed
feat(interaction): name the covering element in covered refusals
The occlusion scan already finds which node draws on top of a covered target but published only the boolean, so a target_covered refusal gave the agent no hint about what to dismiss. Carry a bounded, non-actionable description of the cover (presented role, clipped label, rect) on the annotated node as `coveredBy`, surface it as a typed detail and in the hint of both refusal producers, and compare it in snapshot unchanged-metadata. Backend-policy covers (Android replacement surfaces) report only the boolean and refuse without a guessed culprit. Closes #3488
1 parent 4edab40 commit 9571d9c

11 files changed

Lines changed: 315 additions & 13 deletions

File tree

‎packages/capture-kit/src/snapshot-occlusion.test.ts‎

Lines changed: 71 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,11 @@
11
import { test } from 'vitest';
22
import assert from 'node:assert/strict';
33
import type { RawSnapshotNode } from '@agent-device/kernel/snapshot';
4-
import { annotateCoveredSnapshotNodes } from './snapshot-occlusion.ts';
4+
import { SNAPSHOT_COVERING_LABEL_MAX_LENGTH } from '@agent-device/kernel/snapshot';
5+
import {
6+
annotateCoveredSnapshotNodes,
7+
annotateSnapshotNodesCoveredByPolicy,
8+
} from './snapshot-occlusion.ts';
59

610
// #1478 P5 codec-extraction regression: `annotateCoveredSnapshotNodes` marks
711
// a touch candidate `interactionBlocked: 'covered'` when a later, floating
@@ -93,6 +97,72 @@ test('annotateCoveredSnapshotNodes still marks a touch target covered by a later
9397
assert.equal(result[0]?.hittable, false);
9498
});
9599

100+
test('annotateCoveredSnapshotNodes carries a bounded description of the covering node', () => {
101+
const target: RawSnapshotNode = {
102+
index: 0,
103+
type: 'button',
104+
role: 'button',
105+
hittable: true,
106+
label: 'Save',
107+
rect: { x: 10, y: 10, width: 100, height: 40 },
108+
};
109+
const overlay: RawSnapshotNode = {
110+
index: 1,
111+
type: 'XCUIElementTypeDialog',
112+
label: ' Cookie banner ',
113+
rect: { x: 0, y: 0, width: 200, height: 200 },
114+
};
115+
116+
const result = annotateCoveredSnapshotNodes([target, overlay]);
117+
118+
// The description is the presented role + a trimmed label + the cover's rect, and it carries no
119+
// ref: naming the culprit is diagnosis, never an actionable re-target.
120+
assert.deepEqual(result[0]?.coveredBy, {
121+
kind: 'dialog',
122+
label: 'Cookie banner',
123+
rect: { x: 0, y: 0, width: 200, height: 200 },
124+
});
125+
});
126+
127+
test('annotateCoveredSnapshotNodes clips a covering label to the bounded length', () => {
128+
const target: RawSnapshotNode = {
129+
index: 0,
130+
type: 'button',
131+
role: 'button',
132+
hittable: true,
133+
label: 'Save',
134+
rect: { x: 10, y: 10, width: 100, height: 40 },
135+
};
136+
const overlay: RawSnapshotNode = {
137+
index: 1,
138+
type: 'dialog',
139+
label: 'x'.repeat(SNAPSHOT_COVERING_LABEL_MAX_LENGTH + 40),
140+
rect: { x: 0, y: 0, width: 200, height: 200 },
141+
};
142+
143+
const result = annotateCoveredSnapshotNodes([target, overlay]);
144+
145+
assert.equal(result[0]?.coveredBy?.label?.length, SNAPSHOT_COVERING_LABEL_MAX_LENGTH);
146+
});
147+
148+
test('annotateSnapshotNodesCoveredByPolicy refuses without guessing a cover', () => {
149+
// The Android replacement-surface lane reports only the covered boolean; the refusal must not
150+
// invent a culprit it never observed.
151+
const target: RawSnapshotNode = {
152+
index: 0,
153+
type: 'button',
154+
role: 'button',
155+
hittable: true,
156+
label: 'Save',
157+
rect: { x: 10, y: 10, width: 100, height: 40 },
158+
};
159+
160+
const result = annotateSnapshotNodesCoveredByPolicy([target], (node) => node.index === 0);
161+
162+
assert.equal(result[0]?.interactionBlocked, 'covered');
163+
assert.equal(result[0]?.coveredBy, undefined);
164+
});
165+
96166
test('annotateCoveredSnapshotNodes leaves an uncovered touch target unchanged', () => {
97167
const target: RawSnapshotNode = {
98168
index: 0,

‎packages/capture-kit/src/snapshot-occlusion.ts‎

Lines changed: 29 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
1-
import type { RawSnapshotNode, Rect } from '@agent-device/kernel/snapshot';
2-
import { centerOfRect } from '@agent-device/kernel/snapshot';
1+
import type { RawSnapshotNode, Rect, SnapshotCoveringNode } from '@agent-device/kernel/snapshot';
2+
import {
3+
centerOfRect,
4+
formatRole,
5+
SNAPSHOT_COVERING_LABEL_MAX_LENGTH,
6+
} from '@agent-device/kernel/snapshot';
37
import { areRectsApproximatelyEqual, normalizeRect } from '@agent-device/kernel/rect-center';
48
import { containsPoint, rectContains } from '@agent-device/kernel/rect';
59
import {
@@ -113,38 +117,55 @@ export function annotateCoveredSnapshotNodes(
113117
),
114118
coverCache: new Map(),
115119
};
116-
const coveredPositions: number[] = [];
120+
const coveredPositions: Array<{ position: number; coveredBy?: SnapshotCoveringNode }> = [];
117121
for (const [position, node] of nodes.entries()) {
118122
if (!isCandidateTouchNode(node)) continue;
119-
if (findCoveringNode(scan, position, node, options)) coveredPositions.push(position);
123+
const coveringNode = findCoveringNode(scan, position, node, options);
124+
if (coveringNode)
125+
coveredPositions.push({ position, coveredBy: describeCoveringNode(coveringNode) });
120126
}
121127
return annotateCoveredPositions(nodes, coveredPositions);
122128
}
123129

130+
/**
131+
* The bounded, non-actionable description of the cover the scan found. No ref is ever carried — see
132+
* {@link SnapshotCoveringNode}.
133+
*/
134+
function describeCoveringNode(node: RawSnapshotNode): SnapshotCoveringNode {
135+
const label = node.label?.trim();
136+
const rect = positiveRect(node.rect);
137+
return {
138+
kind: formatRole(node.type ?? 'Element'),
139+
...(label ? { label: label.slice(0, SNAPSHOT_COVERING_LABEL_MAX_LENGTH) } : {}),
140+
...(rect ? { rect } : {}),
141+
};
142+
}
143+
124144
/** Applies backend-owned covered decisions through the same publication annotation contract. */
125145
export function annotateSnapshotNodesCoveredByPolicy(
126146
nodes: RawSnapshotNode[],
127147
isCovered: (node: RawSnapshotNode) => boolean,
128148
): RawSnapshotNode[] {
129-
const coveredPositions: number[] = [];
149+
const coveredPositions: Array<{ position: number; coveredBy?: undefined }> = [];
130150
for (const [position, node] of nodes.entries()) {
131-
if (isCandidateTouchNode(node) && isCovered(node)) coveredPositions.push(position);
151+
if (isCandidateTouchNode(node) && isCovered(node)) coveredPositions.push({ position });
132152
}
133153
return annotateCoveredPositions(nodes, coveredPositions);
134154
}
135155

136156
function annotateCoveredPositions(
137157
nodes: RawSnapshotNode[],
138-
coveredPositions: readonly number[],
158+
coveredPositions: ReadonlyArray<{ position: number; coveredBy?: SnapshotCoveringNode }>,
139159
): RawSnapshotNode[] {
140160
if (coveredPositions.length === 0) return nodes;
141161
const annotated = [...nodes];
142-
for (const position of coveredPositions) {
162+
for (const { position, coveredBy } of coveredPositions) {
143163
const node = nodes[position]!;
144164
annotated[position] = {
145165
...node,
146166
hittable: false,
147167
interactionBlocked: 'covered' as const,
168+
...(coveredBy ? { coveredBy } : {}),
148169
presentationHints: mergeCoveredHint(node.presentationHints),
149170
};
150171
}

‎packages/kernel/src/snapshot.ts‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,50 @@ export type SnapshotPresentationFlagInput = SnapshotOptionFlagFields<
246246
(typeof SNAPSHOT_PRESENTATION_OPTION_KEYS)[number]
247247
>;
248248

249+
/**
250+
* The bounded, non-actionable description of the element that occludes a covered node. It carries
251+
* only what the occlusion scan observed — presented role, an optional clipped accessible name, and
252+
* the cover's rect — and deliberately no ref: the cover may be a container the snapshot never
253+
* issues refs for, and naming it is diagnosis, not an invitation to target it.
254+
*/
255+
export type SnapshotCoveringNode = {
256+
/** The cover's presented role, from the same table the text presenter reads (`formatRole`). */
257+
kind: string;
258+
/** The cover's accessible name, trimmed and clipped to {@link SNAPSHOT_COVERING_LABEL_MAX_LENGTH}. */
259+
label?: string;
260+
/** The cover's positive rect as the scan measured it. */
261+
rect?: Rect;
262+
};
263+
264+
/** Upper bound for a covering-node label carried in diagnostics; longer names are clipped. */
265+
export const SNAPSHOT_COVERING_LABEL_MAX_LENGTH = 80;
266+
267+
/**
268+
* One-line human rendering of a {@link SnapshotCoveringNode} for refusal hints, e.g.
269+
* `the dialog "Cookie banner" at (16, 420 358x180)`. Coordinates are rounded; the detail payload
270+
* carries the unrounded rect.
271+
*/
272+
function formatSnapshotCoveringNode(cover: SnapshotCoveringNode): string {
273+
const named = cover.label ? `${cover.kind} "${cover.label}"` : cover.kind;
274+
const rect = cover.rect;
275+
if (!rect) return `the ${named}`;
276+
return (
277+
`the ${named} at (${Math.round(rect.x)}, ${Math.round(rect.y)} ` +
278+
`${Math.round(rect.width)}x${Math.round(rect.height)})`
279+
);
280+
}
281+
282+
/**
283+
* The recovery hint every covered-interaction refusal publishes. When the scan named the cover it
284+
* says what to dismiss; a cover-less refusal (a backend-policy annotation that reports only the
285+
* boolean) keeps the generic recovery rather than guessing a culprit.
286+
*/
287+
export function coveredInteractionHint(coveredBy: SnapshotCoveringNode | undefined): string {
288+
return coveredBy
289+
? `Covered by ${formatSnapshotCoveringNode(coveredBy)}. Dismiss or close that element, scroll the target clear of it, or inspect with snapshot/screenshot before retrying.`
290+
: 'Use a different visible target, scroll it clear of the overlay, or inspect with snapshot/screenshot before retrying.';
291+
}
292+
249293
export type RawSnapshotNode = {
250294
index: number;
251295
type?: string;
@@ -297,6 +341,14 @@ export type RawSnapshotNode = {
297341
hiddenContentAbove?: boolean;
298342
hiddenContentBelow?: boolean;
299343
interactionBlocked?: 'covered';
344+
/**
345+
* Bounded description of the element the geometric occlusion scan found drawing on top of this
346+
* node, published together with `interactionBlocked: 'covered'` so a refusal can name the culprit
347+
* instead of only the fact of coverage. Absent when the covered decision came from a
348+
* backend-policy annotation that reports a boolean and names no cover. This is a description, not
349+
* an authorization: it carries no ref and is never an actionable target.
350+
*/
351+
coveredBy?: SnapshotCoveringNode;
300352
presentationHints?: string[];
301353
/**
302354
* Backend-minted ref for this node, when the capture backend already assigns a

‎src/commands/capture/runtime/snapshot-unchanged.test.ts‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,38 @@ test('unchanged metadata compares action and presentation hint arrays by value',
203203
expect(buildUnchangedSnapshotMetadata({ previous, current, options: {} })).toBeUndefined();
204204
});
205205

206+
test('unchanged metadata compares covering-node descriptions by value', () => {
207+
const cover = { kind: 'tab-bar', label: 'Media', rect: { x: 0, y: 760, width: 390, height: 84 } };
208+
const previous = snapshot('Create');
209+
previous.nodes[0]!.interactionBlocked = 'covered';
210+
previous.nodes[0]!.coveredBy = { ...cover, rect: { ...cover.rect } };
211+
const current = snapshot('Create', {
212+
nodes: [
213+
{
214+
...previous.nodes[0]!,
215+
coveredBy: { ...cover, rect: { ...cover.rect } },
216+
},
217+
],
218+
});
219+
220+
// A fresh object identity per capture is the norm; equivalence must read components.
221+
expect(buildUnchangedSnapshotMetadata({ previous, current, options: {} })).toMatchObject({
222+
nodeCount: 1,
223+
});
224+
225+
current.nodes[0]!.coveredBy = { ...cover, kind: 'toolbar' };
226+
expect(buildUnchangedSnapshotMetadata({ previous, current, options: {} })).toBeUndefined();
227+
228+
current.nodes[0]!.coveredBy = { ...cover, label: undefined };
229+
expect(buildUnchangedSnapshotMetadata({ previous, current, options: {} })).toBeUndefined();
230+
231+
current.nodes[0]!.coveredBy = { ...cover, rect: { ...cover.rect, y: 700 } };
232+
expect(buildUnchangedSnapshotMetadata({ previous, current, options: {} })).toBeUndefined();
233+
234+
current.nodes[0]!.coveredBy = undefined;
235+
expect(buildUnchangedSnapshotMetadata({ previous, current, options: {} })).toBeUndefined();
236+
});
237+
206238
test('unchanged metadata requires comparison-safe snapshots', () => {
207239
expect(
208240
buildUnchangedSnapshotMetadata({

‎src/commands/capture/runtime/snapshot-unchanged.ts‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ type ComparableSnapshotNode = Omit<
9595

9696
type ScalarPresentationField = Exclude<
9797
keyof ComparableSnapshotNode,
98-
'rect' | 'presentationHints' | 'actions'
98+
'rect' | 'coveredBy' | 'presentationHints' | 'actions'
9999
>;
100100

101101
const PRESENTATION_SCALAR_FIELDS = {
@@ -137,11 +137,25 @@ function areSnapshotNodesEquivalent(previous: SnapshotNode, current: SnapshotNod
137137
return (
138138
PRESENTATION_SCALAR_KEYS.every((field) => previous[field] === current[field]) &&
139139
areRectsEquivalent(previous.rect, current.rect) &&
140+
areCoveringNodesEquivalent(previous.coveredBy, current.coveredBy) &&
140141
areStringArraysEquivalent(previous.presentationHints, current.presentationHints) &&
141142
areStringArraysEquivalent(previous.actions, current.actions)
142143
);
143144
}
144145

146+
/** The occlusion scan's cover description is presentation evidence: two captures agree when every carried component does. */
147+
function areCoveringNodesEquivalent(
148+
previous: SnapshotNode['coveredBy'],
149+
current: SnapshotNode['coveredBy'],
150+
): boolean {
151+
if (!previous || !current) return previous === current;
152+
return (
153+
previous.kind === current.kind &&
154+
previous.label === current.label &&
155+
areRectsEquivalent(previous.rect, current.rect)
156+
);
157+
}
158+
145159
function areRectsEquivalent(previous: Rect | undefined, current: Rect | undefined): boolean {
146160
if (!previous || !current) return previous === current;
147161
return (

‎src/commands/interaction/runtime/__tests__/test-utils/index.ts‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import type { AgentDeviceBackend, BackendSnapshotResult } from '@agent-device/contracts/backend';
2+
import { annotateCoveredSnapshotNodes } from '@agent-device/capture-kit/snapshot-occlusion';
23
import { createLocalArtifactAdapter } from '../../../../../io.ts';
34
import type { SnapshotState } from '@agent-device/kernel/snapshot';
45
import { createAgentDevice } from '../../../../../runtime.ts';
@@ -183,6 +184,42 @@ export function coveredByTabBarSnapshot(): SnapshotState {
183184
]);
184185
}
185186

187+
/**
188+
* The same tab-bar-over-button shape run through the real annotation pass, so the covered node
189+
* carries the scan's `coveredBy` description exactly as a capture publishes it.
190+
*/
191+
export function coveredByScanSnapshot(): SnapshotState {
192+
return makeSnapshotState(
193+
annotateCoveredSnapshotNodes([
194+
{
195+
index: 0,
196+
depth: 0,
197+
type: 'Application',
198+
label: 'Example',
199+
rect: { x: 0, y: 0, width: 390, height: 844 },
200+
},
201+
{
202+
index: 1,
203+
depth: 1,
204+
parentIndex: 0,
205+
type: 'Button',
206+
label: 'Save draft',
207+
rect: { x: 16, y: 790, width: 140, height: 44 },
208+
hittable: true,
209+
},
210+
{
211+
index: 2,
212+
depth: 1,
213+
parentIndex: 0,
214+
type: 'TabBar',
215+
label: 'Media',
216+
rect: { x: 0, y: 760, width: 390, height: 84 },
217+
hittable: true,
218+
},
219+
]),
220+
);
221+
}
222+
186223
export function duplicateCoveredLabelSnapshot(): SnapshotState {
187224
return makeSnapshotState([
188225
{

0 commit comments

Comments
 (0)