Skip to content

Commit 454d38a

Browse files
committed
feat: warn on out-of-viewport coordinate taps (ADR 0011)
Implements the coordinate/offscreen guarantee: when a point tap target falls outside the last-known viewport, attach a warning to the result instead of silently forwarding the tap. This is a WARNING not an error, maintaining the escape-hatch semantics of coordinate taps. The warning is generated at the runtime layer in resolvePointInteractionTarget using the session's last snapshot to infer viewport bounds via resolveViewportRect. No snapshot is captured (by design), so the implementation reuses the session's existing snapshot state. When no snapshot is available, no warning is emitted. Changes: - Add `warning?: string` field to PressCommandResult and LongPressCommandResult types (FillCommandResult already had it for --verify evidence) - Implement tryResolveOutOfBoundsPointWarning in resolution.ts and export it for registry verification - Update interaction-guarantees.ts to mark coordinate/offscreen as runtime-enforced - Remove coordinate/offscreen from the pinned gap list in the test registry - Add tests: out-of-bounds point with snapshot → warning; in-bounds → no warning; no snapshot → no warning - Carry the warning field through daemon handlers for press/click/longpress Closes #1081 (coordinate/offscreen gap).
1 parent b25ef7b commit 454d38a

6 files changed

Lines changed: 116 additions & 6 deletions

File tree

‎src/commands/interaction/runtime/interactions.test.ts‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1379,6 +1379,85 @@ test('runtime interaction commands are available from the command namespace', as
13791379
assert.equal(result.kind, 'selector');
13801380
});
13811381

1382+
test('coordinate tap with out-of-bounds point warns when session has viewport', async () => {
1383+
const device = createInteractionDevice(
1384+
makeSnapshotState([
1385+
{
1386+
index: 0,
1387+
depth: 0,
1388+
type: 'Application',
1389+
rect: { x: 0, y: 0, width: 400, height: 800 },
1390+
hittable: true,
1391+
},
1392+
]),
1393+
{
1394+
tap: async () => ({ ok: true }),
1395+
},
1396+
);
1397+
1398+
const result = await device.interactions.click(
1399+
{ kind: 'point', x: 500, y: 500 },
1400+
{
1401+
session: 'default',
1402+
},
1403+
);
1404+
1405+
assert.equal(result.kind, 'point');
1406+
assert.equal(
1407+
result.warning,
1408+
'Coordinates (500, 500) are outside the last-known viewport (400x800). The tap will be forwarded anyway; take a fresh snapshot if the screen changed.',
1409+
);
1410+
});
1411+
1412+
test('coordinate tap with in-bounds point has no warning', async () => {
1413+
const device = createInteractionDevice(
1414+
makeSnapshotState([
1415+
{
1416+
index: 0,
1417+
depth: 0,
1418+
type: 'Application',
1419+
rect: { x: 0, y: 0, width: 400, height: 800 },
1420+
hittable: true,
1421+
},
1422+
]),
1423+
{
1424+
tap: async () => ({ ok: true }),
1425+
},
1426+
);
1427+
1428+
const result = await device.interactions.click(
1429+
{ kind: 'point', x: 200, y: 400 },
1430+
{
1431+
session: 'default',
1432+
},
1433+
);
1434+
1435+
assert.equal(result.kind, 'point');
1436+
assert.equal('warning' in result, false);
1437+
});
1438+
1439+
test('coordinate tap with no session snapshot has no warning', async () => {
1440+
const device = createAgentDevice({
1441+
backend: {
1442+
platform: 'ios',
1443+
tap: async () => ({ ok: true }),
1444+
} satisfies AgentDeviceBackend,
1445+
artifacts: createLocalArtifactAdapter(),
1446+
sessions: createMemorySessionStore([{ name: 'default' }]),
1447+
policy: localCommandPolicy(),
1448+
});
1449+
1450+
const result = await device.interactions.click(
1451+
{ kind: 'point', x: 500, y: 500 },
1452+
{
1453+
session: 'default',
1454+
},
1455+
);
1456+
1457+
assert.equal(result.kind, 'point');
1458+
assert.equal('warning' in result, false);
1459+
});
1460+
13821461
function selectorSnapshot(): SnapshotState {
13831462
return makeSnapshotState([
13841463
{

‎src/commands/interaction/runtime/resolution.ts‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import {
1616
isNodeVisibleOnScreen,
1717
resolveEffectiveViewportRect,
1818
} from '../../../snapshot/mobile-snapshot-semantics.ts';
19+
import { containsPoint, resolveViewportRect } from '../../../utils/rect-visibility.ts';
1920
import { isSnapshotNodeInteractionBlocked } from '../../../snapshot/snapshot-occlusion.ts';
2021
import type {
2122
InteractionTarget,
@@ -27,6 +28,10 @@ import { resolveActionableTouchResolution } from '../../../core/interaction-targ
2728

2829
export type { InteractionTarget, PointTarget, ResolvedInteractionTarget };
2930

31+
// Referenced in interaction-guarantees.ts registry via dynamic symbol lookup.
32+
// fallow-ignore-next-line unused-export
33+
export { tryResolveOutOfBoundsPointWarning };
34+
3035
export type InteractionAction =
3136
| 'click'
3237
| 'press'
@@ -74,20 +79,46 @@ export async function resolveInteractionTarget(
7479
return await resolveSelectorInteractionTarget(runtime, options, options.target, params);
7580
}
7681

82+
async function tryResolveOutOfBoundsPointWarning(
83+
runtime: AgentDeviceRuntime,
84+
options: CommandContext,
85+
target: PointTarget,
86+
): Promise<string | undefined> {
87+
const sessionName = options.session ?? 'default';
88+
const session = await runtime.sessions.get(sessionName);
89+
if (!session?.snapshot) return undefined;
90+
91+
// Create a synthetic rect from the point for viewport lookup
92+
const pointRect = { x: target.x, y: target.y, width: 0, height: 0 };
93+
const viewport = resolveViewportRect(session.snapshot.nodes, pointRect);
94+
if (!viewport) return undefined;
95+
96+
const point = { x: target.x, y: target.y };
97+
if (containsPoint(viewport, point.x, point.y)) return undefined;
98+
99+
return `Coordinates (${point.x}, ${point.y}) are outside the last-known viewport (${viewport.width}x${viewport.height}). The tap will be forwarded anyway; take a fresh snapshot if the screen changed.`;
100+
}
101+
77102
async function resolvePointInteractionTarget(
78103
runtime: AgentDeviceRuntime,
79104
options: CommandContext,
80105
target: PointTarget,
81106
params: ResolveInteractionTargetParams,
82107
): Promise<ResolvedInteractionTarget> {
108+
const warning = await tryResolveOutOfBoundsPointWarning(runtime, options, target);
83109
if (!params.captureEvidenceBaseline) {
84-
return { kind: 'point', point: { x: target.x, y: target.y } };
110+
return {
111+
kind: 'point',
112+
point: { x: target.x, y: target.y },
113+
...(warning ? { warning } : {}),
114+
};
85115
}
86116
const preActionNodes = await tryCaptureEvidenceBaseline(runtime, options);
87117
return {
88118
kind: 'point',
89119
point: { x: target.x, y: target.y },
90120
...(preActionNodes ? { preActionNodes } : {}),
121+
...(warning ? { warning } : {}),
91122
};
92123
}
93124

‎src/contracts/__tests__/interaction-guarantees.test.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,6 @@ test('acknowledged gaps are visible and bounded', () => {
179179
// updates it here with a linked issue. It is the diffable debt list
180180
// (umbrella: https://github.com/callstack/agent-device/issues/1081).
181181
assert.deepEqual(gaps.sort(), [
182-
'coordinate/offscreen',
183182
'direct-ios-selector/disambiguation',
184183
'direct-ios-selector/errorTaxonomy',
185184
'direct-ios-selector/nonHittable',

‎src/contracts/interaction-guarantees.ts‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -302,10 +302,8 @@ export const INTERACTION_DISPATCH_PATHS: Record<InteractionPathId, InteractionPa
302302
reason: 'Coordinates bypass element semantics by design (escape hatch).',
303303
},
304304
offscreen: {
305-
kind: 'waived',
306-
reason:
307-
'gap: out-of-viewport coordinates are forwarded as-is; a bounds warning would be cheap.',
308-
trackingIssue: GAPS_UMBRELLA_ISSUE,
305+
kind: 'runtime',
306+
via: 'src/commands/interaction/runtime/resolution.ts#tryResolveOutOfBoundsPointWarning',
309307
},
310308
nonHittable: {
311309
kind: 'inapplicable',

‎src/contracts/interaction.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ export type InteractionEvidence = {
8080
export type PressCommandResult = ResolvedInteractionTarget & {
8181
backendResult?: Record<string, unknown>;
8282
message?: string;
83+
warning?: string;
8384
evidence?: InteractionEvidence;
8485
};
8586

@@ -95,4 +96,5 @@ export type LongPressCommandResult = ResolvedInteractionTarget & {
9596
durationMs?: number;
9697
backendResult?: Record<string, unknown>;
9798
message?: string;
99+
warning?: string;
98100
};

‎src/daemon/handlers/interaction-touch.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -236,6 +236,7 @@ async function buildTargetedTouchResponsePayloads(params: {
236236
referenceFrame,
237237
extra,
238238
});
239+
return responseData;
239240
}
240241

241242
function readLongPressResultDuration(result: TargetedTouchResult): number | undefined {

0 commit comments

Comments
 (0)