Skip to content

Commit 248e08a

Browse files
authored
feat: warn on out-of-viewport coordinate taps (ADR 0011) (#1085)
* feat: warn on out-of-viewport coordinate taps (ADR 0011) Implements the coordinate/offscreen guarantee: when a point tap target falls outside the last-known viewport, attach a warning to the result instead of silently forwarding the tap. This is a WARNING not an error, maintaining the escape-hatch semantics of coordinate taps. The warning is generated at the runtime layer in resolvePointInteractionTarget using the session's last snapshot to infer viewport bounds via resolveViewportRect. No snapshot is captured (by design), so the implementation reuses the session's existing snapshot state. When no snapshot is available, no warning is emitted. Changes: - Add `warning?: string` field to PressCommandResult and LongPressCommandResult types (FillCommandResult already had it for --verify evidence) - Implement tryResolveOutOfBoundsPointWarning in resolution.ts and export it for registry verification - Update interaction-guarantees.ts to mark coordinate/offscreen as runtime-enforced - Remove coordinate/offscreen from the pinned gap list in the test registry - Add tests: out-of-bounds point with snapshot → warning; in-bounds → no warning; no snapshot → no warning - Carry the warning field through daemon handlers for press/click/longpress Closes #1081 (coordinate/offscreen gap). * refactor: drop suppressed re-export; registry points at the enforcement carrier The registry via now references resolveInteractionTarget (the exported carrier, same precedent as the verifyEvidence cell) instead of forcing a suppressed unused-export of the internal helper.
1 parent b25ef7b commit 248e08a

5 files changed

Lines changed: 111 additions & 6 deletions

File tree

‎src/commands/interaction/runtime/interactions.test.ts‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1379,6 +1379,85 @@ test('runtime interaction commands are available from the command namespace', as
13791379
assert.equal(result.kind, 'selector');
13801380
});
13811381

1382+
test('coordinate tap with out-of-bounds point warns when session has viewport', async () => {
1383+
const device = createInteractionDevice(
1384+
makeSnapshotState([
1385+
{
1386+
index: 0,
1387+
depth: 0,
1388+
type: 'Application',
1389+
rect: { x: 0, y: 0, width: 400, height: 800 },
1390+
hittable: true,
1391+
},
1392+
]),
1393+
{
1394+
tap: async () => ({ ok: true }),
1395+
},
1396+
);
1397+
1398+
const result = await device.interactions.click(
1399+
{ kind: 'point', x: 500, y: 500 },
1400+
{
1401+
session: 'default',
1402+
},
1403+
);
1404+
1405+
assert.equal(result.kind, 'point');
1406+
assert.equal(
1407+
result.warning,
1408+
'Coordinates (500, 500) are outside the last-known viewport (400x800). The tap will be forwarded anyway; take a fresh snapshot if the screen changed.',
1409+
);
1410+
});
1411+
1412+
test('coordinate tap with in-bounds point has no warning', async () => {
1413+
const device = createInteractionDevice(
1414+
makeSnapshotState([
1415+
{
1416+
index: 0,
1417+
depth: 0,
1418+
type: 'Application',
1419+
rect: { x: 0, y: 0, width: 400, height: 800 },
1420+
hittable: true,
1421+
},
1422+
]),
1423+
{
1424+
tap: async () => ({ ok: true }),
1425+
},
1426+
);
1427+
1428+
const result = await device.interactions.click(
1429+
{ kind: 'point', x: 200, y: 400 },
1430+
{
1431+
session: 'default',
1432+
},
1433+
);
1434+
1435+
assert.equal(result.kind, 'point');
1436+
assert.equal('warning' in result, false);
1437+
});
1438+
1439+
test('coordinate tap with no session snapshot has no warning', async () => {
1440+
const device = createAgentDevice({
1441+
backend: {
1442+
platform: 'ios',
1443+
tap: async () => ({ ok: true }),
1444+
} satisfies AgentDeviceBackend,
1445+
artifacts: createLocalArtifactAdapter(),
1446+
sessions: createMemorySessionStore([{ name: 'default' }]),
1447+
policy: localCommandPolicy(),
1448+
});
1449+
1450+
const result = await device.interactions.click(
1451+
{ kind: 'point', x: 500, y: 500 },
1452+
{
1453+
session: 'default',
1454+
},
1455+
);
1456+
1457+
assert.equal(result.kind, 'point');
1458+
assert.equal('warning' in result, false);
1459+
});
1460+
13821461
function selectorSnapshot(): SnapshotState {
13831462
return makeSnapshotState([
13841463
{

‎src/commands/interaction/runtime/resolution.ts‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import {
1616
isNodeVisibleOnScreen,
1717
resolveEffectiveViewportRect,
1818
} from '../../../snapshot/mobile-snapshot-semantics.ts';
19+
import { containsPoint, resolveViewportRect } from '../../../utils/rect-visibility.ts';
1920
import { isSnapshotNodeInteractionBlocked } from '../../../snapshot/snapshot-occlusion.ts';
2021
import type {
2122
InteractionTarget,
@@ -74,20 +75,46 @@ export async function resolveInteractionTarget(
7475
return await resolveSelectorInteractionTarget(runtime, options, options.target, params);
7576
}
7677

78+
async function tryResolveOutOfBoundsPointWarning(
79+
runtime: AgentDeviceRuntime,
80+
options: CommandContext,
81+
target: PointTarget,
82+
): Promise<string | undefined> {
83+
const sessionName = options.session ?? 'default';
84+
const session = await runtime.sessions.get(sessionName);
85+
if (!session?.snapshot) return undefined;
86+
87+
// Create a synthetic rect from the point for viewport lookup
88+
const pointRect = { x: target.x, y: target.y, width: 0, height: 0 };
89+
const viewport = resolveViewportRect(session.snapshot.nodes, pointRect);
90+
if (!viewport) return undefined;
91+
92+
const point = { x: target.x, y: target.y };
93+
if (containsPoint(viewport, point.x, point.y)) return undefined;
94+
95+
return `Coordinates (${point.x}, ${point.y}) are outside the last-known viewport (${viewport.width}x${viewport.height}). The tap will be forwarded anyway; take a fresh snapshot if the screen changed.`;
96+
}
97+
7798
async function resolvePointInteractionTarget(
7899
runtime: AgentDeviceRuntime,
79100
options: CommandContext,
80101
target: PointTarget,
81102
params: ResolveInteractionTargetParams,
82103
): Promise<ResolvedInteractionTarget> {
104+
const warning = await tryResolveOutOfBoundsPointWarning(runtime, options, target);
83105
if (!params.captureEvidenceBaseline) {
84-
return { kind: 'point', point: { x: target.x, y: target.y } };
106+
return {
107+
kind: 'point',
108+
point: { x: target.x, y: target.y },
109+
...(warning ? { warning } : {}),
110+
};
85111
}
86112
const preActionNodes = await tryCaptureEvidenceBaseline(runtime, options);
87113
return {
88114
kind: 'point',
89115
point: { x: target.x, y: target.y },
90116
...(preActionNodes ? { preActionNodes } : {}),
117+
...(warning ? { warning } : {}),
91118
};
92119
}
93120

‎src/contracts/__tests__/interaction-guarantees.test.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,6 @@ test('acknowledged gaps are visible and bounded', () => {
179179
// updates it here with a linked issue. It is the diffable debt list
180180
// (umbrella: https://github.com/callstack/agent-device/issues/1081).
181181
assert.deepEqual(gaps.sort(), [
182-
'coordinate/offscreen',
183182
'direct-ios-selector/disambiguation',
184183
'direct-ios-selector/errorTaxonomy',
185184
'direct-ios-selector/nonHittable',

‎src/contracts/interaction-guarantees.ts‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -302,10 +302,8 @@ export const INTERACTION_DISPATCH_PATHS: Record<InteractionPathId, InteractionPa
302302
reason: 'Coordinates bypass element semantics by design (escape hatch).',
303303
},
304304
offscreen: {
305-
kind: 'waived',
306-
reason:
307-
'gap: out-of-viewport coordinates are forwarded as-is; a bounds warning would be cheap.',
308-
trackingIssue: GAPS_UMBRELLA_ISSUE,
305+
kind: 'runtime',
306+
via: 'src/commands/interaction/runtime/resolution.ts#resolveInteractionTarget',
309307
},
310308
nonHittable: {
311309
kind: 'inapplicable',

‎src/contracts/interaction.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ export type InteractionEvidence = {
8080
export type PressCommandResult = ResolvedInteractionTarget & {
8181
backendResult?: Record<string, unknown>;
8282
message?: string;
83+
warning?: string;
8384
evidence?: InteractionEvidence;
8485
};
8586

@@ -95,4 +96,5 @@ export type LongPressCommandResult = ResolvedInteractionTarget & {
9596
durationMs?: number;
9697
backendResult?: Record<string, unknown>;
9798
message?: string;
99+
warning?: string;
98100
};

0 commit comments

Comments
 (0)