Skip to content

Commit b9fb4db

Browse files
authored
feat: preflight native-ref fast path with shared interaction guards (ADR 0011) (#1082)
click @ref / fill @ref dispatch straight to backend.tapTarget/fillTarget, and a backend fast path can silently "succeed" — delegation-on-error never triggers there. The ref came from the stored session snapshot, so the node is already in hand: preflightNativeRefInteraction runs the SAME shared guards the runtime path uses against that node before the backend call. - occlusion: isSnapshotNodeInteractionBlocked via assertInteractionNotBlocked (same covered-element error shape as the runtime path) - offscreen: isNodeVisibleOnScreen via assertVisibleRefTarget (same offscreen_ref error shape) - nonHittable: annotation only — targetHittable/hint attach to the result; promotion stays a runtime-path behavior and the tapped element never changes No snapshot in the session, an unresolvable ref, or a missing/unusable rect make the preflight a no-op: it never adds a capture round trip. Registry: the three native-ref gap cells flip to kind 'runtime' and leave the pinned gap list. Refs #1081
1 parent c506ddf commit b9fb4db

5 files changed

Lines changed: 219 additions & 17 deletions

File tree

‎src/commands/interaction/runtime/interactions.test.ts‎

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,145 @@ test('runtime fill uses backend ref primitive without resolving snapshot geometr
127127
assert.deepEqual(result.backendResult, { ref: 'e1', text: 'hello' });
128128
});
129129

130+
test('native ref click preflight refuses an off-screen ref without calling the backend', async () => {
131+
// Closed-drawer shape (ADR 0011): the stored session snapshot already holds
132+
// the node, so the fast path must refuse it with the runtime path's exact
133+
// offscreen_ref shape instead of letting the backend silently "succeed".
134+
const calls: string[] = [];
135+
const device = createInteractionDevice(offscreenDrawerSnapshot(), {
136+
platform: 'web',
137+
captureSnapshot: async () => {
138+
throw new Error('native ref preflight must not capture a snapshot');
139+
},
140+
tapTarget: async (_context, target) => {
141+
calls.push(target.ref);
142+
return {};
143+
},
144+
});
145+
146+
await assert.rejects(
147+
() => device.interactions.click(ref('@e2'), { session: 'default' }),
148+
(error: unknown) => {
149+
assert.ok(error instanceof Error);
150+
assert.match(error.message, /Ref @e2 is off-screen and not safe to click/);
151+
const details = (error as { details?: Record<string, unknown> }).details;
152+
assert.equal(details?.reason, 'offscreen_ref');
153+
assert.equal(details?.ref, 'e2');
154+
assert.ok(typeof details?.hint === 'string');
155+
return true;
156+
},
157+
);
158+
assert.deepEqual(calls, []);
159+
});
160+
161+
test('native ref fill preflight refuses an off-screen ref without calling the backend', async () => {
162+
const calls: string[] = [];
163+
const device = createInteractionDevice(offscreenDrawerSnapshot(), {
164+
platform: 'web',
165+
captureSnapshot: async () => {
166+
throw new Error('native ref preflight must not capture a snapshot');
167+
},
168+
fillTarget: async (_context, target) => {
169+
calls.push(target.ref);
170+
return {};
171+
},
172+
});
173+
174+
await assert.rejects(
175+
() => device.interactions.fill(ref('@e2'), 'hello', { session: 'default' }),
176+
(error: unknown) => {
177+
assert.ok(error instanceof Error);
178+
assert.match(error.message, /Ref @e2 is off-screen and not safe to fill/);
179+
const details = (error as { details?: Record<string, unknown> }).details;
180+
assert.equal(details?.reason, 'offscreen_ref');
181+
return true;
182+
},
183+
);
184+
assert.deepEqual(calls, []);
185+
});
186+
187+
test('native ref click preflight refuses a covered ref without calling the backend', async () => {
188+
const calls: string[] = [];
189+
const device = createInteractionDevice(coveredByTabBarSnapshot(), {
190+
platform: 'web',
191+
captureSnapshot: async () => {
192+
throw new Error('native ref preflight must not capture a snapshot');
193+
},
194+
tapTarget: async (_context, target) => {
195+
calls.push(target.ref);
196+
return {};
197+
},
198+
});
199+
200+
await assert.rejects(
201+
() => device.interactions.click(ref('@e2'), { session: 'default' }),
202+
(error: unknown) => {
203+
assert.ok(error instanceof Error);
204+
// Same shape as the runtime path's buildCoveredInteractionError.
205+
assert.match(error.message, /Ref @e2 is covered by another visible element/);
206+
const details = (error as { details?: Record<string, unknown> }).details;
207+
assert.equal(details?.ref, '@e2');
208+
assert.equal(details?.interactionBlocked, 'covered');
209+
return true;
210+
},
211+
);
212+
assert.deepEqual(calls, []);
213+
});
214+
215+
test('native ref click preflight annotates non-hittable targets but still calls the backend', async () => {
216+
const calls: string[] = [];
217+
const device = createInteractionDevice(nonHittableCellSnapshot(), {
218+
platform: 'web',
219+
captureSnapshot: async () => {
220+
throw new Error('native ref preflight must not capture a snapshot');
221+
},
222+
tapTarget: async (_context, target) => {
223+
calls.push(target.ref);
224+
return { ref: target.ref.replace(/^@/, '') };
225+
},
226+
});
227+
228+
const result = await device.interactions.click(ref('@e2'), { session: 'default' });
229+
230+
// Annotation only: the backend still acts on the ref (no promotion on the
231+
// fast path), and the result carries the same targetHittable/hint fields
232+
// the runtime path attaches.
233+
assert.deepEqual(calls, ['@e2']);
234+
assert.equal(result.kind, 'ref');
235+
assert.equal(result.targetHittable, false);
236+
assert.match(result.hint ?? '', /hittable: false/);
237+
assert.deepEqual(result.backendResult, { ref: 'e2' });
238+
});
239+
240+
test('native ref fast path proceeds untouched when the session has no snapshot', async () => {
241+
const calls: string[] = [];
242+
const device = createAgentDevice({
243+
backend: {
244+
platform: 'web',
245+
captureSnapshot: async () => {
246+
throw new Error('native ref preflight must not capture a snapshot');
247+
},
248+
tap: async () => {},
249+
typeText: async () => {},
250+
tapTarget: async (_context, target) => {
251+
calls.push(target.ref);
252+
return { ref: target.ref.replace(/^@/, '') };
253+
},
254+
} satisfies AgentDeviceBackend,
255+
artifacts: createLocalArtifactAdapter(),
256+
sessions: createMemorySessionStore([{ name: 'default' }]),
257+
policy: localCommandPolicy(),
258+
});
259+
260+
const result = await device.interactions.click(ref('@e2'), { session: 'default' });
261+
262+
assert.deepEqual(calls, ['@e2']);
263+
assert.equal(result.kind, 'ref');
264+
assert.equal(result.targetHittable, undefined);
265+
assert.equal(result.hint, undefined);
266+
assert.deepEqual(result.backendResult, { ref: 'e2' });
267+
});
268+
130269
test('runtime interactions pass runtime signal to backend primitives', async () => {
131270
const controller = new AbortController();
132271
let signal: AbortSignal | undefined;
@@ -1254,6 +1393,29 @@ function selectorSnapshot(): SnapshotState {
12541393
]);
12551394
}
12561395

1396+
// Closed-drawer shape shared by the native-ref preflight tests: the only
1397+
// interactive node (@e2) sits fully left of the Application viewport.
1398+
function offscreenDrawerSnapshot(): SnapshotState {
1399+
return makeSnapshotState([
1400+
{
1401+
index: 0,
1402+
depth: 0,
1403+
type: 'Application',
1404+
rect: { x: 0, y: 0, width: 400, height: 800 },
1405+
hittable: true,
1406+
},
1407+
{
1408+
index: 1,
1409+
depth: 2,
1410+
parentIndex: 0,
1411+
type: 'Button',
1412+
label: 'Explore',
1413+
rect: { x: -320, y: 240, width: 300, height: 50 },
1414+
hittable: true,
1415+
},
1416+
]);
1417+
}
1418+
12571419
function runtimeScrollSnapshot(options: { hiddenBelow: boolean; message?: string }): SnapshotState {
12581420
return makeSnapshotState([
12591421
{

‎src/commands/interaction/runtime/interactions.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import type { RepeatedInput } from '../../command-input.ts';
2424
import {
2525
captureInteractionSnapshot,
2626
type InteractionTarget,
27+
preflightNativeRefInteraction,
2728
resolveInteractionTarget,
2829
} from './resolution.ts';
2930

@@ -272,6 +273,11 @@ async function maybeTapRefTarget(
272273
return null;
273274
}
274275
if (hasNonDefaultTapOptions(options)) return null;
276+
// ADR 0011 native-ref preflight: the shared occlusion/offscreen guards run
277+
// against the stored session snapshot node before the backend call (a
278+
// backend fast path can silently "succeed", so errors must be raised here).
279+
// No snapshot / no usable rect → no-op; never adds a capture round trip.
280+
const preflight = await preflightNativeRefInteraction(runtime, options, options.target, action);
275281
const backendResult = await runtime.backend.tapTarget(toBackendContext(runtime, options), {
276282
kind: 'ref',
277283
ref: options.target.ref,
@@ -281,6 +287,7 @@ async function maybeTapRefTarget(
281287
return {
282288
kind: 'ref',
283289
target: { kind: 'ref', ref: options.target.ref },
290+
...preflight,
284291
...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}),
285292
};
286293
}
@@ -290,6 +297,8 @@ async function maybeFillRefTarget(
290297
options: FillCommandOptions,
291298
): Promise<FillCommandResult | null> {
292299
if (options.target.kind !== 'ref' || !runtime.backend.fillTarget) return null;
300+
// ADR 0011 native-ref preflight — see maybeTapRefTarget.
301+
const preflight = await preflightNativeRefInteraction(runtime, options, options.target, 'fill');
293302
const backendResult = await runtime.backend.fillTarget(
294303
toBackendContext(runtime, options),
295304
{
@@ -305,6 +314,7 @@ async function maybeFillRefTarget(
305314
kind: 'ref',
306315
target: { kind: 'ref', ref: options.target.ref },
307316
text: options.text,
317+
...preflight,
308318
...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}),
309319
};
310320
}

‎src/commands/interaction/runtime/resolution.ts‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -443,6 +443,41 @@ function assertVisibleRefTarget(
443443
});
444444
}
445445

446+
/**
447+
* ADR 0011 native-ref preflight: `click @ref` / `fill @ref` fast paths
448+
* dispatch straight to `backend.tapTarget`/`fillTarget`, and a backend fast
449+
* path can silently "succeed" — delegation-on-error never triggers there. The
450+
* ref came from the stored session snapshot, so the node is already in hand:
451+
* run the SAME shared guards the runtime path uses against it before the
452+
* backend call — occlusion (`isSnapshotNodeInteractionBlocked` via
453+
* `assertInteractionNotBlocked`) and offscreen (`isNodeVisibleOnScreen` via
454+
* `assertVisibleRefTarget`) ERROR with the runtime path's exact shapes, and
455+
* the non-hittable annotation is returned for the fast-path result.
456+
*
457+
* Zero extra round trips by construction: no session, no stored snapshot, an
458+
* unresolvable/invalid ref, or a node without a usable rect all make the
459+
* preflight a no-op and the fast path proceeds exactly as before. Promotion
460+
* to a hittable ancestor stays a runtime-path behavior — the preflight never
461+
* changes which element the backend acts on.
462+
*/
463+
export async function preflightNativeRefInteraction(
464+
runtime: AgentDeviceRuntime,
465+
options: CommandContext,
466+
target: Extract<InteractionTarget, { kind: 'ref' }>,
467+
action: InteractionAction,
468+
): Promise<{ targetHittable?: boolean; hint?: string }> {
469+
const session = await runtime.sessions.get(options.session ?? 'default');
470+
const nodes = session?.snapshot?.nodes;
471+
if (!nodes || normalizeRef(target.ref) === null) return {};
472+
const resolved = tryResolveRefNode(nodes, target.ref, {
473+
fallbackLabel: target.fallbackLabel ?? '',
474+
});
475+
if (!resolved) return {};
476+
assertInteractionNotBlocked(resolved.node, `Ref ${target.ref}`, action);
477+
assertVisibleRefTarget(resolved.node, nodes, target.ref, action);
478+
return describeNonHittableTarget(resolved.node, action);
479+
}
480+
446481
// isNodeVisibleOnScreen (not the effective-viewport form): items inside an
447482
// off-screen scrollable container (closed drawer) must also count as
448483
// off-screen, not just items scrolled out of an on-screen container.

‎src/contracts/__tests__/interaction-guarantees.test.ts‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -189,9 +189,6 @@ test('acknowledged gaps are visible and bounded', () => {
189189
'direct-ios-selector/responseIdentity',
190190
'maestro-non-hittable-fallback/errorTaxonomy',
191191
'maestro-non-hittable-fallback/responseConstruction',
192-
'native-ref/nonHittable',
193-
'native-ref/occlusion',
194-
'native-ref/offscreen',
195192
'native-ref/responseConstruction',
196193
'runtime-ref/responseConstruction',
197194
'runtime-selector/responseConstruction',

‎src/contracts/interaction-guarantees.ts‎

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@
2323
* where the fast path can succeed on a candidate the runtime rules would
2424
* refuse stay gaps until proven); and a shared runtime preflight against the
2525
* already-captured snapshot node for the native-ref path, because a backend
26-
* fast path can silently succeed and delegation-on-error never triggers.
26+
* fast path can silently succeed and delegation-on-error never triggers
27+
* (implemented: preflightNativeRefInteraction, #1081).
2728
*/
2829

2930
export const INTERACTION_GUARANTEES = [
@@ -256,30 +257,27 @@ export const INTERACTION_DISPATCH_PATHS: Record<InteractionPathId, InteractionPa
256257
},
257258
'native-ref': {
258259
description:
259-
'click @ref / fill @ref dispatch to backend.tapTarget/fillTarget without runtime resolution when no non-default options are set.',
260+
'click @ref / fill @ref dispatch to backend.tapTarget/fillTarget without runtime resolution when no non-default options are set. A zero-round-trip preflight (preflightNativeRefInteraction) runs the shared guards against the stored session snapshot node first; no snapshot / no usable rect makes the preflight a no-op.',
260261
commands: ['click', 'fill'],
261262
guarantees: {
262263
disambiguation: {
263264
kind: 'inapplicable',
264265
reason: 'Refs identify exactly one node by construction.',
265266
},
266267
occlusion: {
267-
kind: 'waived',
268-
reason:
269-
'gap: no covered-element check before the native ref tap; closure strategy is a shared runtime preflight against the snapshot node before the backend call — a backend fast path can silently succeed, so delegation-on-error never triggers.',
270-
trackingIssue: GAPS_UMBRELLA_ISSUE,
268+
kind: 'runtime',
269+
via: 'src/snapshot/snapshot-occlusion.ts#isSnapshotNodeInteractionBlocked',
271270
},
272271
offscreen: {
273-
kind: 'waived',
274-
reason:
275-
'gap: no viewport check before the native ref tap; closure strategy is the same shared runtime preflight (the ref came from a daemon snapshot, so the node is already available).',
276-
trackingIssue: GAPS_UMBRELLA_ISSUE,
272+
kind: 'runtime',
273+
via: 'src/snapshot/mobile-snapshot-semantics.ts#isNodeVisibleOnScreen',
277274
},
275+
// Annotation only (targetHittable/hint on the result): promotion to a
276+
// hittable ancestor stays a runtime-path behavior — the preflight never
277+
// changes which element the backend acts on.
278278
nonHittable: {
279-
kind: 'waived',
280-
reason:
281-
'gap: no promotion/annotation on the native ref path; closure strategy is the same shared runtime preflight.',
282-
trackingIssue: GAPS_UMBRELLA_ISSUE,
279+
kind: 'runtime',
280+
via: 'src/commands/interaction/runtime/resolution.ts#preflightNativeRefInteraction',
283281
},
284282
responseConstruction: {
285283
kind: 'waived',

0 commit comments

Comments
 (0)