Skip to content

Commit 9f98fc7

Browse files
committed
fix(interaction): direction-named, selector-first off-screen recovery hints (#1366)
An agent that hits a scrollable form section gets correctly rejected for targeting an off-screen element, but the recovery hint didn't name a concrete next move. Its "obvious" retry — re-issue the same @ref after scrolling — is exactly what both live guards reject: the off-screen guard still sees it off-screen, and the scroll expired the ref frame (ADR 0014), so the @ref is refused as stale. Two bootstrap-bench runs burned all 60 turns on a single checkout-form screen this way (arm-independent). Make the rejection self-sufficient for recovery instead of relaxing either guard: - Off-screen selector/ref rejection now names the exact scroll direction (computed from the target rect vs its effective viewport) and steers the retry to a *selector*, which re-resolves against a fresh snapshot and bypasses the ref-frame admission guard entirely. The direction is also surfaced as a machine-readable `scrollDirection` detail. - `scroll`'s unknown-direction error now carries a grammar hint, since the transcripts show agents mis-shaping it as `scroll @ref down` — scroll takes a direction and no target. New geometry helper `classifyOffscreenScrollDirection` maps a rect + viewport to the reveal direction (largest-overshoot axis), following the same convention the CLI off-screen summary already uses. Both guards keep their exact rejection semantics and messages; only the hints and one added detail field change.
1 parent 00ef287 commit 9f98fc7

8 files changed

Lines changed: 146 additions & 10 deletions

File tree

‎src/commands/interaction/interactions.test.ts‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
import { expect, test } from 'vitest';
2-
import { interactionDaemonWriters } from './interactions.ts';
2+
import { interactionCliReaders, interactionDaemonWriters } from './interactions.ts';
3+
import { AppError } from '../../kernel/errors.ts';
4+
import type { CliFlags } from '../cli-grammar/flag-types.ts';
5+
6+
const BASE_FLAGS: CliFlags = { json: false, help: false, version: false };
37

48
test('swipe writes only typed daemon input', () => {
59
const request = interactionDaemonWriters.swipe({
@@ -19,3 +23,18 @@ test('swipe writes only typed daemon input', () => {
1923
pattern: 'ping-pong',
2024
});
2125
});
26+
27+
test('scroll reader rejects a @ref in the direction slot with a grammar hint (#1366)', () => {
28+
try {
29+
interactionCliReaders.scroll(['@e29'], BASE_FLAGS);
30+
expect.unreachable('scroll should reject a ref where a direction is expected');
31+
} catch (error) {
32+
expect(error).toBeInstanceOf(AppError);
33+
const appError = error as AppError;
34+
expect(appError.code).toBe('INVALID_ARGS');
35+
// The hint teaches the direction-first grammar and that scroll takes no target,
36+
// so the agent stops cycling `scroll @ref down` / `scroll down @ref`.
37+
expect(appError.details?.hint).toMatch(/direction first/i);
38+
expect(appError.details?.hint).toMatch(/no @ref or selector/i);
39+
}
40+
});

‎src/commands/interaction/interactions.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,12 @@ function readScrollDirection(value: string | undefined): ScrollInputDirection {
177177
) {
178178
return value;
179179
}
180-
throw new AppError('INVALID_ARGS', `Unknown direction: ${String(value)}`);
180+
// #1366: agents recovering from an off-screen target often mis-shape this as
181+
// `scroll @ref down` or `scroll down @ref`. scroll takes no target — name the
182+
// grammar so the retry lands instead of cycling through arg orders.
183+
throw new AppError('INVALID_ARGS', `Unknown direction: ${String(value)}`, {
184+
hint: 'scroll takes a direction first, then an optional amount: scroll <up|down|left|right|top|bottom> [amount]. It takes no @ref or selector — scroll the whole viewport, then retry the target with a selector.',
185+
});
181186
}
182187

183188
function readLongPressTargetPositionals(positionals: string[]): {

‎src/commands/interaction/runtime/interactions.test.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,10 @@ test('native ref click preflight refuses an off-screen ref without calling the b
160160
const details = (error as { details?: Record<string, unknown> }).details;
161161
assert.equal(details?.reason, 'offscreen_ref');
162162
assert.equal(details?.ref, 'e2');
163-
assert.ok(typeof details?.hint === 'string');
163+
// #1366: closed drawer sits left of the viewport -> `scroll left`, selector-first.
164+
assert.equal(details?.scrollDirection, 'left');
165+
assert.match(String(details?.hint), /scroll left/i);
166+
assert.match(String(details?.hint), /selector/i);
164167
return true;
165168
},
166169
);

‎src/commands/interaction/runtime/resolution.test.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,11 @@ test('runtime press refuses a selector that resolves to an off-screen element',
120120
assert.match(error.message, /off-screen element and is not safe to press/);
121121
const details = (error as { details?: Record<string, unknown> }).details;
122122
assert.equal(details?.reason, 'offscreen_selector');
123-
assert.ok(typeof details?.hint === 'string');
123+
// #1366: the closed-drawer shape sits fully left of the viewport, so the
124+
// hint names `scroll left` and steers back through the same selector.
125+
assert.equal(details?.scrollDirection, 'left');
126+
assert.match(String(details?.hint), /scroll left/i);
127+
assert.match(String(details?.hint), /selector/i);
124128
return true;
125129
},
126130
);

‎src/commands/interaction/runtime/resolution.ts‎

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,10 @@ import {
2424
resolveRefLabel,
2525
} from '../../../snapshot/snapshot-processing.ts';
2626
import {
27+
classifyOffscreenScrollDirection,
2728
isNodeVisibleOnScreen,
2829
resolveEffectiveViewportRect,
30+
type OffscreenScrollDirection,
2931
} from '../../../snapshot/mobile-snapshot-semantics.ts';
3032
import { containsPoint, resolveViewportRect } from '../../../utils/rect-visibility.ts';
3133
import { isSnapshotNodeInteractionBlocked } from '../../../snapshot/snapshot-occlusion.ts';
@@ -697,7 +699,12 @@ function assertVisibleSelectorTarget(
697699
throwIfOffscreenInteractionTarget(node, nodes, {
698700
message: `Selector ${selector} resolved to an off-screen element and is not safe to ${action}`,
699701
details: { reason: 'offscreen_selector', selector },
700-
hint: `The element is outside the visible viewport — likely inside a closed drawer, another tab, or scrolled content. Scroll toward it or open its container, take a fresh snapshot, then retry ${action}.`,
702+
// A selector re-resolves against a fresh snapshot on every attempt, so the
703+
// recovery is just: scroll the named direction, then retry THIS selector —
704+
// no separate snapshot step, and no @ref (a scroll expires the ref frame,
705+
// #1366). Naming the direction stops the wrong-way / retry-the-same-ref loop.
706+
hint: (direction) =>
707+
`${scrollRevealClause(direction)}, then retry ${action} with the same selector — a selector re-resolves against a fresh snapshot, so no separate snapshot is needed. If it is inside a closed drawer or another tab, open that container first.`,
701708
});
702709
}
703710

@@ -710,10 +717,23 @@ function assertVisibleRefTarget(
710717
throwIfOffscreenInteractionTarget(node, nodes, {
711718
message: `Ref ${refInput} is off-screen and not safe to ${action}`,
712719
details: { reason: 'offscreen_ref', ref: normalizeRef(refInput) },
713-
hint: `Use scroll with the direction from the off-screen summary, take a fresh snapshot, then retry ${action} with the new ref or a selector.`,
720+
// The scroll that reveals the target expires the ref frame (#1366, ADR
721+
// 0014), so retrying this @ref would be rejected next. Steer to a selector,
722+
// which re-resolves against a fresh snapshot and bypasses the ref-frame guard.
723+
hint: (direction) =>
724+
`${scrollRevealClause(direction)}, then retry ${action} with a selector (e.g. text=/id=) rather than this @ref — the scroll expires the ref frame, so re-run snapshot -i before reusing any @ref.`,
714725
});
715726
}
716727

728+
// Shared lead-in for both off-screen hints. Names the concrete `scroll <dir>`
729+
// when the geometry gives one, and falls back to the container-aware phrasing
730+
// when the target is off more than one edge in a way that has no single reveal.
731+
function scrollRevealClause(direction: OffscreenScrollDirection | null): string {
732+
return direction
733+
? `Scroll ${direction} to bring it on-screen`
734+
: 'Scroll toward it (open its container if it is in a closed drawer or another tab)';
735+
}
736+
717737
/**
718738
* ADR 0011 native-ref preflight: `click @ref` / `fill @ref` fast paths
719739
* dispatch straight to `backend.tapTarget`/`fillTarget`, and a backend fast
@@ -766,14 +786,23 @@ export async function preflightNativeRefInteraction(
766786
function throwIfOffscreenInteractionTarget(
767787
node: SnapshotNode,
768788
nodes: SnapshotState['nodes'],
769-
failure: { message: string; details: Record<string, unknown>; hint: string },
789+
failure: {
790+
message: string;
791+
details: Record<string, unknown>;
792+
hint: (direction: OffscreenScrollDirection | null) => string;
793+
},
770794
): void {
771795
const viewport = node.rect ? resolveEffectiveViewportRect(node, nodes) : null;
772796
if (!node.rect || !viewport || isNodeVisibleOnScreen(node, nodes)) return;
797+
// The direction that scrolls this off-screen target into view. Named in the
798+
// hint (and surfaced as a machine-readable detail) so the recovery is a single
799+
// deterministic move instead of a guess (#1366).
800+
const scrollDirection = classifyOffscreenScrollDirection(node.rect, viewport);
773801
throw new AppError('COMMAND_FAILED', failure.message, {
774802
...failure.details,
775803
rect: node.rect,
776804
viewport,
777-
hint: failure.hint,
805+
...(scrollDirection ? { scrollDirection } : {}),
806+
hint: failure.hint(scrollDirection),
778807
});
779808
}

‎src/daemon/handlers/__tests__/interaction.test.ts‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2593,8 +2593,12 @@ test('press @ref fails fast when the target is off-screen', async () => {
25932593
if (response && !response.ok) {
25942594
expect(response.error.code).toBe('COMMAND_FAILED');
25952595
expect(response.error.message).toMatch(/off-screen/i);
2596-
expect(response.error.hint).toMatch(/scroll.*fresh snapshot/i);
2596+
// #1366: the hint names the concrete scroll direction and steers to a
2597+
// selector-based retry (a @ref would be rejected as expired after the scroll).
2598+
expect(response.error.hint).toMatch(/scroll down/i);
2599+
expect(response.error.hint).toMatch(/selector/i);
25972600
expect(response.error.details?.reason).toBe('offscreen_ref');
2601+
expect(response.error.details?.scrollDirection).toBe('down');
25982602
}
25992603
});
26002604

@@ -2715,8 +2719,11 @@ test('fill @ref fails fast when the target is off-screen', async () => {
27152719
if (response && !response.ok) {
27162720
expect(response.error.code).toBe('COMMAND_FAILED');
27172721
expect(response.error.message).toMatch(/off-screen/i);
2718-
expect(response.error.hint).toMatch(/scroll.*fresh snapshot/i);
2722+
// #1366: direction-named, selector-first recovery hint (see press test above).
2723+
expect(response.error.hint).toMatch(/scroll down/i);
2724+
expect(response.error.hint).toMatch(/selector/i);
27192725
expect(response.error.details?.reason).toBe('offscreen_ref');
2726+
expect(response.error.details?.scrollDirection).toBe('down');
27202727
}
27212728
});
27222729

‎src/snapshot/mobile-snapshot-semantics.ts‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,38 @@ export function resolveEffectiveViewportRect(
150150
return resolveViewportRect(nodes, node.rect ?? { x: 0, y: 0, width: 0, height: 0 });
151151
}
152152

153+
/** The concrete `scroll <direction>` that brings an off-screen target into view. */
154+
export type OffscreenScrollDirection = 'up' | 'down' | 'left' | 'right';
155+
156+
/**
157+
* The direction to `scroll` so an off-screen target comes into view, from its
158+
* rect and effective viewport. Follows the reveal convention the CLI hints
159+
* already use (`cli-help.ts`): off-screen below -> scroll down, above -> up, and
160+
* the horizontal mirror. When a target sits off more than one edge (a diagonal
161+
* closed drawer), the axis with the largest overshoot wins so the agent's first
162+
* move is the dominant one. Returns `null` when the rect is not fully past any
163+
* edge — i.e. it is not off-screen on a single axis — leaving the caller's
164+
* generic "scroll toward it" phrasing in place.
165+
*/
166+
export function classifyOffscreenScrollDirection(
167+
targetRect: Rect,
168+
viewportRect: Rect,
169+
): OffscreenScrollDirection | null {
170+
const overshoots: Array<{ direction: OffscreenScrollDirection; amount: number }> = [
171+
{ direction: 'down', amount: targetRect.y - (viewportRect.y + viewportRect.height) },
172+
{ direction: 'up', amount: viewportRect.y - (targetRect.y + targetRect.height) },
173+
{ direction: 'right', amount: targetRect.x - (viewportRect.x + viewportRect.width) },
174+
{ direction: 'left', amount: viewportRect.x - (targetRect.x + targetRect.width) },
175+
];
176+
let best: { direction: OffscreenScrollDirection; amount: number } | null = null;
177+
for (const candidate of overshoots) {
178+
if (candidate.amount > 0 && (!best || candidate.amount > best.amount)) {
179+
best = candidate;
180+
}
181+
}
182+
return best?.direction ?? null;
183+
}
184+
153185
function deriveContainerHints(
154186
allNodes: SnapshotNode[],
155187
offscreenNodes: SnapshotNode[],

‎src/utils/__tests__/mobile-snapshot-semantics.test.ts‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { test } from 'vitest';
22
import assert from 'node:assert/strict';
33
import {
44
buildMobileSnapshotPresentation,
5+
classifyOffscreenScrollDirection,
56
isNodeVisibleInEffectiveViewport,
67
} from '../../snapshot/mobile-snapshot-semantics.ts';
78
import type { SnapshotNode } from '../../kernel/snapshot.ts';
@@ -324,6 +325,42 @@ test('visibility is false for node just outside viewport boundary', () => {
324325
assert.equal(isNodeVisibleInEffectiveViewport(nodes[1]!, nodes), false);
325326
});
326327

328+
test('classifyOffscreenScrollDirection names the reveal direction per edge', () => {
329+
const viewport = { x: 0, y: 0, width: 400, height: 800 };
330+
// Below the viewport bottom -> scroll down; above the top -> scroll up.
331+
assert.equal(
332+
classifyOffscreenScrollDirection({ x: 20, y: 1200, width: 120, height: 44 }, viewport),
333+
'down',
334+
);
335+
assert.equal(
336+
classifyOffscreenScrollDirection({ x: 20, y: -100, width: 120, height: 44 }, viewport),
337+
'up',
338+
);
339+
// Horizontal mirror: fully left of the viewport -> scroll left; right -> right.
340+
assert.equal(
341+
classifyOffscreenScrollDirection({ x: -320, y: 240, width: 300, height: 50 }, viewport),
342+
'left',
343+
);
344+
assert.equal(
345+
classifyOffscreenScrollDirection({ x: 500, y: 240, width: 120, height: 44 }, viewport),
346+
'right',
347+
);
348+
});
349+
350+
test('classifyOffscreenScrollDirection picks the dominant axis and ignores on-screen rects', () => {
351+
const viewport = { x: 0, y: 0, width: 400, height: 800 };
352+
// Off both the bottom (overshoot 400) and the right (overshoot 100): down wins.
353+
assert.equal(
354+
classifyOffscreenScrollDirection({ x: 500, y: 1200, width: 120, height: 44 }, viewport),
355+
'down',
356+
);
357+
// Overlapping the viewport on every edge is not off-screen on a single axis.
358+
assert.equal(
359+
classifyOffscreenScrollDirection({ x: 20, y: 40, width: 120, height: 44 }, viewport),
360+
null,
361+
);
362+
});
363+
327364
test('mobile presentation infers hidden content from vertical scroll indicator value at top', () => {
328365
const nodes: SnapshotNode[] = [
329366
{

0 commit comments

Comments
 (0)