Skip to content

Commit 0e99f4d

Browse files
committed
fix(scroll): name what a scroll observed and read an edge off one tree
Two pieces the directional-scroll answer needs, with no behavior change on its own. `movement` is the vocabulary a scroll uses to say what happened to the surface: `moved`, `at-edge`, `unchanged` when the visible content did not change, and `unobserved` when the command could not compare anything. It is absent when the runtime cannot read a screen at all, so a caller never has to guess whether a missing claim means "did not move". `readScrollEdgeState` is the one pure door into the edge analyzer — the question `canScrollFurtherAtEdge` and `scrollSurfaceFingerprint` already ask, exposed with the frame of the container that answer was taken on. Without `containerRect`, `canScroll: false` cannot distinguish content that ends here from a tree that named no container, and a caller checking whether its gesture landed inside the list has nothing to check against.
1 parent ca2970b commit 0e99f4d

7 files changed

Lines changed: 288 additions & 21 deletions

File tree

‎packages/capture-kit/src/snapshot/__tests__/scroll-edge-state-pass-orchestration.test.ts‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,63 @@ test('an amount-based message names the honored travel when the planner reports
7777
);
7878
});
7979

80+
/**
81+
* #2714. A travel figure describes the swipe that was dispatched, not content that moved, so an
82+
* observation of no change outranks it. `unobserved` is the pair to these: the disclosure travels in
83+
* the `movement` field, and the prose keeps the number the caller asked for.
84+
*/
85+
test('formatScrollEdgeMessage: an observed edge answers without a distance', () => {
86+
assert.equal(
87+
formatScrollEdgeMessage({ direction: 'down', passes: 1, amount: 0.75, movement: 'at-edge' }),
88+
'Scrolled down and no hidden content below was detected',
89+
);
90+
assert.equal(
91+
formatScrollEdgeMessage({ direction: 'up', passes: 1, amount: 0.75, movement: 'at-edge' }),
92+
'Scrolled up and no hidden content above was detected',
93+
);
94+
});
95+
96+
test('formatScrollEdgeMessage: a horizontal scroll never claims a vertical edge', () => {
97+
assert.equal(
98+
formatScrollEdgeMessage({ direction: 'left', passes: 1, amount: 0.75, movement: 'at-edge' }),
99+
'Scrolled left by 0.75',
100+
);
101+
});
102+
103+
test('formatScrollEdgeMessage: an unchanged surface reports the measurement, not the request', () => {
104+
assert.equal(
105+
formatScrollEdgeMessage({ direction: 'left', passes: 1, pixels: 500, movement: 'unchanged' }),
106+
'Scrolled left and the visible content did not change',
107+
);
108+
});
109+
110+
test('formatScrollEdgeMessage: an observation outranks the distance it disagrees with', () => {
111+
assert.equal(
112+
formatScrollEdgeMessage({
113+
direction: 'down',
114+
passes: 1,
115+
amount: 3,
116+
pixels: 5000,
117+
honoredPixels: 640,
118+
movement: 'at-edge',
119+
}),
120+
'Scrolled down and no hidden content below was detected',
121+
);
122+
});
123+
124+
test('formatScrollEdgeMessage: an unobserved movement keeps the distance the caller asked for', () => {
125+
assert.equal(
126+
formatScrollEdgeMessage({
127+
direction: 'down',
128+
passes: 1,
129+
amount: 0.75,
130+
honoredPixels: 656,
131+
movement: 'unobserved',
132+
}),
133+
'Scrolled down by 0.75 of the viewport (656px)',
134+
);
135+
});
136+
80137
// ---------------------------------------------------------------------------
81138
// captureScrollEdgeState: retry-without-scope on an empty scoped capture
82139
// ---------------------------------------------------------------------------
Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
import { test } from 'vitest';
2+
import assert from 'node:assert/strict';
3+
import { captureScrollEdgeState, readScrollEdgeState } from '../scroll-edge-state.ts';
4+
import { scrollNode, windowRoot } from './scroll-edge-state-fixtures.ts';
5+
6+
/**
7+
* The pure door into the edge analyzer. `scroll <dir>` reads an edge decision off a tree it already
8+
* holds to say what its gesture did (#2714), which makes `containerRect` part of the answer: the
9+
* caller has to be able to tell "this content ends here" from "the tree named no container", and
10+
* `canScroll: false` alone cannot tell them apart.
11+
*/
12+
13+
const CONTAINER = { x: 18, y: 178, width: 366, height: 662 };
14+
15+
test('readScrollEdgeState: a resolved container reports the frame the decision was taken on', async () => {
16+
const nodes = [
17+
windowRoot(),
18+
scrollNode(1, { hiddenContentBelow: true, rect: CONTAINER }),
19+
{
20+
ref: 'e3',
21+
index: 2,
22+
parentIndex: 1,
23+
type: 'StaticText',
24+
label: 'Row',
25+
rect: { x: 24, y: 700, width: 300, height: 40 },
26+
},
27+
];
28+
29+
const state = await readScrollEdgeState(nodes, 'bottom');
30+
31+
assert.equal(state.canScroll, true);
32+
assert.deepEqual(state.containerRect, CONTAINER);
33+
});
34+
35+
test('readScrollEdgeState: a tree with no scroll container reports no frame to check a gesture against', async () => {
36+
const nodes = [
37+
windowRoot(),
38+
{
39+
ref: 'e2',
40+
index: 1,
41+
parentIndex: 0,
42+
type: 'Button',
43+
label: 'Tap me',
44+
rect: { x: 20, y: 40, width: 100, height: 40 },
45+
},
46+
];
47+
48+
const state = await readScrollEdgeState(nodes, 'bottom');
49+
50+
assert.equal(state.canScroll, false);
51+
assert.equal(state.containerRect, undefined);
52+
});
53+
54+
test('readScrollEdgeState: a zero-area scrollable resolves to no container and therefore no frame', async () => {
55+
const state = await readScrollEdgeState(
56+
[windowRoot(), scrollNode(1, { hiddenContentBelow: true, rect: { ...CONTAINER, height: 0 } })],
57+
'bottom',
58+
);
59+
60+
assert.equal(state.containerRect, undefined);
61+
});
62+
63+
/**
64+
* The reader must not become a second source of truth: the captured and pure doors disagreeing about
65+
* one tree would mean an edge loop and a movement claim reading different rules.
66+
*/
67+
test('readScrollEdgeState answers what captureScrollEdgeState answers for the same tree', async () => {
68+
const nodes = [windowRoot(), scrollNode(1, { hiddenContentBelow: true, rect: CONTAINER })];
69+
70+
const captured = await captureScrollEdgeState({
71+
edge: 'bottom',
72+
captureNodes: async () => nodes,
73+
});
74+
const read = await readScrollEdgeState(nodes, 'bottom');
75+
76+
assert.equal(read.canScroll, captured.canScroll);
77+
assert.deepEqual(read.containerRect, captured.containerRect);
78+
assert.equal(read.fingerprint, captured.fingerprint);
79+
});

‎packages/capture-kit/src/snapshot/scroll-edge-state.ts‎

Lines changed: 78 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,31 @@
11
import { AppError } from '@agent-device/kernel/errors';
2+
import type { ScrollMovementObservation } from '@agent-device/contracts/scroll-command';
23
import type { ScrollDirection } from '@agent-device/contracts/scroll-gesture';
3-
import type { Point, RawSnapshotNode, SnapshotNode } from '@agent-device/kernel/snapshot';
4+
import type { Point, RawSnapshotNode, Rect, SnapshotNode } from '@agent-device/kernel/snapshot';
45

56
export type ScrollEdge = 'top' | 'bottom';
67

8+
/**
9+
* The end-of-content analyzer reads vertical edges only, so a horizontal scroll has no end signal
10+
* and is bounded by whatever pass or observation budget its caller owns.
11+
*/
12+
export function verticalEdgeFor(direction: ScrollDirection): ScrollEdge | undefined {
13+
if (direction === 'down') return 'bottom';
14+
if (direction === 'up') return 'top';
15+
return undefined;
16+
}
17+
718
export type ScrollEdgeState = {
819
canScroll: boolean;
920
emptySnapshot: boolean;
1021
scope?: string;
22+
/**
23+
* The frame of the container every other field on this record describes, absent when the tree
24+
* named no usable scroll container at all. Without it `canScroll: false` cannot be read as
25+
* "already at this edge": a tree that names no scroller and a scroller with nothing left to
26+
* reveal answer the same way, and only one of them is the end of the content (#2714).
27+
*/
28+
containerRect?: Rect;
1129
/**
1230
* A cheap signature of what is on screen right now. Two consecutive captures with the same
1331
* fingerprint but `canScroll` still true mean the scroll did not move the container — the actuator
@@ -55,6 +73,19 @@ export async function captureScrollEdgeState(params: {
5573
}
5674
}
5775

76+
/**
77+
* Everything one tree can answer about this edge — hidden content left, the container that answer
78+
* was taken on, and the surface signature — with no capture. The one pure door into the analyzer,
79+
* so every reader of an edge decision goes through the same selection rules.
80+
*/
81+
export async function readScrollEdgeState(
82+
nodes: readonly (RawSnapshotNode | SnapshotNode)[],
83+
edge: ScrollEdge,
84+
): Promise<ScrollEdgeState> {
85+
const { analyzeScrollEdgeState } = await import('./scroll-edge-state/selection.ts');
86+
return analyzeScrollEdgeState(nodes, edge);
87+
}
88+
5889
/**
5990
* Is there hidden content left at this edge? The same question `runScrollEdgePasses` loops on,
6091
* exposed for callers with their own stop condition (`scroll --until`) so both read one signal.
@@ -63,8 +94,7 @@ export async function canScrollFurtherAtEdge(
6394
nodes: readonly (RawSnapshotNode | SnapshotNode)[],
6495
edge: ScrollEdge,
6596
): Promise<boolean> {
66-
const { analyzeScrollEdgeState } = await import('./scroll-edge-state/selection.ts');
67-
return analyzeScrollEdgeState(nodes, edge).canScroll;
97+
return (await readScrollEdgeState(nodes, edge)).canScroll;
6898
}
6999

70100
/**
@@ -167,6 +197,10 @@ export async function runScrollEdgePasses<TResult>(params: {
167197
* travel that was asked for: one gesture cannot cross more than the viewport axis minus its edge
168198
* padding, so a large `amount` saturates. Naming the honored distance is what keeps
169199
* `scroll down 3` from reporting a three-viewport scroll it never performed.
200+
*
201+
* `movement` is what the owner measured after the gesture (#2714), and it outranks the requested
202+
* distance: a directional scroll that observed no change cannot answer with a travel figure, since
203+
* that number describes the swipe that was dispatched, not content that moved.
170204
*/
171205
export function formatScrollEdgeMessage(params: {
172206
direction: ScrollDirection;
@@ -175,12 +209,25 @@ export function formatScrollEdgeMessage(params: {
175209
amount?: number | undefined;
176210
pixels?: number | undefined;
177211
honoredPixels?: number | undefined;
212+
movement?: ScrollMovementObservation | undefined;
178213
}): string {
179-
const { direction, edge, passes, amount, pixels, honoredPixels } = params;
214+
const { direction, edge, passes, amount, pixels, honoredPixels, movement } = params;
180215
if (edge && passes === 0) {
181216
return `Already at ${edge}; no hidden content ${edge === 'bottom' ? 'below' : 'above'} detected`;
182217
}
183218
if (edge) return `Scrolled to ${edge} with ${passes} ${direction} passes`;
219+
const verticalEdge = verticalEdgeFor(direction);
220+
if (movement === 'at-edge' && verticalEdge) {
221+
// The same honesty the zero-pass branch above holds: the analyzer reports no hidden content
222+
// DETECTED, which is not a claim that the content provably ends here. A horizontal scroll has
223+
// no edge signal to report, so it falls through to the distance it was asked for.
224+
return `Scrolled ${direction} and no hidden content ${
225+
verticalEdge === 'bottom' ? 'below' : 'above'
226+
} was detected`;
227+
}
228+
if (movement === 'unchanged') {
229+
return `Scrolled ${direction} and the visible content did not change`;
230+
}
184231
if (pixels !== undefined) return `Scrolled ${direction} by ${honoredPixels ?? pixels}px`;
185232
if (amount !== undefined) {
186233
return honoredPixels === undefined
@@ -199,13 +246,37 @@ function buildScrollEdgeNoProgressError(edge: ScrollEdge, passes: number): AppEr
199246
reason: 'scroll_edge_no_progress',
200247
edge,
201248
passes,
202-
hint:
203-
`The scroll is not reaching this container. If a field is focused, dismiss the keyboard first; if it is nested inside another scroller, target it directly with scroll <dir> --until <selector>. ` +
204-
`Some lists ignore synthesized scrolls — a raw drag moves them: swipe x1 y1 x2 y2 started inside the list.`,
249+
hint: scrollNoProgressHint({ targetDirectly: 'with scroll <dir> --until <selector>' }),
205250
},
206251
);
207252
}
208253

254+
/**
255+
* What a caller does next when a scroll moved nothing while the container still had content to
256+
* reveal: one hint, because every loop that notices has the same recovery.
257+
*
258+
* `targetDirectly` names the form the failing command has for reaching an inner scroller — `scroll
259+
* --until` already IS that form, so it says only "target it directly". The raw-drag sentence belongs
260+
* only where the command can say where a swipe would land: a tvOS scroll is a remote keypress with no
261+
* coordinates to name, and advice nobody can follow is noise.
262+
*/
263+
export function scrollNoProgressHint(
264+
params: Readonly<{ targetDirectly?: string; rawDrag?: boolean }> = {},
265+
): string {
266+
const target =
267+
params.targetDirectly === undefined
268+
? 'target it directly'
269+
: `target it directly ${params.targetDirectly}`;
270+
const reached =
271+
`The scroll is not reaching this container. If a field is focused, dismiss the keyboard first; ` +
272+
`if it is nested inside another scroller, ${target}.`;
273+
if (params.rawDrag === false) return reached;
274+
return (
275+
reached +
276+
` Some lists ignore synthesized scrolls — a raw drag moves them: swipe x1 y1 x2 y2 started inside the list.`
277+
);
278+
}
279+
209280
function buildScrollEdgeVerificationError(
210281
edge: ScrollEdge,
211282
scope: string | undefined,

‎packages/capture-kit/src/snapshot/scroll-edge-state/selection.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,9 @@ export function analyzeScrollEdgeState(
3838
return {
3939
canScroll,
4040
emptySnapshot: false,
41+
// Every selection path admits only `isUsableRect` containers, so a resolved container always
42+
// has a frame to report and its absence always means no container was resolved.
43+
...(container.rect ? { containerRect: container.rect } : {}),
4144
scope: buildScrollContainerScope(container, nodes),
4245
fingerprint: buildSurfaceFingerprint(container, nodes),
4346
};

‎packages/contracts/src/scroll-command.test.ts‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import assert from 'node:assert/strict';
22
import { test } from 'vitest';
3-
import { resolveScrollExecutionOptions } from './scroll-command.ts';
3+
import { honoredScrollSwipeMidpoint, resolveScrollExecutionOptions } from './scroll-command.ts';
44

55
test('resolveScrollExecutionOptions keeps ordinary scrolls controlled', () => {
66
assert.deepEqual(resolveScrollExecutionOptions({ amount: 0.5 }), {
@@ -18,3 +18,21 @@ test('resolveScrollExecutionOptions marks edge scrolls inertial without changing
1818
releaseBehavior: 'inertial',
1919
});
2020
});
21+
22+
/**
23+
* The swipe midpoint is what lets a scroll ask whether its own gesture landed inside the container it
24+
* blamed for a no-op, so the pair that pins it is a leaf that reported coordinates and one that did
25+
* not: a tvOS scroll is a remote keypress, and guessing a midpoint there would invent evidence.
26+
*/
27+
test('honoredScrollSwipeMidpoint reports the middle of a swipe the leaf ran', () => {
28+
assert.deepEqual(honoredScrollSwipeMidpoint({ x1: 201, y1: 665, x2: 201, y2: 209 }), {
29+
x: 201,
30+
y: 437,
31+
});
32+
});
33+
34+
test('honoredScrollSwipeMidpoint declines to guess where the owner reported no gesture', () => {
35+
assert.equal(honoredScrollSwipeMidpoint({ button: 'down' }), undefined);
36+
assert.equal(honoredScrollSwipeMidpoint({ x1: 201, y1: 665, x2: 201 }), undefined);
37+
assert.equal(honoredScrollSwipeMidpoint({ x1: 201, y1: 665, x2: 201, y2: 'none' }), undefined);
38+
});

‎packages/contracts/src/scroll-command.ts‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,24 @@ export const DEFAULT_IOS_SCROLL_AMOUNT = 0.65;
99

1010
export type ScrollReleaseBehavior = 'controlled' | 'inertial';
1111

12+
/**
13+
* What a directional scroll actually SAW after its gesture, which is the only evidence that can
14+
* back the distance the same response reports (#2714).
15+
*
16+
* - `'moved'`: the post-gesture surface differs from the pre-gesture one, so content did move.
17+
* - `'at-edge'`: the surface is identical and the resolved container names no hidden content in
18+
* that direction — the scroll was a legitimate no-op at the end of the content.
19+
* - `'unchanged'`: the surface is identical and the direction has no end-of-content signal to read
20+
* (a horizontal scroll: the hidden-content analyzer only covers the vertical axis), so the
21+
* response says what it measured without guessing which of the two it was.
22+
* - `'unobserved'`: nothing comparable was available, so the distance rests on the gesture plan
23+
* alone. Callers that need the effect confirmed ask for a capture or a `--settle` observation.
24+
*
25+
* A directional scroll that measures an unchanged surface WITH hidden content still in that
26+
* direction does not answer at all: it fails with `scroll_no_progress`.
27+
*/
28+
export type ScrollMovementObservation = 'moved' | 'at-edge' | 'unchanged' | 'unobserved';
29+
1230
export type ScrollDistanceOptions = {
1331
amount?: number;
1432
pixels?: number;
@@ -95,6 +113,29 @@ export function honoredScrollDurationMs(
95113
return typeof result?.durationMs === 'number' ? result.durationMs : undefined;
96114
}
97115

116+
/**
117+
* Where the leaf's swipe ran, as the midpoint of the coordinates it reported — the same absolute space
118+
* its snapshots use, which is what lets a container rect say whether the gesture landed inside it.
119+
* An owner that reports no coordinates is answered `undefined` rather than guessed at: a tvOS scroll
120+
* is a remote keypress, so there is no midpoint to name.
121+
*/
122+
export function honoredScrollSwipeMidpoint(
123+
result: Record<string, unknown> | undefined,
124+
): { x: number; y: number } | undefined {
125+
const x1 = readReportedCoordinate(result?.x1);
126+
const y1 = readReportedCoordinate(result?.y1);
127+
const x2 = readReportedCoordinate(result?.x2);
128+
const y2 = readReportedCoordinate(result?.y2);
129+
if (x1 === undefined || y1 === undefined || x2 === undefined || y2 === undefined) {
130+
return undefined;
131+
}
132+
return { x: (x1 + x2) / 2, y: (y1 + y2) / 2 };
133+
}
134+
135+
function readReportedCoordinate(value: unknown): number | undefined {
136+
return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
137+
}
138+
98139
/**
99140
* `scroll` — the generic-route result built by `buildDispatchedScrollResult`
100141
* (src/core/dispatch-scroll.ts): the resolved direction, the edge-pass
@@ -118,6 +159,14 @@ export type ScrollCommandResult = {
118159
durationMs?: number;
119160
message?: string;
120161
settle?: SettleObservation;
162+
/**
163+
* The observation that gated this response's distance claim. See
164+
* {@link ScrollMovementObservation}: `scroll` answers with what it measured after the gesture,
165+
* and only `'moved'` and `'at-edge'` confirm the surface's fate. Absent on the tiers that verify
166+
* per pass instead of per gesture (`scroll top`/`bottom` and `--until`), and on platforms whose
167+
* scroll owner never dispatches a swipe (the Linux wheel).
168+
*/
169+
movement?: ScrollMovementObservation;
121170
/**
122171
* Set only when an on-screen keyboard made the owner clip the swipe into the band above it
123172
* (#2500). Absent means the swipe was not clipped, which is not the same claim as `false`: a

0 commit comments

Comments
 (0)