Skip to content

Commit c8baeb3

Browse files
airowethymikee
authored andcommitted
fix(scroll): keep Android directional swipe endpoints inside the resolved scrollable
A directional scroll centred its swipe on the window, so on an inset scrollable the drag could start on a header above the list and the command failed with scroll_no_progress. The daemon now resolves the scroll container from the current stored tree before the gesture and hands its rect to the owner. The shared planner keeps both endpoints inside the container's intersection with the padded frame: a span that fits is unchanged, one that leaves the band moves into it, and one longer than the band shrinks to it and reports `requestedPixels` beside the dispatched `pixels`. Android applies it after the keyboard clip, so the two compose. iOS plans in the runner and follows separately. Refs callstack#2714
1 parent 82c0d70 commit c8baeb3

8 files changed

Lines changed: 353 additions & 25 deletions

File tree

‎packages/contracts/src/scroll-command.ts‎

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { AppError } from '@agent-device/kernel/errors';
22
import type { SettleObservation } from './interaction.ts';
33
import type { ScrollDirection } from './scroll-gesture.ts';
4+
import type { Rect } from '@agent-device/kernel/snapshot';
45

56
export const SCROLL_DURATION_MAX_MS = 10_000;
67
export const DEFAULT_MOBILE_SCROLL_DURATION_MS = 300;
@@ -38,13 +39,24 @@ export type ScrollTimingOptions = {
3839

3940
export type ScrollCommandOptions = ScrollDistanceOptions & ScrollTimingOptions;
4041

41-
export type ScrollExecutionOptions = ScrollCommandOptions & {
42-
releaseBehavior?: ScrollReleaseBehavior;
42+
/**
43+
* The scroll container the caller resolved from the current tree, in the snapshot's absolute space.
44+
* An owner that plans its own swipe keeps both endpoints inside it (#2714); an owner without that
45+
* planner ignores it.
46+
*/
47+
export type ScrollContainerTarget = {
48+
containerRect?: Rect;
4349
};
4450

45-
export type ResolvedScrollExecutionOptions = ScrollCommandOptions & {
46-
releaseBehavior: ScrollReleaseBehavior;
47-
};
51+
export type ScrollExecutionOptions = ScrollCommandOptions &
52+
ScrollContainerTarget & {
53+
releaseBehavior?: ScrollReleaseBehavior;
54+
};
55+
56+
export type ResolvedScrollExecutionOptions = ScrollCommandOptions &
57+
ScrollContainerTarget & {
58+
releaseBehavior: ScrollReleaseBehavior;
59+
};
4860

4961
export function resolveScrollExecutionOptions(
5062
options: ScrollCommandOptions,
Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
import { test } from 'vitest';
2+
import assert from 'node:assert/strict';
3+
import type { Rect } from '@agent-device/kernel/snapshot';
4+
import { buildScrollGesturePlan, type ScrollDirection } from './scroll-gesture.ts';
5+
6+
// The iPhone geometry #2714 was reported on: a 402x874 frame, the default 10% edge padding leaving
7+
// y 87..787 for a swipe.
8+
const FRAME = { referenceWidth: 402, referenceHeight: 874 };
9+
const PADDED_TOP = 87;
10+
const PADDED_BOTTOM = 787;
11+
const INSET_CONTAINER: Rect = { x: 18, y: 178, width: 366, height: 662 };
12+
const FITTING_CONTAINER: Rect = { x: 18, y: 62, width: 366, height: 729 };
13+
14+
function endpointsInside(
15+
plan: { x1: number; y1: number; x2: number; y2: number },
16+
rect: Rect,
17+
): boolean {
18+
const inside = (x: number, y: number) =>
19+
x >= rect.x && x <= rect.x + rect.width && y >= rect.y && y <= rect.y + rect.height;
20+
return inside(plan.x1, plan.y1) && inside(plan.x2, plan.y2);
21+
}
22+
23+
test.each<ScrollDirection>(['up', 'down'])(
24+
'a %s swipe longer than an inset container shrinks to the container and reports both spans',
25+
(direction) => {
26+
const unclamped = buildScrollGesturePlan({ direction, amount: 0.75, ...FRAME });
27+
const plan = buildScrollGesturePlan({
28+
direction,
29+
amount: 0.75,
30+
...FRAME,
31+
container: INSET_CONTAINER,
32+
});
33+
34+
assert.ok(endpointsInside(plan, INSET_CONTAINER), JSON.stringify(plan));
35+
assert.deepEqual(
36+
[Math.min(plan.y1, plan.y2), Math.max(plan.y1, plan.y2)],
37+
[INSET_CONTAINER.y, PADDED_BOTTOM],
38+
);
39+
assert.equal(plan.pixels, PADDED_BOTTOM - INSET_CONTAINER.y);
40+
assert.equal(plan.requestedPixels, unclamped.pixels);
41+
assert.equal(Math.sign(plan.y2 - plan.y1), Math.sign(unclamped.y2 - unclamped.y1));
42+
},
43+
);
44+
45+
test.each<ScrollDirection>(['up', 'down', 'left', 'right'])(
46+
'a %s swipe already inside its container keeps today’s endpoints and distance',
47+
(direction) => {
48+
const unclamped = buildScrollGesturePlan({ direction, amount: 0.75, ...FRAME });
49+
const plan = buildScrollGesturePlan({
50+
direction,
51+
amount: 0.75,
52+
...FRAME,
53+
container: FITTING_CONTAINER,
54+
});
55+
56+
assert.deepEqual(plan, unclamped);
57+
assert.equal('requestedPixels' in plan, false);
58+
},
59+
);
60+
61+
test('a swipe that fits an offset container moves into it without shortening', () => {
62+
const lowerContainer: Rect = { x: 0, y: 300, width: 402, height: 574 };
63+
const unclamped = buildScrollGesturePlan({ direction: 'down', amount: 0.5, ...FRAME });
64+
const plan = buildScrollGesturePlan({
65+
direction: 'down',
66+
amount: 0.5,
67+
...FRAME,
68+
container: lowerContainer,
69+
});
70+
71+
assert.ok(endpointsInside(plan, lowerContainer), JSON.stringify(plan));
72+
assert.equal(Math.min(plan.y1, plan.y2), lowerContainer.y);
73+
assert.equal(Math.abs(plan.y1 - plan.y2), Math.abs(unclamped.y1 - unclamped.y2));
74+
assert.equal(plan.pixels, unclamped.pixels);
75+
assert.equal('requestedPixels' in plan, false);
76+
});
77+
78+
test('a vertical swipe beside a narrow container runs down its centre line', () => {
79+
const rightColumn: Rect = { x: 250, y: 0, width: 152, height: 874 };
80+
const unclamped = buildScrollGesturePlan({ direction: 'up', ...FRAME });
81+
const plan = buildScrollGesturePlan({ direction: 'up', ...FRAME, container: rightColumn });
82+
83+
assert.equal(plan.x1, rightColumn.x + rightColumn.width / 2);
84+
assert.equal(plan.x2, plan.x1);
85+
assert.deepEqual([plan.y1, plan.y2, plan.pixels], [unclamped.y1, unclamped.y2, unclamped.pixels]);
86+
});
87+
88+
test('a container lying wholly in the edge padding leaves the padded swipe as it was', () => {
89+
const statusBand: Rect = { x: 0, y: 0, width: 402, height: PADDED_TOP - 1 };
90+
const unclamped = buildScrollGesturePlan({ direction: 'up', ...FRAME });
91+
const plan = buildScrollGesturePlan({ direction: 'up', ...FRAME, container: statusBand });
92+
93+
assert.deepEqual(plan, unclamped);
94+
});

‎packages/contracts/src/scroll-gesture.ts‎

Lines changed: 68 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,12 @@ export type ScrollGestureOptions = {
4949
pixels?: number;
5050
referenceWidth: number;
5151
referenceHeight: number;
52+
/**
53+
* The scroll container the swipe is meant for, in the reference frame's own zero-origin space.
54+
* Both endpoints stay inside its intersection with the padded frame (#2714): a span that already
55+
* fits is unchanged, one that leaves it is moved in, and one longer than it is shortened.
56+
*/
57+
container?: Rect;
5258
};
5359

5460
export type ScrollGesturePlan = {
@@ -60,7 +66,10 @@ export type ScrollGesturePlan = {
6066
referenceWidth: number;
6167
referenceHeight: number;
6268
amount?: number;
69+
/** The span dispatched, which is less than `requestedPixels` when the container shortened it. */
6370
pixels: number;
71+
/** Present only when the container shortened the swipe: the span the padded frame would carry. */
72+
requestedPixels?: number;
6473
};
6574

6675
export type SwipePresetGesturePlan = {
@@ -125,16 +134,65 @@ export function buildScrollGesturePlan(options: ScrollGestureOptions): ScrollGes
125134
pixels: travelPixels,
126135
});
127136

128-
switch (direction) {
129-
case 'up':
130-
return buildPlan(centerX, centerY - halfTravel, centerX, centerY + halfTravel);
131-
case 'down':
132-
return buildPlan(centerX, centerY + halfTravel, centerX, centerY - halfTravel);
133-
case 'left':
134-
return buildPlan(centerX - halfTravel, centerY, centerX + halfTravel, centerY);
135-
case 'right':
136-
return buildPlan(centerX + halfTravel, centerY, centerX - halfTravel, centerY);
137-
}
137+
const vertical = direction === 'up' || direction === 'down';
138+
const unclamped = {
139+
axisStart: (vertical ? centerY : centerX) - halfTravel,
140+
axisEnd: (vertical ? centerY : centerX) + halfTravel,
141+
cross: vertical ? centerX : centerY,
142+
};
143+
const lane = options.container
144+
? keepSwipeInContainer(unclamped, options.container, vertical, {
145+
min: edgePadding,
146+
max: axisLength - edgePadding,
147+
})
148+
: unclamped;
149+
const shortened = lane.axisEnd - lane.axisStart < unclamped.axisEnd - unclamped.axisStart;
150+
const plan = buildPlan(
151+
vertical ? lane.cross : lane.axisStart,
152+
vertical ? lane.axisStart : lane.cross,
153+
vertical ? lane.cross : lane.axisEnd,
154+
vertical ? lane.axisEnd : lane.cross,
155+
);
156+
const placed = shortened
157+
? { ...plan, pixels: lane.axisEnd - lane.axisStart, requestedPixels: travelPixels }
158+
: plan;
159+
// `up`/`left` drag from the low end toward the high end; `down`/`right` drag the other way.
160+
return direction === 'up' || direction === 'left'
161+
? placed
162+
: { ...placed, x1: placed.x2, y1: placed.y2, x2: placed.x1, y2: placed.y1 };
163+
}
164+
165+
type SwipeLane = { axisStart: number; axisEnd: number; cross: number };
166+
167+
/**
168+
* Moves a centred swipe lane inside the container's intersection with the padded axis, shortening
169+
* it only when the intersection is shorter than the lane. A container wholly inside the padding
170+
* leaves nothing to swipe in, so the lane stays as the padded frame planned it.
171+
*/
172+
function keepSwipeInContainer(
173+
lane: SwipeLane,
174+
container: Rect,
175+
vertical: boolean,
176+
padded: { min: number; max: number },
177+
): SwipeLane {
178+
const containerMin = vertical ? container.y : container.x;
179+
const containerSize = vertical ? container.height : container.width;
180+
const bandMin = Math.max(padded.min, Math.round(containerMin));
181+
const bandMax = Math.min(padded.max, Math.round(containerMin + containerSize));
182+
if (bandMax <= bandMin) return lane;
183+
184+
const crossMin = vertical ? container.x : container.y;
185+
const crossSize = vertical ? container.width : container.height;
186+
const cross =
187+
lane.cross >= crossMin && lane.cross <= crossMin + crossSize
188+
? lane.cross
189+
: Math.round(crossMin + crossSize / 2);
190+
191+
const length = lane.axisEnd - lane.axisStart;
192+
if (length >= bandMax - bandMin) return { axisStart: bandMin, axisEnd: bandMax, cross };
193+
const shift =
194+
lane.axisStart < bandMin ? bandMin - lane.axisStart : Math.min(0, bandMax - lane.axisEnd);
195+
return { axisStart: lane.axisStart + shift, axisEnd: lane.axisEnd + shift, cross };
138196
}
139197

140198
/**

‎packages/daemon/src/execution/__tests__/scroll-runtime.test.ts‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -786,3 +786,70 @@ test('bound scroll will not bill an unread earlier gesture to the scroll it is d
786786
assert.equal(result.movement, 'unobserved');
787787
assert.equal(captures.calls(), 0);
788788
});
789+
790+
/**
791+
* The container a directional swipe is planned inside (#2714): read off the same current tree the
792+
* movement claim compares against, before the gesture, and never off a tree that may no longer be
793+
* the screen.
794+
*/
795+
async function containerHandedToOwner(
796+
positionals: string[],
797+
session: SessionState,
798+
flags?: CommandFlags,
799+
): Promise<unknown> {
800+
const calls: ScrollCall[] = [];
801+
await runScroll(
802+
positionals,
803+
{},
804+
{
805+
scroll: async (direction, options) => {
806+
calls.push({ direction, options });
807+
return { ...OBSERVED_SWIPE };
808+
},
809+
},
810+
{ session, ...(flags === undefined ? {} : { flags }) },
811+
);
812+
const [call] = calls;
813+
assert.ok(call && calls.length === 1, `expected one scroll, got ${calls.length}`);
814+
return (call.options as { containerRect?: unknown }).containerRect;
815+
}
816+
817+
test.each<[string, CommandFlags | undefined]>([
818+
['an observed scroll', undefined],
819+
['a --settle scroll', { settle: true }],
820+
])('bound scroll hands %s the container resolved from the current tree', async (_, flags) => {
821+
const containerRect = await containerHandedToOwner(
822+
['down', '0.75'],
823+
sessionWithStoredScreen(automationScreen(0)),
824+
flags,
825+
);
826+
827+
assert.deepEqual(containerRect, SCREEN_CONTAINER);
828+
});
829+
830+
test('bound scroll hands no container from a tree that predates the last mutation', async () => {
831+
const session = sessionWithStoredScreen(automationScreen(0));
832+
expireRefFrame(session);
833+
834+
assert.equal(await containerHandedToOwner(['down', '0.75'], session), undefined);
835+
});
836+
837+
test('bound scroll hands no container for a horizontal swipe the edge analyzer cannot place', async () => {
838+
const containerRect = await containerHandedToOwner(
839+
['left'],
840+
sessionWithStoredScreen(automationScreen(0)),
841+
);
842+
843+
assert.equal(containerRect, undefined);
844+
});
845+
846+
test('bound scroll hands no container to a desktop owner that never synthesizes a swipe', async () => {
847+
const session = makeMacOsSession('scroll-desktop');
848+
session.snapshot = buildSnapshotState(
849+
{ nodes: automationScreen(0), backend: 'xctest', producer: 'apple-runner' },
850+
undefined,
851+
);
852+
activateCompleteRefFrame(session);
853+
854+
assert.equal(await containerHandedToOwner(['down', '0.75'], session), undefined);
855+
});

‎packages/daemon/src/execution/scroll-movement.ts‎

Lines changed: 31 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -164,17 +164,40 @@ export function readScrollSurfaceBaseline(
164164
function freezeScrollSurfaceBaseline(session: SessionState): ScrollMovementPlan {
165165
const baseline = readScrollSurfaceBaseline(session.snapshot);
166166
if (!baseline) return { kind: 'unobservable', reason: 'stored-surface-unusable' };
167-
if (refFrameState(session) !== 'active') {
168-
return { kind: 'unobservable', reason: 'stored-surface-not-current' };
169-
}
170-
if (isPostGestureStabilizationPending(session)) {
171-
// A gesture nobody has read yet may still be moving this tree, which would bill its motion to
172-
// the scroll being dispatched now.
173-
return { kind: 'unobservable', reason: 'prior-gesture-unsettled' };
174-
}
167+
const staleness = storedSurfaceStaleness(session);
168+
if (staleness) return { kind: 'unobservable', reason: staleness };
175169
return { kind: 'observe', baseline };
176170
}
177171

172+
function storedSurfaceStaleness(
173+
session: SessionState,
174+
): Exclude<ScrollSurfaceBaselineAbsence, 'stored-surface-unusable'> | undefined {
175+
if (refFrameState(session) !== 'active') return 'stored-surface-not-current';
176+
// A gesture nobody has read yet may still be moving this tree, which would bill its motion to
177+
// the scroll being dispatched now.
178+
if (isPostGestureStabilizationPending(session)) return 'prior-gesture-unsettled';
179+
return undefined;
180+
}
181+
182+
/**
183+
* The scroll container a directional swipe should stay inside (#2714), read before the gesture off
184+
* the same current tree a movement claim would compare against, through the edge verdict's own
185+
* selection. A tree that may no longer be the screen names no container, and neither does a
186+
* horizontal scroll, which the vertical-only edge analyzer cannot place, nor a platform that moves
187+
* content without a synthesized swipe.
188+
*/
189+
export async function readScrollTargetContainer(
190+
device: DeviceInfo,
191+
session: SessionState,
192+
direction: ScrollDirection,
193+
): Promise<Rect | undefined> {
194+
if (!isMobilePlatform(device)) return undefined;
195+
const edge = verticalEdgeFor(direction);
196+
const nodes = session.snapshot?.nodes;
197+
if (!edge || !nodes || nodes.length === 0 || storedSurfaceStaleness(session)) return undefined;
198+
return (await readScrollEdgeState(nodes, edge)).containerRect;
199+
}
200+
178201
/**
179202
* Observes one directional scroll and returns the movement its response may claim.
180203
*

‎packages/daemon/src/execution/scroll-runtime.ts‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ import {
4747
import {
4848
observeScrollMovement,
4949
planScrollMovement,
50+
readScrollTargetContainer,
5051
reportScrollMovementNotApplicable,
5152
reportScrollMovementUnobserved,
5253
type ScrollMovementPlan,
@@ -145,10 +146,23 @@ export async function resolveBoundScrollRuntime(
145146
flags: params.flags,
146147
session: params.session,
147148
});
149+
const containerRect = await readScrollTargetContainer(
150+
params.device,
151+
params.session,
152+
target.direction,
153+
);
154+
const directionOptions =
155+
containerRect === undefined ? options : { ...options, containerRect };
148156
return await resolveBoundGenericRuntime(
149157
{ ...admission, use: plan.use },
150158
async (runtime, dispatchContext) =>
151-
await executeDirectionScroll(runtime, target, options, dispatchContext, movementPlan),
159+
await executeDirectionScroll(
160+
runtime,
161+
target,
162+
directionOptions,
163+
dispatchContext,
164+
movementPlan,
165+
),
152166
);
153167
}
154168
case 'edge': {

0 commit comments

Comments
 (0)