Skip to content

Commit f728698

Browse files
committed
fix(scroll): observe the movement a directional scroll can claim
1 parent a624ec1 commit f728698

4 files changed

Lines changed: 879 additions & 12 deletions

File tree

Lines changed: 315 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,315 @@
1+
import assert from 'node:assert/strict';
2+
import { test } from 'vitest';
3+
import type { SnapshotResult } from '@agent-device/contracts/interactor-types';
4+
import { buildSnapshotState } from '@agent-device/capture-kit/snapshot-state';
5+
import { AppError } from '@agent-device/kernel/errors';
6+
import type { CommandFlags } from '@agent-device/contracts/command';
7+
import type { Rect, SnapshotNode } from '@agent-device/kernel/snapshot';
8+
import {
9+
observeScrollMovement,
10+
readScrollSurfaceBaseline,
11+
type ScrollSurfaceBaseline,
12+
} from '../scroll-movement.ts';
13+
14+
/**
15+
* The observation that gates a directional scroll's distance claim (#2714). Node shapes follow the
16+
* live pair `interaction-surface-baseline-evidence.test.ts` transcribed: a scroller holding labelled
17+
* rows, plus fixed chrome that never moves.
18+
*
19+
* Every verdict here is paired with the closest input that must NOT produce it, because the whole
20+
* risk of this rule is failing a scroll that was fine: a no-op and a scroll at the end of its list
21+
* differ by one hidden-content hint, a no-op and a scroll over a busy screen differ by whether the
22+
* surface will hold still, and a no-op and a scroll whose content was replaced differ by the strict
23+
* no-effect bar rather than by the distance it names.
24+
*/
25+
26+
const CONTAINER: Rect = { x: 18, y: 178, width: 366, height: 662 };
27+
const SWIPE_MIDPOINT = { x: 201, y: 437 };
28+
const REQUESTED_PIXELS = 656;
29+
30+
/** The trees one case feeds the observer: a fixed list of them, or one that never repeats. */
31+
type SnapshotFrames = SnapshotNode[][] | ((attempt: number) => SnapshotNode[] | Error);
32+
33+
function screen(rowOffset: number, hiddenBelow = true): SnapshotNode[] {
34+
return [
35+
{
36+
type: 'ScrollView',
37+
identifier: 'lab-list',
38+
rect: CONTAINER,
39+
...(hiddenBelow ? { hiddenContentBelow: true } : {}),
40+
},
41+
{
42+
type: 'StaticText',
43+
label: 'Row one',
44+
rect: { x: 24, y: 200 + rowOffset, width: 300, height: 20 },
45+
},
46+
{
47+
type: 'StaticText',
48+
label: 'Row two',
49+
rect: { x: 24, y: 260 + rowOffset, width: 300, height: 20 },
50+
},
51+
{
52+
type: 'Button',
53+
identifier: 'automation-press',
54+
label: 'Press canary',
55+
rect: { x: 24, y: 780, width: 200, height: 44 },
56+
},
57+
] as SnapshotNode[];
58+
}
59+
60+
function state(nodes: SnapshotNode[], flags?: Partial<CommandFlags>) {
61+
return buildSnapshotState({ nodes, backend: 'xctest', producer: 'apple-runner' }, flags);
62+
}
63+
64+
function baselineOf(nodes: SnapshotNode[], flags?: Partial<CommandFlags>): ScrollSurfaceBaseline {
65+
const baseline = readScrollSurfaceBaseline(state(nodes, flags));
66+
if (!baseline) throw new Error('the fixture produced no comparable surface');
67+
return baseline;
68+
}
69+
70+
/**
71+
* The captures the observation is allowed to take. A case that under-provides frames fails loudly:
72+
* reusing the last frame would let a quiet pair form by accident, turning a budget case into a
73+
* no-op case that passes for the wrong reason.
74+
*/
75+
function captures(screens: SnapshotFrames): {
76+
calls: () => number;
77+
capture: () => Promise<SnapshotResult>;
78+
} {
79+
const next: (attempt: number) => SnapshotNode[] | Error =
80+
typeof screens === 'function' ? screens : queueReader(screens);
81+
let calls = 0;
82+
return {
83+
calls: () => calls,
84+
capture: async (): Promise<SnapshotResult> => {
85+
const frame = next(calls);
86+
calls += 1;
87+
if (frame instanceof Error) throw frame;
88+
return { nodes: frame, backend: 'xctest', producer: 'apple-runner' };
89+
},
90+
};
91+
}
92+
93+
function queueReader(frames: SnapshotNode[][]) {
94+
return (attempt: number): SnapshotNode[] => {
95+
const frame = frames[attempt];
96+
if (frame === undefined) {
97+
throw new Error('the observation captured more times than the case supplied frames for');
98+
}
99+
return frame;
100+
};
101+
}
102+
103+
function observe(params: {
104+
direction?: 'up' | 'down' | 'left' | 'right';
105+
baseline: ScrollSurfaceBaseline;
106+
screens: SnapshotFrames;
107+
midpoint?: { x: number; y: number };
108+
pixels?: number;
109+
budgetMs?: number;
110+
}) {
111+
const spy = captures(params.screens);
112+
const observation = observeScrollMovement({
113+
direction: params.direction ?? 'down',
114+
baseline: params.baseline,
115+
swipe: {
116+
midpoint: params.midpoint ?? SWIPE_MIDPOINT,
117+
pixels: params.pixels ?? REQUESTED_PIXELS,
118+
},
119+
capture: spy.capture,
120+
pollMs: 1,
121+
budgetMs: params.budgetMs ?? 5_000,
122+
});
123+
return { observation, spy };
124+
}
125+
126+
test('a surface that no longer holds the pre-gesture content answers moved on the first capture', async () => {
127+
const { observation, spy } = observe({
128+
baseline: baselineOf(screen(0)),
129+
screens: [screen(-300)],
130+
});
131+
132+
assert.equal(await observation, 'moved');
133+
// The cost claim of the whole feature: a scroll that worked is confirmed by one capture.
134+
assert.equal(spy.calls(), 1);
135+
});
136+
137+
test('a surface that never shifted while the container still hides content refuses with a typed reason', async () => {
138+
const { observation, spy } = observe({
139+
baseline: baselineOf(screen(0)),
140+
screens: [screen(0), screen(0)],
141+
});
142+
143+
await assert.rejects(
144+
() => observation,
145+
(error: unknown) =>
146+
error instanceof AppError &&
147+
error.code === 'COMMAND_FAILED' &&
148+
error.details?.reason === 'scroll_no_progress' &&
149+
error.details.direction === 'down' &&
150+
error.details.hiddenContentAt === 'bottom' &&
151+
error.details.requestedPixels === REQUESTED_PIXELS &&
152+
typeof error.details.hint === 'string' &&
153+
/swipe x1 y1 x2 y2/.test(error.details.hint),
154+
);
155+
// Refusing needs the surface at rest, which needs the second read.
156+
assert.equal(spy.calls(), 2);
157+
});
158+
159+
/**
160+
* The refusal above names a raw drag because it knows where the swipe ran. An owner that reports no
161+
* coordinates (a tvOS scroll is a remote keypress) must not be told to swipe where it cannot.
162+
*/
163+
test('a refusal without gesture coordinates does not recommend a swipe', async () => {
164+
const spy = captures([screen(0), screen(0)]);
165+
166+
await assert.rejects(
167+
() =>
168+
observeScrollMovement({
169+
direction: 'down',
170+
baseline: baselineOf(screen(0)),
171+
swipe: {},
172+
capture: spy.capture,
173+
pollMs: 1,
174+
budgetMs: 5_000,
175+
}),
176+
(error: unknown) =>
177+
error instanceof AppError && !/swipe x1 y1 x2 y2/.test(String(error.details?.hint)),
178+
);
179+
});
180+
181+
test('a surface that never shifted with nothing left to reveal answers at-edge, not a refusal', async () => {
182+
const { observation } = observe({
183+
baseline: baselineOf(screen(0)),
184+
screens: [screen(0, false), screen(0, false)],
185+
});
186+
187+
assert.equal(await observation, 'at-edge');
188+
});
189+
190+
test('a tree that names no scroll container is not read as the end of the content', async () => {
191+
const rowsOnly = screen(0).filter((node) => node.type !== 'ScrollView');
192+
const { observation } = observe({
193+
baseline: baselineOf(rowsOnly),
194+
screens: [rowsOnly, rowsOnly],
195+
});
196+
197+
assert.equal(await observation, 'unobserved');
198+
});
199+
200+
test('a container the gesture never ran inside is not blamed for the no-op', async () => {
201+
const { observation } = observe({
202+
baseline: baselineOf(screen(0)),
203+
screens: [screen(0), screen(0)],
204+
midpoint: { x: 201, y: 40 },
205+
});
206+
207+
assert.equal(await observation, 'unobserved');
208+
});
209+
210+
test('a horizontal scroll that moved nothing reports what it measured instead of refusing', async () => {
211+
const pager = [
212+
{ type: 'ScrollView', identifier: 'pager', rect: CONTAINER, hiddenContentBelow: true },
213+
{ type: 'StaticText', label: 'Page', rect: { x: 24, y: 400, width: 200, height: 20 } },
214+
] as SnapshotNode[];
215+
const { observation } = observe({
216+
direction: 'left',
217+
baseline: baselineOf(pager),
218+
screens: [pager, pager],
219+
});
220+
221+
assert.equal(await observation, 'unchanged');
222+
});
223+
224+
/**
225+
* A pair from different capture lineages is not two views of one screen — the XCTest-channel fallback
226+
* swaps the producer mid-request (#1569) — so its difference is neither movement nor a no-op. The
227+
* frames here genuinely differ: the gate has to hold on the path that would otherwise answer `moved`,
228+
* not only on the quiet one.
229+
*/
230+
test('a capture lineage that changed mid-request withholds the claim, even as the frames differ', async () => {
231+
const baseline = readScrollSurfaceBaseline({
232+
...state(screen(0)),
233+
comparisonKey: 'lineage-before',
234+
});
235+
if (!baseline) throw new Error('the fixture produced no comparable surface');
236+
const { observation, spy } = observe({
237+
baseline,
238+
screens: [screen(-300)],
239+
});
240+
241+
assert.equal(await observation, 'unobserved');
242+
assert.equal(spy.calls(), 1);
243+
});
244+
245+
/** The same gate on the other lineage axis: `snapshot -i` stored the baseline, and the broad tree this
246+
* command reads shares too little with it to refuse a scroll on — or to credit one. */
247+
test('a baseline captured interactively-only withholds the claim, even as the frames differ', async () => {
248+
const { observation } = observe({
249+
baseline: baselineOf(screen(0), { snapshotInteractiveOnly: true }),
250+
screens: [screen(-300)],
251+
});
252+
253+
assert.equal(await observation, 'unobserved');
254+
});
255+
256+
test('content appearing under unchanged chrome is not judged a no-op either', async () => {
257+
// A toast over an untouched list: nothing the baseline named disappeared, so the tolerant
258+
// classifier says 'unchanged' — and the strict no-effect bar refuses to call it a scroll that did
259+
// nothing, in either direction.
260+
const toast = {
261+
type: 'StaticText',
262+
label: 'Saved',
263+
rect: { x: 24, y: 60, width: 120, height: 20 },
264+
} as SnapshotNode;
265+
const { observation } = observe({
266+
baseline: baselineOf(screen(0)),
267+
screens: [
268+
[...screen(0), toast],
269+
[...screen(0), toast],
270+
],
271+
});
272+
273+
assert.equal(await observation, 'unobserved');
274+
});
275+
276+
test('a surface still in motion on the first read is a scroll that worked, not a no-op', async () => {
277+
const { observation, spy } = observe({
278+
baseline: baselineOf(screen(0)),
279+
screens: [screen(0), screen(-320)],
280+
});
281+
282+
assert.equal(await observation, 'moved');
283+
assert.equal(spy.calls(), 2);
284+
});
285+
286+
test('a surface that never holds still spends the budget and answers unobserved', async () => {
287+
// A spinner that comes and goes keeps every consecutive pair different while every element that
288+
// can be identified sits exactly where it was: `unchanged` against the baseline, never at rest.
289+
// Refusing here would fail a scroll over a busy screen, so the budget expires instead.
290+
const flicker = {
291+
type: 'ActivityIndicator',
292+
label: 'Loading',
293+
rect: { x: 340, y: 20, width: 20, height: 20 },
294+
} as SnapshotNode;
295+
const { observation, spy } = observe({
296+
baseline: baselineOf(screen(0)),
297+
screens: (attempt) => (attempt % 2 === 0 ? screen(0) : [...screen(0), flicker]),
298+
budgetMs: 40,
299+
});
300+
301+
assert.equal(await observation, 'unobserved');
302+
// The budget, not an accidental quiet pair: more than the two reads a verdict needs were taken.
303+
assert.ok(spy.calls() > 2);
304+
});
305+
306+
test('a capture this command cannot read withholds the claim instead of failing the scroll', async () => {
307+
const unreadable = new Error('snapshot source unsupported');
308+
const { observation } = observe({
309+
baseline: baselineOf(screen(0)),
310+
screens: () => unreadable,
311+
budgetMs: 40,
312+
});
313+
314+
assert.equal(await observation, 'unobserved');
315+
});

‎src/daemon/deferred-interaction-outcome.ts‎

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import {
2727
summarizeDiscriminatingSurfaceDivergence,
2828
markPendingInteractionOutcome,
2929
retryPendingInteractionOutcome,
30+
snapshotSurfaceComparisonKey,
3031
type InteractionRetryTap,
3132
} from './interaction-outcome-policy.ts';
3233
import { runPostGestureStabilityLoop } from '@agent-device/capture-kit/post-gesture-stability';
@@ -124,7 +125,7 @@ function markPostGestureStabilization(
124125
baselineSignature,
125126
// Recorded so the loop can tell a comparable quiet capture from one
126127
// served by a different backend, which is not comparable at all.
127-
baselineBackend: snapshotComparisonKey(session.snapshot),
128+
baselineBackend: snapshotSurfaceComparisonKey(session.snapshot),
128129
}
129130
: {}),
130131
};
@@ -350,7 +351,7 @@ export async function capturePostGestureStabilizedResult<T>(params: {
350351
const snapshot = readSnapshot(value);
351352
return {
352353
signature: buildInteractionSurfaceSignature(snapshot.nodes),
353-
backend: snapshotComparisonKey(snapshot),
354+
backend: snapshotSurfaceComparisonKey(snapshot),
354355
};
355356
},
356357
signaturesStable: areInteractionSurfaceSignaturesStable,
@@ -363,16 +364,6 @@ export async function capturePostGestureStabilizedResult<T>(params: {
363364
return outcome;
364365
}
365366

366-
/**
367-
* What makes two captures comparable at all. The iOS comparison key already carries the whole
368-
* presentation identity, including the surface the capture described — an in-place system surface (a
369-
* web sign-in sheet) is captured under its own host lineage (#2438) — so a sheet appearing or
370-
* dismissing mid-poll reads as incomparable rather than as a stable surface.
371-
*/
372-
function snapshotComparisonKey(snapshot: SnapshotState | undefined): string | undefined {
373-
return snapshot?.comparisonKey ?? snapshot?.snapshotQuality?.backend;
374-
}
375-
376367
function isPostGestureStabilizingAction(
377368
action: string,
378369
positionals: string[],

‎src/daemon/interaction-outcome-policy.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -256,6 +256,17 @@ export function buildInteractionSurfaceSignature(
256256
return entries;
257257
}
258258

259+
/**
260+
* What makes two captures comparable at all: the iOS comparison key when the capture carries one,
261+
* and the capturing backend otherwise. Two trees from different producers are not two views of one
262+
* screen — the XCTest-channel fallback swapping mid-request (#1569) is the case this exists for.
263+
*/
264+
export function snapshotSurfaceComparisonKey(
265+
snapshot: SnapshotState | undefined,
266+
): string | undefined {
267+
return snapshot?.comparisonKey ?? snapshot?.snapshotQuality?.backend;
268+
}
269+
259270
export function classifyInteractionSurfaceChange(
260271
before: InteractionSurfaceSignature,
261272
after: InteractionSurfaceSignature,

0 commit comments

Comments
 (0)