Repository navigation
Expand file tree
/
Copy pathscroll-movement.ts
More file actions
475 lines (445 loc) · 19.1 KB
/
Copy pathscroll-movement.ts
File metadata and controls
475 lines (445 loc) · 19.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
import type { ScrollMovementObservation } from '@agent-device/contracts/scroll-command';
import type { ScrollDirection } from '@agent-device/contracts/scroll-gesture';
import type { SnapshotResult } from '@agent-device/contracts/interactor-types';
import { buildSnapshotState } from '@agent-device/capture-kit/snapshot-state';
import { readScrollEdgeState, type ScrollEdge } from '@agent-device/capture-kit/scroll-edge-state';
import { containsPoint } from '@agent-device/kernel/rect';
import { AppError } from '@agent-device/kernel/errors';
import type { Point, Rect, SnapshotState } from '@agent-device/kernel/snapshot';
import { emitDiagnostic } from '@agent-device/host-kit/diagnostics';
import { sleep } from '@agent-device/host-kit/retry';
import {
areInteractionSurfaceSignaturesStable,
buildInteractionSurfaceSignature,
classifyBaselineSurfaceEvidence,
haveIdenticalDiscriminatingSurfaces,
snapshotSurfaceComparisonKey,
summarizeDiscriminatingSurfaceDivergence,
type InteractionSurfaceChange,
type InteractionSurfaceSignature,
} from './interaction-outcome-policy.ts';
import { refFrameState } from './ref-frame.ts';
import { isPostGestureStabilizationPending } from './deferred-interaction-outcome.ts';
import type { SessionState } from './session-state.ts';
/**
* What one directional scroll saw after its gesture, and the only thing that can back the distance
* the same response reports (#2714).
*
* `scroll <direction>` used to answer with the travel its gesture plan had computed. That number
* describes the swipe that was dispatched, not the content that moved, so a scroll whose gesture
* never reached the container reported success with a distance — repeatedly, on every retry, until a
* later assertion failed for a reason no log explained. This module reads the difference the command
* can actually observe: the tree the session already held before the gesture, and one tree after it.
*
* The evidence rules are not a second opinion on that pair. The signature, the subset-tolerant
* baseline classifier, and the strict no-effect bar all belong to
* `interaction-outcome-policy.ts`, the same comparators the deferred post-gesture stabilization
* applies to every other gesture, so "that scroll did nothing" means one thing in this daemon.
* What is new here is only WHEN the answer is owed: a directional scroll pays one capture to gate its
* own reply instead of leaving the proof to the next command's snapshot.
*
* The shape of the loop is the mirror image of that stabilization: a surface that differs from the
* baseline is answered on the first capture, so a scroll that worked pays nothing extra; only a
* surface that looks untouched keeps polling (a mid-flight or stale read is indistinguishable from a
* no-op until it either moves or goes quiet), and only an untouched surface at rest is ever reported.
*/
/** How long a scroll keeps asking whether an untouched surface is really untouched (#1542's window). */
const MOVEMENT_VERDICT_BUDGET_MS = 1_500;
const MOVEMENT_POLL_MS = 200;
/** The pre-gesture surface, already in hand: no capture is spent to produce it. */
export type ScrollSurfaceBaseline = Readonly<{
signature: InteractionSurfaceSignature;
presentationKey: string | undefined;
comparisonKey: string | undefined;
}>;
/**
* The comparison frozen before the command's own gesture can invalidate the stored tree, and why
* there is nothing to compare when there is none.
*/
/** Why there is nothing honest to compare this scroll's effect against. */
export type ScrollSurfaceBaselineAbsence =
| 'stored-surface-unusable'
| 'stored-surface-not-current'
| 'prior-gesture-unsettled';
export type FrozenScrollSurfaceBaseline =
| Readonly<{ surface: ScrollSurfaceBaseline }>
| Readonly<{ surface: undefined; reason: ScrollSurfaceBaselineAbsence }>;
/** Why this command owes no observation of its own effect. A reason, never text to match. */
export type ScrollMovementInapplicability =
| 'owner-without-capture'
| 'non-swipe-platform'
| 'caller-declined-observation'
| 'settle-observer-owns-outcome';
/** Where the swipe ran, so a container elsewhere on the screen cannot be blamed for the no-op. */
export type ScrollSwipeEvidence = Readonly<{
/** Midpoint of the dispatched gesture, in the same space as a snapshot rect. */
midpoint?: Point;
/** The travel the gesture plan produced, reported on the refusal. */
pixels?: number;
}>;
export function readScrollSurfaceBaseline(
snapshot: SnapshotState | undefined,
): ScrollSurfaceBaseline | undefined {
if (!snapshot || snapshot.nodes.length === 0) return undefined;
const signature = buildInteractionSurfaceSignature(snapshot.nodes);
if (signature.length === 0) return undefined;
return {
signature,
presentationKey: snapshot.presentationKey,
comparisonKey: snapshotSurfaceComparisonKey(snapshot),
};
}
/**
* Freezes what a directional scroll can compare its own effect against, at the one moment that
* answer is available: before the command dispatches.
*
* The tree the session stores is not automatically the screen as it stands — a mutation may have
* happened since it was captured, and its own gesture is about to be another one. ADR 0014 already
* encodes exactly that question: the ref frame stays `active` while no device side effect has
* crossed since the last publication, which is while the stored tree still is the newest
* observation. Anything else and there is nothing honest to compare against, so this returns no
* surface and names why instead of letting a stale tree back a movement claim (#2714).
*/
export function freezeScrollSurfaceBaseline(session: SessionState): FrozenScrollSurfaceBaseline {
const surface = readScrollSurfaceBaseline(session.snapshot);
if (!surface) return { surface: undefined, reason: 'stored-surface-unusable' };
if (refFrameState(session) !== 'active') {
return { surface: undefined, reason: 'stored-surface-not-current' };
}
if (isPostGestureStabilizationPending(session)) {
// A gesture nobody has read yet may still be moving this tree, which would bill its motion to
// the scroll being dispatched now.
return { surface: undefined, reason: 'prior-gesture-unsettled' };
}
return { surface };
}
/**
* Observes one directional scroll and returns the movement its response may claim.
*
* Throws `scroll_no_progress` when it proves the gesture did not land: the surface is byte-for-byte
* what it was, the tree still names hidden content in the direction that was scrolled, and the swipe
* ran inside that container. Every other untouched surface answers honestly instead — at the edge of
* the content, in a direction with no end-of-content signal, or as `unobserved` when the evidence
* would not support the claim either way.
*
* A capture that fails does not fail the scroll: the gesture already happened, so an unreadable tree
* answers `unobserved` and leaves the caller's scroll succeeded.
*/
export async function observeScrollMovement(params: {
direction: ScrollDirection;
baseline: ScrollSurfaceBaseline;
swipe: ScrollSwipeEvidence;
capture: () => Promise<SnapshotResult>;
/** The same two overrides `pollForScrollRest` takes: how long to ask, and how often. */
budgetMs?: number;
pollMs?: number;
}): Promise<ScrollMovementObservation> {
const { direction, baseline, swipe } = params;
const verdict = await pollForSurfaceVerdict(baseline, params);
switch (verdict.kind) {
case 'moved':
return reportObserved(direction, 'moved', swipe, verdict.attempts, verdict.startedAt);
case 'blind':
return reportBlind(direction, verdict.reason, swipe);
case 'settled':
return await decideUnchanged({ ...verdict, direction, baseline, swipe });
}
}
/** What the polling could say about the surface, before any of it is interpreted. */
type SurfaceVerdict =
| Readonly<{ kind: 'moved'; attempts: number; startedAt: number }>
| Readonly<{ kind: 'blind'; reason: 'capture-unreadable' | 'surface-unsettled' }>
| Readonly<{
kind: 'settled';
observed: ObservedSurface;
evidence: InteractionSurfaceChange;
attempts: number;
startedAt: number;
}>;
/**
* Polls until an untouched surface proves itself. A surface that differs from the baseline is a
* verdict on the first capture, so a scroll that worked pays for one read. One that looks untouched
* needs a quiet pair, because a gesture still in flight and a gesture that did nothing answer a
* single read identically. A surface that never holds still expires as `surface-unsettled` rather
* than being called a no-op — and that answer is worth a warning the others are not, since it spent
* the whole budget confirming nothing.
*/
async function pollForSurfaceVerdict(
baseline: ScrollSurfaceBaseline,
params: {
direction: ScrollDirection;
capture: () => Promise<SnapshotResult>;
budgetMs?: number;
pollMs?: number;
swipe: ScrollSwipeEvidence;
},
): Promise<SurfaceVerdict> {
const startedAt = Date.now();
const deadline = startedAt + (params.budgetMs ?? MOVEMENT_VERDICT_BUDGET_MS);
let previous: InteractionSurfaceSignature | undefined;
let attempts = 0;
while (true) {
const observed = await captureSurface(params.capture);
attempts += 1;
if (!observed) return { kind: 'blind', reason: 'capture-unreadable' };
const evidence = classifyBaselineSurfaceEvidence(baseline.signature, observed.signature);
if (evidence === 'changed') return { kind: 'moved', attempts, startedAt };
if (surfaceIsAtRest(previous, observed.signature)) {
return { kind: 'settled', observed, evidence, attempts, startedAt };
}
if (Date.now() >= deadline) {
emitDiagnostic({
level: 'warn',
phase: 'scroll_movement_budget_expired',
data: {
direction: params.direction,
attempts,
durationMs: Date.now() - startedAt,
...(params.swipe.pixels === undefined ? {} : { requestedPixels: params.swipe.pixels }),
},
});
return { kind: 'blind', reason: 'surface-unsettled' };
}
previous = observed.signature;
await sleep(params.pollMs ?? MOVEMENT_POLL_MS);
}
}
function surfaceIsAtRest(
previous: InteractionSurfaceSignature | undefined,
next: InteractionSurfaceSignature,
): boolean {
return previous !== undefined && areInteractionSurfaceSignaturesStable(previous, next);
}
type ObservedSurface = Readonly<{
signature: InteractionSurfaceSignature;
presentationKey: string | undefined;
comparisonKey: string | undefined;
/** The tree itself: the edge question is asked of nodes, not of the identity signature. */
nodes: SnapshotState['nodes'];
}>;
async function captureSurface(
capture: () => Promise<SnapshotResult>,
): Promise<ObservedSurface | undefined> {
let result: SnapshotResult;
try {
result = await capture();
} catch {
// The gesture is already done; a tree this command cannot read withholds the claim, it does not
// undo the scroll.
return undefined;
}
const state = buildSnapshotState(result, undefined);
return {
signature: buildInteractionSurfaceSignature(state.nodes),
presentationKey: state.presentationKey,
comparisonKey: snapshotSurfaceComparisonKey(state),
nodes: state.nodes,
};
}
async function decideUnchanged(params: {
direction: ScrollDirection;
baseline: ScrollSurfaceBaseline;
swipe: ScrollSwipeEvidence;
observed: ObservedSurface;
evidence: InteractionSurfaceChange;
attempts: number;
startedAt: number;
}): Promise<ScrollMovementObservation> {
const { direction, baseline, observed, evidence, swipe } = params;
const comparison = compareWithBaseline(baseline, observed, evidence);
if (comparison) return reportBlind(direction, comparison.reason, swipe, comparison.evidence);
return await decideEdgeVerdict(params);
}
type SurfaceComparison = Readonly<{
reason: ScrollMovementBlindReason;
evidence?: Record<string, unknown>;
}>;
/**
* Are these two captures even two views of one screen? Two captures that describe different
* presentations are not: the interactive-only tree `snapshot -i` stores and the broad tree this
* command reads share almost no elements, and a lineage change is the XCTest-channel fallback
* swapping the producer mid-request (#1569). Both answer `unobserved` rather than read the
* difference as movement — or as a no-op.
*/
function compareWithBaseline(
baseline: ScrollSurfaceBaseline,
observed: ObservedSurface,
evidence: InteractionSurfaceChange,
): SurfaceComparison | undefined {
if (baseline.presentationKey !== observed.presentationKey) {
return { reason: 'baseline-presentation-drift' };
}
if (baseline.comparisonKey !== observed.comparisonKey) {
return { reason: 'capture-lineage-drift' };
}
if (evidence === 'ambiguous') return { reason: 'no-comparable-content' };
// The subset-tolerant classifier is 'unchanged' for a narrower capture of the same screen, so the
// no-effect claim needs the strict both-directions bar (#1601 review P1): a scroll that replaced
// every list cell under fixed chrome must not be readable as a gesture that did nothing.
if (!haveIdenticalDiscriminatingSurfaces(baseline.signature, observed.signature)) {
return {
reason: 'surface-divergence',
evidence: summarizeDiscriminatingSurfaceDivergence(baseline.signature, observed.signature),
};
}
return undefined;
}
/** What an untouched surface is: the end of the content, a direction with no edge to read, or a gesture that never landed. */
async function decideEdgeVerdict(params: {
direction: ScrollDirection;
swipe: ScrollSwipeEvidence;
observed: ObservedSurface;
attempts: number;
startedAt: number;
}): Promise<ScrollMovementObservation> {
const { direction, observed, swipe } = params;
const edge = verticalEdgeFor(direction);
if (!edge) {
// The hidden-content analyzer reads the vertical axis only, so a horizontal scroll that moved
// nothing cannot tell a pager at its end from a list that ignored the swipe. It reports what it
// measured and leaves the two apart, rather than guessing.
return reportObserved(direction, 'unchanged', swipe, params.attempts, params.startedAt);
}
const edgeState = await readScrollEdgeState(observed.nodes, edge);
const containerRect = edgeState.containerRect;
if (!containerRect) {
// `canScroll: false` here means the tree named no scroll container at all, which says nothing
// about where the content ends.
return reportBlind(direction, 'no-scroll-container', swipe);
}
if (swipe.midpoint && !containerHoldsSwipe(containerRect, swipe.midpoint)) {
return reportBlind(direction, 'container-outside-swipe', swipe, { containerRect });
}
if (edgeState.canScroll) {
throw scrollNoProgressError(direction, edge, swipe, containerRect);
}
return reportObserved(direction, 'at-edge', swipe, params.attempts, params.startedAt);
}
function containerHoldsSwipe(containerRect: Rect, midpoint: Point): boolean {
return containsPoint(containerRect, midpoint.x, midpoint.y);
}
/**
* The gesture did not reach the container: hidden content is still there, and the surface never
* moved. Same typed family as `scroll_edge_no_progress` and `scroll_until_no_progress`, and the same
* hint, because the caller's next move is the same whichever loop noticed.
*/
function scrollNoProgressError(
direction: ScrollDirection,
edge: ScrollEdge,
swipe: ScrollSwipeEvidence,
containerRect: Rect,
): AppError {
return new AppError(
'COMMAND_FAILED',
`scroll ${direction} moved nothing: the container still reports hidden content ${
edge === 'bottom' ? 'below' : 'above'
} and its contents never shifted`,
{
reason: 'scroll_no_progress',
direction,
hiddenContentAt: edge,
containerRect,
...(swipe.pixels === undefined ? {} : { requestedPixels: swipe.pixels }),
hint: scrollNoProgressHint(direction, swipe),
},
);
}
/**
* What the caller can do next. The raw-drag sentence only makes sense where the command knows where
* a swipe would land: a tvOS scroll is a remote keypress with no coordinates to name, and telling
* someone to swipe there is advice they cannot follow.
*/
function scrollNoProgressHint(direction: ScrollDirection, swipe: ScrollSwipeEvidence): string {
const reached =
`The scroll is not reaching this container. If a field is focused, dismiss the keyboard first; ` +
`if it is nested inside another scroller, target it directly with scroll ${direction} --until <selector>.`;
if (!swipe.midpoint) return reached;
return (
reached +
` Some lists ignore synthesized scrolls — a raw drag moves them: swipe x1 y1 x2 y2 started inside the list.`
);
}
/** Why a scroll could not confirm its own effect. A reason, never text a caller has to match. */
type ScrollMovementBlindReason =
| 'capture-unreadable'
| 'surface-unsettled'
| 'no-comparable-content'
| 'baseline-presentation-drift'
| 'capture-lineage-drift'
| 'surface-divergence'
| 'no-scroll-container'
| 'container-outside-swipe';
/**
* Tells the daemon log that this command owed no observation at all — the owner cannot read a
* screen, the device has no swipe to verify, or the caller (or another observer) already declined
* or owns that read. The response carries no `movement` field, so the reason has to live somewhere.
*/
export function reportScrollMovementNotApplicable(
direction: ScrollDirection,
reason: ScrollMovementInapplicability,
): void {
emitDiagnostic({
level: 'debug',
phase: 'scroll_movement_not_applicable',
data: { direction, reason },
});
}
/**
* Tells the daemon log why a scroll that DID owe an observation could not make one, and answers
* `unobserved`: the distance it reports rests on the gesture plan alone.
*/
export function reportScrollMovementUnobservable(
direction: ScrollDirection,
reason: ScrollSurfaceBaselineAbsence,
): ScrollMovementObservation {
emitDiagnostic({
level: 'info',
phase: 'scroll_movement_unobserved',
data: { direction, reason },
});
return 'unobserved';
}
function reportBlind(
direction: ScrollDirection,
reason: ScrollMovementBlindReason,
swipe: ScrollSwipeEvidence,
evidence?: Record<string, unknown>,
): ScrollMovementObservation {
emitDiagnostic({
level: 'debug',
phase: 'scroll_movement_unobserved',
data: {
direction,
reason,
...(swipe.pixels === undefined ? {} : { requestedPixels: swipe.pixels }),
...(evidence ?? {}),
},
});
return 'unobserved';
}
function reportObserved(
direction: ScrollDirection,
movement: Exclude<ScrollMovementObservation, 'unobserved'>,
swipe: ScrollSwipeEvidence,
attempts: number,
startedAt: number,
): ScrollMovementObservation {
emitDiagnostic({
level: movement === 'moved' ? 'debug' : 'info',
phase: 'scroll_movement_observed',
data: {
direction,
movement,
attempts,
durationMs: Date.now() - startedAt,
...(swipe.pixels === undefined ? {} : { requestedPixels: swipe.pixels }),
},
});
return movement;
}
/** The end-of-content analyzer only reads the vertical edges, same as the `--until` loop's. */
function verticalEdgeFor(direction: ScrollDirection): ScrollEdge | undefined {
if (direction === 'down') return 'bottom';
if (direction === 'up') return 'top';
return undefined;
}