Skip to content

Commit 62eb122

Browse files
committed
fix(scroll): refuse an unreadable capture before scroll top/bottom decides
`scroll bottom` coerced a failed or sparse capture to an empty tree, which the edge analyzer reads as "no hidden content below", so it reported "Already at bottom" about a screen it never saw. Both scroll loops now run on one `runScrollPasses` loop in capture-kit: observe, refuse an unreadable observation, ask the caller's stop condition, then spend a pass. The daemon defines the refusal once (`scrollCaptureRefusal`, built on `sparseCaptureQuality`) and both `scroll top`/`scroll bottom` and `scroll --until` pass it in, so an edge scroll now fails with `scroll_edge_capture_unreadable` instead of an edge verdict. Fixes #2468
1 parent 4db1b7d commit 62eb122

6 files changed

Lines changed: 382 additions & 158 deletions

File tree

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
import { test } from 'vitest';
2+
import assert from 'node:assert/strict';
3+
import { AppError } from '@agent-device/kernel/errors';
4+
import { runScrollEdgePasses, runScrollPasses } from '../scroll-edge-state.ts';
5+
6+
type Frame = { readable: boolean; done: boolean; fingerprint?: string };
7+
8+
const REFUSED = new AppError('COMMAND_FAILED', 'refused', { reason: 'refused' });
9+
const PASS_LIMIT = new AppError('COMMAND_FAILED', 'limit', { reason: 'limit' });
10+
const NO_PROGRESS = new AppError('COMMAND_FAILED', 'stuck', { reason: 'stuck' });
11+
12+
/** Drives the shared loop over scripted frames, counting what it asked for. */
13+
async function run(frames: Frame[], passLimit = 10) {
14+
const calls = { observe: 0, shouldStop: 0, scroll: 0, settle: 0 };
15+
const outcome = runScrollPasses({
16+
observe: async () => frames[Math.min(calls.observe++, frames.length - 1)]!,
17+
refuse: (frame) => (frame.readable ? undefined : REFUSED),
18+
shouldStop: async (frame) => {
19+
calls.shouldStop += 1;
20+
return frame.done;
21+
},
22+
passLimit,
23+
passLimitError: () => PASS_LIMIT,
24+
fingerprint: async (frame) => frame.fingerprint,
25+
noProgressError: () => NO_PROGRESS,
26+
scroll: async () => ({ pass: ++calls.scroll }),
27+
settleAfterPass: async () => {
28+
calls.settle += 1;
29+
},
30+
});
31+
return { outcome, calls };
32+
}
33+
34+
test('runScrollPasses: a stop on the first observation costs no gesture', async () => {
35+
const { outcome, calls } = await run([{ readable: true, done: true }]);
36+
assert.deepEqual(await outcome, { passes: 0, result: undefined });
37+
assert.equal(calls.scroll, 0);
38+
});
39+
40+
test('runScrollPasses: scrolls and settles once per pass and reports the last gesture', async () => {
41+
const { outcome, calls } = await run([
42+
{ readable: true, done: false, fingerprint: 'a' },
43+
{ readable: true, done: false, fingerprint: 'b' },
44+
{ readable: true, done: true },
45+
]);
46+
assert.deepEqual(await outcome, { passes: 2, result: { pass: 2 } });
47+
assert.equal(calls.settle, 2);
48+
});
49+
50+
test('runScrollPasses: refuses an unreadable observation before asking the stop condition', async () => {
51+
const { outcome, calls } = await run([{ readable: false, done: true }]);
52+
await assert.rejects(outcome, (error) => error === REFUSED);
53+
assert.equal(calls.shouldStop, 0);
54+
assert.equal(calls.scroll, 0);
55+
});
56+
57+
test('runScrollPasses: refuses an observation that goes unreadable after a pass', async () => {
58+
const { outcome, calls } = await run([
59+
{ readable: true, done: false, fingerprint: 'a' },
60+
{ readable: false, done: true },
61+
]);
62+
await assert.rejects(outcome, (error) => error === REFUSED);
63+
assert.equal(calls.scroll, 1);
64+
});
65+
66+
test('runScrollPasses: spends exactly its pass budget, then raises the caller error', async () => {
67+
const budget = 3;
68+
const frames = Array.from({ length: budget + 1 }, (_, index) => ({
69+
readable: true,
70+
done: false,
71+
fingerprint: `f${index}`,
72+
}));
73+
const { outcome, calls } = await run(frames, budget);
74+
await assert.rejects(outcome, (error) => error === PASS_LIMIT);
75+
assert.equal(calls.scroll, budget);
76+
});
77+
78+
/**
79+
* The budget is checked before the stuck window, so an explicit small budget reports the budget
80+
* rather than no-progress (the order `scroll --until` documented before the loops were shared).
81+
*/
82+
test('runScrollPasses: a surface that never shifts stops as no-progress, the budget winning ties', async () => {
83+
const stuck = { readable: true, done: false, fingerprint: 'same' };
84+
const unbounded = await run([stuck], 10);
85+
await assert.rejects(unbounded.outcome, (error) => error === NO_PROGRESS);
86+
assert.equal(unbounded.calls.scroll, 3);
87+
88+
const bounded = await run([stuck], 3);
89+
await assert.rejects(bounded.outcome, (error) => error === PASS_LIMIT);
90+
});
91+
92+
test('runScrollPasses: an observation without a fingerprint never trips no-progress', async () => {
93+
const { outcome, calls } = await run([{ readable: true, done: false }], 5);
94+
await assert.rejects(outcome, (error) => error === PASS_LIMIT);
95+
assert.equal(calls.scroll, 5);
96+
});
97+
98+
test('runScrollEdgePasses: a refused capture raises the typed edge refusal before any gesture', async () => {
99+
let scrolls = 0;
100+
await assert.rejects(
101+
runScrollEdgePasses({
102+
edge: 'bottom',
103+
captureState: async () => ({
104+
refused: { reason: 'sparse-tree', detail: 'AX bridge unavailable' },
105+
}),
106+
scroll: async () => {
107+
scrolls += 1;
108+
},
109+
}),
110+
(error: unknown) => {
111+
assert.ok(error instanceof AppError);
112+
assert.equal(error.details?.reason, 'scroll_edge_capture_unreadable');
113+
assert.equal(error.details?.edge, 'bottom');
114+
assert.equal(error.details?.captureRefusal, 'sparse-tree');
115+
assert.match(error.message, /AX bridge unavailable/);
116+
return true;
117+
},
118+
);
119+
assert.equal(scrolls, 0);
120+
});

‎packages/capture-kit/src/snapshot/scroll-edge-state.ts‎

Lines changed: 100 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,16 @@ export type ScrollEdgeState = {
3535
fingerprint?: string;
3636
};
3737

38+
/**
39+
* Why a capture cannot answer a scroll loop's question at all. Never an outcome about the content:
40+
* a failed or sparse read coerced to an empty tree is what the edge analyzer reads as "no hidden
41+
* content", so it has to be refused before anything decides on it.
42+
*/
43+
export type ScrollCaptureRefusal = { reason: 'no-capture' | 'sparse-tree'; detail: string };
44+
45+
/** What one edge capture yields: an analyzed state, or the refusal that kept it from being read. */
46+
export type ScrollEdgeObservation = ScrollEdgeState | { refused: ScrollCaptureRefusal };
47+
3848
export type ScrollEdgeTarget = {
3949
point?: Point;
4050
nodeIndex?: number;
@@ -143,9 +153,24 @@ export function pushScrollSurfaceSignature(
143153
if (recentSignatures.length > window) recentSignatures.shift();
144154
}
145155

146-
export async function runScrollEdgePasses<TResult>(params: {
147-
edge: ScrollEdge;
148-
captureState: (scope?: string) => Promise<ScrollEdgeState>;
156+
/**
157+
* The one scroll pass loop: observe, refuse an unreadable observation, ask the caller's stop
158+
* condition, then spend a pass. `scroll top`/`scroll bottom` and `scroll --until` differ only in
159+
* what stops them and what they call a failure, so every other rule lives here once: the refusal
160+
* lands before the stop condition reads anything, the budget is checked before the stuck window so
161+
* an explicit small budget still reports itself, and the first observation precedes the first
162+
* gesture so an already-satisfied stop costs no scroll.
163+
*/
164+
export async function runScrollPasses<TObservation, TResult>(params: {
165+
observe: () => Promise<TObservation>;
166+
refuse: (observation: TObservation) => AppError | undefined;
167+
/** True ends the loop with this pass count; a terminal outcome of its own throws instead. */
168+
shouldStop: (observation: TObservation, passes: number) => Promise<boolean>;
169+
passLimit: number;
170+
passLimitError: (passes: number) => AppError;
171+
/** The surface signature the stuck window tracks; `undefined` keeps a pass out of it. */
172+
fingerprint: (observation: TObservation) => Promise<string | undefined>;
173+
noProgressError: (passes: number) => AppError;
149174
scroll: () => Promise<TResult>;
150175
/**
151176
* Waits for the previous fling to come to rest before the next capture. A rubber-banded scroll
@@ -155,19 +180,58 @@ export async function runScrollEdgePasses<TResult>(params: {
155180
*/
156181
settleAfterPass?: () => Promise<void>;
157182
}): Promise<{ passes: number; result?: TResult }> {
158-
const { edge, captureState, scroll, settleAfterPass = async () => {} } = params;
159-
let state = await captureState();
160-
if (state.scope) {
161-
state = await captureState(state.scope);
162-
}
163-
183+
const { observe, refuse, shouldStop, passLimit, scroll, settleAfterPass } = params;
164184
let passes = 0;
165185
let result: TResult | undefined;
166186
const recentSignatures: string[] = [];
167-
pushScrollSurfaceSignature(recentSignatures, state.fingerprint, SCROLL_EDGE_STUCK_WINDOW);
168-
while (state.canScroll) {
169-
if (passes >= SCROLL_EDGE_PASS_LIMIT) {
170-
throw new AppError(
187+
while (true) {
188+
const observation = await observe();
189+
const refusal = refuse(observation);
190+
if (refusal) throw refusal;
191+
if (await shouldStop(observation, passes)) return { passes, result };
192+
if (passes >= passLimit) throw params.passLimitError(passes);
193+
pushScrollSurfaceSignature(
194+
recentSignatures,
195+
await params.fingerprint(observation),
196+
SCROLL_EDGE_STUCK_WINDOW,
197+
);
198+
if (scrollSurfaceIsStuck(recentSignatures)) throw params.noProgressError(passes);
199+
result = await scroll();
200+
passes += 1;
201+
await settleAfterPass?.();
202+
}
203+
}
204+
205+
export async function runScrollEdgePasses<TResult>(params: {
206+
edge: ScrollEdge;
207+
captureState: (scope?: string) => Promise<ScrollEdgeObservation>;
208+
scroll: () => Promise<TResult>;
209+
settleAfterPass?: () => Promise<void>;
210+
}): Promise<{ passes: number; result?: TResult }> {
211+
const { edge, captureState, scroll, settleAfterPass } = params;
212+
// The first capture discovers the container; every later one reads the scope the previous state
213+
// named, so the loop verifies against the scroller it decided on rather than the whole tree.
214+
let scope: string | undefined;
215+
let discovered = false;
216+
const observe = async (): Promise<ScrollEdgeObservation> => {
217+
let observation = await captureState(scope);
218+
if (!discovered) {
219+
discovered = true;
220+
if (!('refused' in observation) && observation.scope) {
221+
observation = await captureState(observation.scope);
222+
}
223+
}
224+
if (!('refused' in observation)) scope = observation.scope;
225+
return observation;
226+
};
227+
return await runScrollPasses({
228+
observe,
229+
refuse: (observation) =>
230+
'refused' in observation ? buildScrollEdgeCaptureError(edge, observation.refused) : undefined,
231+
shouldStop: async (observation) => !('refused' in observation) && !observation.canScroll,
232+
passLimit: SCROLL_EDGE_PASS_LIMIT,
233+
passLimitError: (passes) =>
234+
new AppError(
171235
'COMMAND_FAILED',
172236
`scroll ${edge} reached the safety limit before the snapshot showed the edge`,
173237
{
@@ -176,21 +240,13 @@ export async function runScrollEdgePasses<TResult>(params: {
176240
passes,
177241
hint: 'The scoped scroll container still reports hidden content. Run scroll <dir> --until <selector> to stop on the element you are after, or snapshot -i to inspect the current state.',
178242
},
179-
);
180-
}
181-
182-
result = await scroll();
183-
passes += 1;
184-
await settleAfterPass();
185-
state = await captureState(state.scope);
186-
187-
pushScrollSurfaceSignature(recentSignatures, state.fingerprint, SCROLL_EDGE_STUCK_WINDOW);
188-
if (state.canScroll && scrollSurfaceIsStuck(recentSignatures)) {
189-
throw buildScrollEdgeNoProgressError(edge, passes);
190-
}
191-
}
192-
193-
return { passes, result };
243+
),
244+
fingerprint: async (observation) =>
245+
'refused' in observation ? undefined : observation.fingerprint,
246+
noProgressError: (passes) => buildScrollEdgeNoProgressError(edge, passes),
247+
scroll,
248+
...(settleAfterPass ? { settleAfterPass } : {}),
249+
});
194250
}
195251

196252
/**
@@ -278,6 +334,22 @@ export function scrollNoProgressHint(
278334
);
279335
}
280336

337+
function buildScrollEdgeCaptureError(edge: ScrollEdge, refusal: ScrollCaptureRefusal): AppError {
338+
return new AppError(
339+
'COMMAND_FAILED',
340+
`scroll ${edge} could not read the screen: ${refusal.detail}`,
341+
{
342+
reason: 'scroll_edge_capture_unreadable',
343+
edge,
344+
captureRefusal: refusal.reason,
345+
hint:
346+
refusal.reason === 'no-capture'
347+
? 'Run snapshot -i to see whether the app is producing an accessibility tree at all, and retry once it does.'
348+
: 'The accessibility tree came back sparse, so it cannot show whether content remains. Run screenshot, inspect the image, and navigate by coordinates until snapshot -i reports a full tree.',
349+
},
350+
);
351+
}
352+
281353
function buildScrollEdgeVerificationError(
282354
edge: ScrollEdge,
283355
scope: string | undefined,

‎packages/daemon/src/execution/__tests__/scroll-runtime.test.ts‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -344,6 +344,76 @@ test('bound scroll bottom keeps scoped capture failures scoped', async () => {
344344
assert.equal(snapshotCount, 2);
345345
});
346346

347+
/**
348+
* A failed or sparse read used to coerce to an empty tree, which the edge analyzer reads as "no
349+
* hidden content below", so `scroll bottom` answered "Already at bottom" about a screen it never
350+
* saw (#2468). Each case counts gestures, so the refusal is proven to land before any scroll.
351+
*/
352+
test.each([
353+
{ name: 'a capture with no tree', frame: { backend: 'xctest' }, refusal: 'no-capture' },
354+
{ name: 'an empty tree', frame: { backend: 'xctest', nodes: [] }, refusal: 'no-capture' },
355+
{
356+
name: 'a tree the backend calls sparse',
357+
frame: {
358+
...makeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' }),
359+
quality: { state: 'sparse', backend: 'tree', reason: 'AX bridge unavailable' },
360+
},
361+
refusal: 'sparse-tree',
362+
},
363+
])(
364+
'bound scroll bottom refuses $name instead of reporting the edge',
365+
async ({ frame, refusal }) => {
366+
let scrolls = 0;
367+
await assert.rejects(
368+
() =>
369+
runScroll(
370+
['bottom'],
371+
{},
372+
{
373+
scroll: async () => {
374+
scrolls += 1;
375+
return {};
376+
},
377+
captureSnapshot: async () => frame,
378+
},
379+
),
380+
(error: unknown) => {
381+
assert.ok(error instanceof AppError);
382+
assert.equal(error.code, 'COMMAND_FAILED');
383+
assert.equal(error.details?.reason, 'scroll_edge_capture_unreadable');
384+
assert.equal(error.details?.edge, 'bottom');
385+
assert.equal(error.details?.captureRefusal, refusal);
386+
return true;
387+
},
388+
);
389+
assert.equal(scrolls, 0);
390+
},
391+
);
392+
393+
test('bound scroll bottom refuses a capture that goes unreadable mid-loop', async () => {
394+
let scrolls = 0;
395+
await assert.rejects(
396+
() =>
397+
runScroll(
398+
['bottom'],
399+
{},
400+
{
401+
scroll: async () => {
402+
scrolls += 1;
403+
return {};
404+
},
405+
captureSnapshot: async () =>
406+
scrolls === 0
407+
? makeScrollSnapshot({ hiddenBelow: true, message: 'Middle message' })
408+
: { backend: 'xctest', nodes: [] },
409+
},
410+
),
411+
(error: unknown) =>
412+
error instanceof AppError && error.details?.reason === 'scroll_edge_capture_unreadable',
413+
);
414+
assert.equal(scrolls, 1);
415+
});
416+
347417
function makeScrollSnapshot(options: { hiddenBelow: boolean; message: string }) {
348418
return {
349419
backend: 'xctest' as const,
Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
import type { SnapshotResult } from '@agent-device/contracts/interactor-types';
2+
import type { SnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot';
3+
import { sparseCaptureQuality } from '@agent-device/selectors/absence-observation';
4+
import type { ScrollCaptureRefusal } from '@agent-device/capture-kit/scroll-edge-state';
5+
6+
/**
7+
* Whether a scroll capture can answer anything at all, for every scroll loop. Sparseness is the
8+
* question absence assertions ask, answered by the same helper rather than a second definition of
9+
* readable. Truncation is deliberately NOT refused: a truncated tree is real and readable with its
10+
* tail missing, and refusing it would fail large screens where the answer is plainly in view.
11+
*/
12+
export function scrollCaptureRefusal(result: SnapshotResult): ScrollCaptureRefusal | undefined {
13+
const nodes = result.nodes;
14+
if (nodes === undefined) {
15+
return { reason: 'no-capture', detail: 'the capture returned no accessibility tree' };
16+
}
17+
if (nodes.length === 0) {
18+
return { reason: 'no-capture', detail: 'the capture returned an empty accessibility tree' };
19+
}
20+
const sparse = sparseCaptureQuality({
21+
backend: result.backend as SnapshotState['backend'],
22+
nodes: nodes as SnapshotNode[],
23+
...(result.quality ? { snapshotQuality: result.quality } : {}),
24+
});
25+
if (sparse) {
26+
return {
27+
reason: 'sparse-tree',
28+
detail: sparse.reason ?? 'the capture backend reported a sparse tree',
29+
};
30+
}
31+
return undefined;
32+
}

0 commit comments

Comments
 (0)