Skip to content

Commit 5b97e14

Browse files
authored
fix(scroll): refuse an unreadable capture before scroll top/bottom decides (#3736)
* fix(scroll): refuse an unreadable capture before scroll top/bottom decides `scroll bottom` coerced a failed or sparse capture to an empty tree, which the edge analyzer reads as "no hidden content below", so it reported "Already at bottom" about a screen it never saw. Both scroll loops now run on one `runScrollPasses` loop in capture-kit: observe, refuse an unreadable observation, ask the caller's stop condition, then spend a pass. The daemon defines the refusal once (`scrollCaptureRefusal`, built on `sparseCaptureQuality`) and both `scroll top`/`scroll bottom` and `scroll --until` pass it in, so an edge scroll now fails with `scroll_edge_capture_unreadable` instead of an edge verdict. Fixes #2468 * refactor(scroll): keep the backend's sparse reason and one scope owner Check sparse quality before the empty-tree refusal, so a backend that ships no nodes with a sparse verdict (the Android helper's presentation failure) reports its own reason rather than a generic no-capture. Read refused observations through one `isScrollEdgeRefused` guard, and hand the edge loop's scope to `settleAfterPass` so the daemon no longer tracks a second copy. * fix(scroll): keep the capture refusal inside scroll-until The separate scroll-capture-refusal module added one eagerly evaluated module to every daemon entry, which the eager-closure budget rejects. Define the refusal in scroll-until.ts, which scroll-runtime already imports, so both loops share it at no extra load cost. * fix(scroll): keep the sparse reason when a capture omits its tree A capture with no nodes field and a sparse verdict reported a generic no-capture. Check sparse quality against an empty tree first, so the backend's reason survives whether the tree is empty or missing.
1 parent cc6d91a commit 5b97e14

5 files changed

Lines changed: 399 additions & 152 deletions

File tree

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
import { test } from 'vitest';
2+
import assert from 'node:assert/strict';
3+
import { AppError } from '@agent-device/kernel/errors';
4+
import { runScrollEdgePasses, runScrollPasses } from '../scroll-edge-state.ts';
5+
6+
type Frame = { readable: boolean; done: boolean; fingerprint?: string };
7+
8+
const REFUSED = new AppError('COMMAND_FAILED', 'refused', { reason: 'refused' });
9+
const PASS_LIMIT = new AppError('COMMAND_FAILED', 'limit', { reason: 'limit' });
10+
const NO_PROGRESS = new AppError('COMMAND_FAILED', 'stuck', { reason: 'stuck' });
11+
12+
/** Drives the shared loop over scripted frames, counting what it asked for. */
13+
async function run(frames: Frame[], passLimit = 10) {
14+
const calls = { observe: 0, shouldStop: 0, scroll: 0, settle: 0 };
15+
const outcome = runScrollPasses({
16+
observe: async () => frames[Math.min(calls.observe++, frames.length - 1)]!,
17+
refuse: (frame) => (frame.readable ? undefined : REFUSED),
18+
shouldStop: async (frame) => {
19+
calls.shouldStop += 1;
20+
return frame.done;
21+
},
22+
passLimit,
23+
passLimitError: () => PASS_LIMIT,
24+
fingerprint: async (frame) => frame.fingerprint,
25+
noProgressError: () => NO_PROGRESS,
26+
scroll: async () => ({ pass: ++calls.scroll }),
27+
settleAfterPass: async () => {
28+
calls.settle += 1;
29+
},
30+
});
31+
return { outcome, calls };
32+
}
33+
34+
test('runScrollPasses: a stop on the first observation costs no gesture', async () => {
35+
const { outcome, calls } = await run([{ readable: true, done: true }]);
36+
assert.deepEqual(await outcome, { passes: 0, result: undefined });
37+
assert.equal(calls.scroll, 0);
38+
});
39+
40+
test('runScrollPasses: scrolls and settles once per pass and reports the last gesture', async () => {
41+
const { outcome, calls } = await run([
42+
{ readable: true, done: false, fingerprint: 'a' },
43+
{ readable: true, done: false, fingerprint: 'b' },
44+
{ readable: true, done: true },
45+
]);
46+
assert.deepEqual(await outcome, { passes: 2, result: { pass: 2 } });
47+
assert.equal(calls.settle, 2);
48+
});
49+
50+
test('runScrollPasses: refuses an unreadable observation before asking the stop condition', async () => {
51+
const { outcome, calls } = await run([{ readable: false, done: true }]);
52+
await assert.rejects(outcome, (error) => error === REFUSED);
53+
assert.equal(calls.shouldStop, 0);
54+
assert.equal(calls.scroll, 0);
55+
});
56+
57+
test('runScrollPasses: refuses an observation that goes unreadable after a pass', async () => {
58+
const { outcome, calls } = await run([
59+
{ readable: true, done: false, fingerprint: 'a' },
60+
{ readable: false, done: true },
61+
]);
62+
await assert.rejects(outcome, (error) => error === REFUSED);
63+
assert.equal(calls.scroll, 1);
64+
});
65+
66+
test('runScrollPasses: spends exactly its pass budget, then raises the caller error', async () => {
67+
const budget = 3;
68+
const frames = Array.from({ length: budget + 1 }, (_, index) => ({
69+
readable: true,
70+
done: false,
71+
fingerprint: `f${index}`,
72+
}));
73+
const { outcome, calls } = await run(frames, budget);
74+
await assert.rejects(outcome, (error) => error === PASS_LIMIT);
75+
assert.equal(calls.scroll, budget);
76+
});
77+
78+
/**
79+
* The budget is checked before the stuck window, so an explicit small budget reports the budget
80+
* rather than no-progress (the order `scroll --until` documented before the loops were shared).
81+
*/
82+
test('runScrollPasses: a surface that never shifts stops as no-progress, the budget winning ties', async () => {
83+
const stuck = { readable: true, done: false, fingerprint: 'same' };
84+
const unbounded = await run([stuck], 10);
85+
await assert.rejects(unbounded.outcome, (error) => error === NO_PROGRESS);
86+
assert.equal(unbounded.calls.scroll, 3);
87+
88+
const bounded = await run([stuck], 3);
89+
await assert.rejects(bounded.outcome, (error) => error === PASS_LIMIT);
90+
});
91+
92+
test('runScrollPasses: an observation without a fingerprint never trips no-progress', async () => {
93+
const { outcome, calls } = await run([{ readable: true, done: false }], 5);
94+
await assert.rejects(outcome, (error) => error === PASS_LIMIT);
95+
assert.equal(calls.scroll, 5);
96+
});
97+
98+
test('runScrollEdgePasses: a refused capture raises the typed edge refusal before any gesture', async () => {
99+
let scrolls = 0;
100+
await assert.rejects(
101+
runScrollEdgePasses({
102+
edge: 'bottom',
103+
captureState: async () => ({
104+
refused: { reason: 'sparse-tree', detail: 'AX bridge unavailable' },
105+
}),
106+
scroll: async () => {
107+
scrolls += 1;
108+
},
109+
}),
110+
(error: unknown) => {
111+
assert.ok(error instanceof AppError);
112+
assert.equal(error.details?.reason, 'scroll_edge_capture_unreadable');
113+
assert.equal(error.details?.edge, 'bottom');
114+
assert.equal(error.details?.captureRefusal, 'sparse-tree');
115+
assert.match(error.message, /AX bridge unavailable/);
116+
return true;
117+
},
118+
);
119+
assert.equal(scrolls, 0);
120+
});

‎packages/capture-kit/src/snapshot/scroll-edge-state.ts‎

Lines changed: 112 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,22 @@ export type ScrollEdgeState = {
3535
fingerprint?: string;
3636
};
3737

38+
/**
39+
* Why a capture cannot answer a scroll loop's question at all. Never an outcome about the content:
40+
* a failed or sparse read coerced to an empty tree is what the edge analyzer reads as "no hidden
41+
* content", so it has to be refused before anything decides on it.
42+
*/
43+
export type ScrollCaptureRefusal = { reason: 'no-capture' | 'sparse-tree'; detail: string };
44+
45+
/** What one edge capture yields: an analyzed state, or the refusal that kept it from being read. */
46+
export type ScrollEdgeObservation = ScrollEdgeState | { refused: ScrollCaptureRefusal };
47+
48+
function isScrollEdgeRefused(
49+
observation: ScrollEdgeObservation,
50+
): observation is { refused: ScrollCaptureRefusal } {
51+
return 'refused' in observation;
52+
}
53+
3854
export type ScrollEdgeTarget = {
3955
point?: Point;
4056
nodeIndex?: number;
@@ -130,10 +146,10 @@ export function scrollSurfaceIsStuck(recentSignatures: readonly string[]): boole
130146
/**
131147
* Records one pass's surface signature, capped to the trailing window the stuck detector reads. An
132148
* empty signature is skipped so a container with no positioned descendants stays on its edge/pass
133-
* signals instead of stacking blanks. Both scroll loops push through here so "one window" has one
134-
* definition rather than a copy that can drift.
149+
* signals instead of stacking blanks. `runScrollPasses` is its one caller, so every scroll loop
150+
* reads the same window.
135151
*/
136-
export function pushScrollSurfaceSignature(
152+
function pushScrollSurfaceSignature(
137153
recentSignatures: string[],
138154
fingerprint: string | undefined,
139155
window: number,
@@ -143,9 +159,24 @@ export function pushScrollSurfaceSignature(
143159
if (recentSignatures.length > window) recentSignatures.shift();
144160
}
145161

146-
export async function runScrollEdgePasses<TResult>(params: {
147-
edge: ScrollEdge;
148-
captureState: (scope?: string) => Promise<ScrollEdgeState>;
162+
/**
163+
* The one scroll pass loop: observe, refuse an unreadable observation, ask the caller's stop
164+
* condition, then spend a pass. `scroll top`/`scroll bottom` and `scroll --until` differ only in
165+
* what stops them and what they call a failure, so every other rule lives here once: the refusal
166+
* lands before the stop condition reads anything, the budget is checked before the stuck window so
167+
* an explicit small budget still reports itself, and the first observation precedes the first
168+
* gesture so an already-satisfied stop costs no scroll.
169+
*/
170+
export async function runScrollPasses<TObservation, TResult>(params: {
171+
observe: () => Promise<TObservation>;
172+
refuse: (observation: TObservation) => AppError | undefined;
173+
/** True ends the loop with this pass count; a terminal outcome of its own throws instead. */
174+
shouldStop: (observation: TObservation, passes: number) => Promise<boolean>;
175+
passLimit: number;
176+
passLimitError: (passes: number) => AppError;
177+
/** The surface signature the stuck window tracks; `undefined` keeps a pass out of it. */
178+
fingerprint: (observation: TObservation) => Promise<string | undefined>;
179+
noProgressError: (passes: number) => AppError;
149180
scroll: () => Promise<TResult>;
150181
/**
151182
* Waits for the previous fling to come to rest before the next capture. A rubber-banded scroll
@@ -155,19 +186,61 @@ export async function runScrollEdgePasses<TResult>(params: {
155186
*/
156187
settleAfterPass?: () => Promise<void>;
157188
}): Promise<{ passes: number; result?: TResult }> {
158-
const { edge, captureState, scroll, settleAfterPass = async () => {} } = params;
159-
let state = await captureState();
160-
if (state.scope) {
161-
state = await captureState(state.scope);
162-
}
163-
189+
const { observe, refuse, shouldStop, passLimit, scroll, settleAfterPass } = params;
164190
let passes = 0;
165191
let result: TResult | undefined;
166192
const recentSignatures: string[] = [];
167-
pushScrollSurfaceSignature(recentSignatures, state.fingerprint, SCROLL_EDGE_STUCK_WINDOW);
168-
while (state.canScroll) {
169-
if (passes >= SCROLL_EDGE_PASS_LIMIT) {
170-
throw new AppError(
193+
while (true) {
194+
const observation = await observe();
195+
const refusal = refuse(observation);
196+
if (refusal) throw refusal;
197+
if (await shouldStop(observation, passes)) return { passes, result };
198+
if (passes >= passLimit) throw params.passLimitError(passes);
199+
pushScrollSurfaceSignature(
200+
recentSignatures,
201+
await params.fingerprint(observation),
202+
SCROLL_EDGE_STUCK_WINDOW,
203+
);
204+
if (scrollSurfaceIsStuck(recentSignatures)) throw params.noProgressError(passes);
205+
result = await scroll();
206+
passes += 1;
207+
await settleAfterPass?.();
208+
}
209+
}
210+
211+
export async function runScrollEdgePasses<TResult>(params: {
212+
edge: ScrollEdge;
213+
captureState: (scope?: string) => Promise<ScrollEdgeObservation>;
214+
scroll: () => Promise<TResult>;
215+
/** The rest-wait, handed the scope the loop is deciding on so it watches the same container. */
216+
settleAfterPass?: (scope: string | undefined) => Promise<void>;
217+
}): Promise<{ passes: number; result?: TResult }> {
218+
const { edge, captureState, scroll, settleAfterPass } = params;
219+
// The first capture discovers the container; every later one reads the scope the previous state
220+
// named, so the loop verifies against the scroller it decided on rather than the whole tree.
221+
let scope: string | undefined;
222+
let discovered = false;
223+
const observe = async (): Promise<ScrollEdgeObservation> => {
224+
let observation = await captureState(scope);
225+
if (!discovered) {
226+
discovered = true;
227+
if (!isScrollEdgeRefused(observation) && observation.scope) {
228+
observation = await captureState(observation.scope);
229+
}
230+
}
231+
if (!isScrollEdgeRefused(observation)) scope = observation.scope;
232+
return observation;
233+
};
234+
return await runScrollPasses({
235+
observe,
236+
refuse: (observation) =>
237+
isScrollEdgeRefused(observation)
238+
? buildScrollEdgeCaptureError(edge, observation.refused)
239+
: undefined,
240+
shouldStop: async (observation) => !isScrollEdgeRefused(observation) && !observation.canScroll,
241+
passLimit: SCROLL_EDGE_PASS_LIMIT,
242+
passLimitError: (passes) =>
243+
new AppError(
171244
'COMMAND_FAILED',
172245
`scroll ${edge} reached the safety limit before the snapshot showed the edge`,
173246
{
@@ -176,21 +249,13 @@ export async function runScrollEdgePasses<TResult>(params: {
176249
passes,
177250
hint: 'The scoped scroll container still reports hidden content. Run scroll <dir> --until <selector> to stop on the element you are after, or snapshot -i to inspect the current state.',
178251
},
179-
);
180-
}
181-
182-
result = await scroll();
183-
passes += 1;
184-
await settleAfterPass();
185-
state = await captureState(state.scope);
186-
187-
pushScrollSurfaceSignature(recentSignatures, state.fingerprint, SCROLL_EDGE_STUCK_WINDOW);
188-
if (state.canScroll && scrollSurfaceIsStuck(recentSignatures)) {
189-
throw buildScrollEdgeNoProgressError(edge, passes);
190-
}
191-
}
192-
193-
return { passes, result };
252+
),
253+
fingerprint: async (observation) =>
254+
isScrollEdgeRefused(observation) ? undefined : observation.fingerprint,
255+
noProgressError: (passes) => buildScrollEdgeNoProgressError(edge, passes),
256+
scroll,
257+
...(settleAfterPass ? { settleAfterPass: async () => await settleAfterPass(scope) } : {}),
258+
});
194259
}
195260

196261
/**
@@ -278,6 +343,22 @@ export function scrollNoProgressHint(
278343
);
279344
}
280345

346+
function buildScrollEdgeCaptureError(edge: ScrollEdge, refusal: ScrollCaptureRefusal): AppError {
347+
return new AppError(
348+
'COMMAND_FAILED',
349+
`scroll ${edge} could not read the screen: ${refusal.detail}`,
350+
{
351+
reason: 'scroll_edge_capture_unreadable',
352+
edge,
353+
captureRefusal: refusal.reason,
354+
hint:
355+
refusal.reason === 'no-capture'
356+
? 'Run snapshot -i to see whether the app is producing an accessibility tree at all, and retry once it does.'
357+
: 'The accessibility tree came back sparse, so it cannot show whether content remains. Run screenshot, inspect the image, and navigate by coordinates until snapshot -i reports a full tree.',
358+
},
359+
);
360+
}
361+
281362
function buildScrollEdgeVerificationError(
282363
edge: ScrollEdge,
283364
scope: string | undefined,

0 commit comments

Comments
 (0)