Skip to content

Commit 5ad083d

Browse files
committed
fix(daemon): route sparse verdicts through selector reads
1 parent 26eca74 commit 5ad083d

9 files changed

Lines changed: 222 additions & 31 deletions

File tree

‎src/__tests__/runtime-selector-read.test.ts‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,61 @@ test('runtime find get_text reads the matched node', async () => {
185185
assert.equal(result.node.label, 'Continue');
186186
});
187187

188+
test('runtime find wait reports sparse snapshot verdicts on the selector-read route', async () => {
189+
const initialSnapshot = selectorSnapshot();
190+
const session = { name: 'default', snapshot: initialSnapshot };
191+
const sessions = {
192+
get: () => session,
193+
set: (record) => {
194+
session.snapshot = record.snapshot ?? session.snapshot;
195+
},
196+
} satisfies CommandSessionStore;
197+
const sparseSnapshot = makeSnapshotState([
198+
{
199+
index: 0,
200+
type: 'Application',
201+
},
202+
]);
203+
const device = createAgentDevice({
204+
backend: {
205+
platform: 'ios',
206+
captureSnapshot: async () => ({
207+
nodes: sparseSnapshot.nodes,
208+
backend: 'xctest',
209+
quality: {
210+
state: 'sparse',
211+
backend: 'private-ax',
212+
reason: 'sparse tree',
213+
reasonCode: 'sparse-tree',
214+
},
215+
}),
216+
} satisfies AgentDeviceBackend,
217+
artifacts: createLocalArtifactAdapter(),
218+
sessions,
219+
policy: localCommandPolicy(),
220+
clock: {
221+
now: () => 0,
222+
sleep: async () => {},
223+
},
224+
});
225+
226+
await assert.rejects(
227+
() =>
228+
device.selectors.find({
229+
session: 'default',
230+
locator: 'text',
231+
query: 'Never appears',
232+
action: 'wait',
233+
timeoutMs: 100,
234+
}),
235+
(error: unknown) =>
236+
error instanceof Error &&
237+
error.message === 'find could not read the current accessibility tree' &&
238+
(error as { details?: { reason?: string } }).details?.reason === 'sparse tree',
239+
);
240+
assert.equal(session.snapshot, initialSnapshot);
241+
});
242+
188243
test('runtime wait can use backend text search', async () => {
189244
const device = createSelectorDevice(selectorSnapshot(), {
190245
findText: true,

‎src/commands/selector-read-shared.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import type {
66
import { AppError } from '../utils/errors.ts';
77
import type { SnapshotNode, SnapshotState } from '../utils/snapshot.ts';
88
import { findNodeByRef, normalizeRef } from '../utils/snapshot.ts';
9+
import { isSparseSnapshotQualityVerdict } from '../utils/snapshot-quality.ts';
910
import { extractReadableText } from '../utils/text-surface.ts';
1011
import { findNodeByLabel, now, toBackendContext } from './selector-read-utils.ts';
1112
import type { SelectorSnapshotInput } from './command-input.ts';
@@ -55,9 +56,14 @@ export async function captureSelectorSnapshot(
5556
nodes: result.nodes ?? [],
5657
truncated: result.truncated,
5758
backend: result.backend as SnapshotState['backend'],
59+
...(result.quality ? { snapshotQuality: result.quality } : {}),
5860
createdAt: now(runtime),
5961
} satisfies SnapshotState);
60-
if (captureOptions.updateSession && session) {
62+
if (
63+
captureOptions.updateSession &&
64+
session &&
65+
!isSparseSnapshotQualityVerdict(snapshot.snapshotQuality)
66+
) {
6167
await runtime.sessions.set({ ...session, snapshot });
6268
}
6369
return { sessionName, session, snapshot };

‎src/commands/selector-read.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@ import type { FindAction, FindLocator } from '../utils/finders.ts';
22
import { findBestMatchesByLocator } from '../utils/finders.ts';
33
import type { SnapshotNode } from '../utils/snapshot.ts';
44
import { findNodeByRef, normalizeRef } from '../utils/snapshot.ts';
5+
import {
6+
isSparseSnapshotQualityVerdict,
7+
type SnapshotQualityVerdict,
8+
} from '../utils/snapshot-quality.ts';
59
import type { AgentDeviceRuntime, CommandContext } from '../runtime-contract.ts';
610
import { AppError } from '../utils/errors.ts';
711
import {
@@ -406,12 +410,22 @@ async function findFirstLocatorMatch(
406410
updateSession: true,
407411
scope: shouldScopeFind(locator) ? options.query : undefined,
408412
});
413+
if (isSparseSnapshotQualityVerdict(capture.snapshot.snapshotQuality)) {
414+
throw sparseSelectorSnapshotError(capture.snapshot.snapshotQuality);
415+
}
409416
const match = findBestMatchesByLocator(capture.snapshot.nodes, locator, options.query, {
410417
requireRect: false,
411418
}).matches[0];
412419
return { capture, match };
413420
}
414421

422+
function sparseSelectorSnapshotError(verdict: SnapshotQualityVerdict): AppError {
423+
return new AppError('COMMAND_FAILED', 'find could not read the current accessibility tree', {
424+
reason: verdict.reason,
425+
hint: 'The snapshot quality verdict is sparse. Use screenshot as visual truth, navigate with coordinates if needed, then retry find after reaching a readable screen.',
426+
});
427+
}
428+
415429
async function waitForSelector(
416430
runtime: AgentDeviceRuntime,
417431
options: WaitCommandOptions,

‎src/daemon/handlers/__tests__/find.test.ts‎

Lines changed: 103 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,6 @@ vi.mock('../../../core/dispatch.ts', async (importOriginal) => {
1616
};
1717
});
1818

19-
vi.mock('../../selector-runtime.ts', () => ({
20-
dispatchFindReadOnlyViaRuntime: vi.fn(async () => null),
21-
}));
22-
2319
import { dispatchCommand } from '../../../core/dispatch.ts';
2420

2521
const mockDispatch = vi.mocked(dispatchCommand);
@@ -164,7 +160,7 @@ test('handleFindCommands click prefers on-screen duplicate text matches', async
164160
expect(invokeCalls[0]!.positionals?.[0]).toBe('@e3');
165161
});
166162

167-
test('handleFindCommands click fails without daemon retry when quality verdict is sparse', async () => {
163+
test('handleFindCommands click tries query-scoped full retry before failing sparse verdict', async () => {
168164
const session = makeSession('default');
169165
session.snapshot = {
170166
nodes: [
@@ -223,17 +219,21 @@ test('handleFindCommands click fails without daemon retry when quality verdict i
223219
},
224220
});
225221
const snapshotCalls = mockDispatch.mock.calls.filter((call) => call[1] === 'snapshot');
226-
expect(snapshotCalls).toHaveLength(1);
222+
expect(snapshotCalls).toHaveLength(2);
227223
expect(snapshotCalls[0]![4]).toMatchObject({
228224
snapshotInteractiveOnly: true,
229225
snapshotCompact: true,
230226
});
227+
expect(snapshotCalls[1]![4]).toMatchObject({
228+
snapshotInteractiveOnly: false,
229+
snapshotCompact: false,
230+
snapshotScope: 'Search',
231+
});
231232
});
232233

233-
test('handleFindCommands wait preserves sparse verdict through snapshot cache hits', async () => {
234-
mockDispatch.mockImplementation(async (_device, command) => {
235-
if (command !== 'snapshot') return {};
236-
return {
234+
test('handleFindCommands click uses query-scoped full retry when sparse verdict recovers', async () => {
235+
const snapshotResponses = [
236+
{
237237
backend: 'xctest',
238238
quality: {
239239
state: 'sparse',
@@ -248,23 +248,50 @@ test('handleFindCommands wait preserves sparse verdict through snapshot cache hi
248248
rect: { x: 0, y: 0, width: 0, height: 0 },
249249
},
250250
],
251-
};
251+
},
252+
{
253+
backend: 'xctest',
254+
quality: {
255+
state: 'healthy',
256+
backend: 'tree',
257+
},
258+
nodes: [
259+
{
260+
index: 0,
261+
type: 'Application',
262+
hittable: false,
263+
rect: { x: 0, y: 0, width: 390, height: 844 },
264+
},
265+
{
266+
index: 1,
267+
type: 'Button',
268+
label: 'Search',
269+
hittable: true,
270+
rect: { x: 80, y: 792, width: 78, height: 48 },
271+
parentIndex: 0,
272+
},
273+
],
274+
},
275+
];
276+
mockDispatch.mockImplementation(async (_device, command) => {
277+
if (command !== 'snapshot') return {};
278+
return snapshotResponses.shift() ?? { nodes: [] };
252279
});
253280

254-
const { response } = await runFindClickScenario({
255-
positionals: ['text', 'Never appears', 'wait', '350'],
281+
const { response, invokeCalls } = await runFindClickScenario({
282+
positionals: ['Search', 'click'],
256283
});
257284

258-
expect(response.ok).toBe(false);
259-
expect(!response.ok && response.error).toMatchObject({
260-
code: 'COMMAND_FAILED',
261-
message: 'find could not read the current accessibility tree',
262-
details: {
263-
reason: 'sparse tree',
264-
hint: expect.stringContaining('snapshot quality verdict is sparse'),
265-
},
285+
expect(response.ok).toBe(true);
286+
expect(invokeCalls[0]!.positionals?.[0]).toBe('@e1');
287+
expect(response.ok ? response.data : undefined).toMatchObject({ x: 119, y: 816 });
288+
const snapshotCalls = mockDispatch.mock.calls.filter((call) => call[1] === 'snapshot');
289+
expect(snapshotCalls).toHaveLength(2);
290+
expect(snapshotCalls[1]![4]).toMatchObject({
291+
snapshotInteractiveOnly: false,
292+
snapshotCompact: false,
293+
snapshotScope: 'Search',
266294
});
267-
expect(mockDispatch).toHaveBeenCalledTimes(1);
268295
});
269296

270297
test('handleFindCommands click retries full snapshot for legacy iOS sparse shape without verdict', async () => {
@@ -614,7 +641,59 @@ test('handleFindCommands wait bypasses snapshot cache while Android freshness re
614641
expect(mockDispatch).toHaveBeenCalledTimes(2);
615642
});
616643

617-
test('handleFindCommands wait reuses the snapshot cache while polling', async () => {
644+
test('handleFindCommands wait reports sparse verdict through selector runtime route', async () => {
645+
const session = makeSession('default');
646+
session.snapshot = {
647+
nodes: [
648+
{
649+
index: 0,
650+
ref: 'e1',
651+
type: 'Button',
652+
label: 'Previous screen action',
653+
rect: { x: 24, y: 600, width: 180, height: 52 },
654+
},
655+
],
656+
createdAt: Date.now(),
657+
backend: 'xctest',
658+
};
659+
const previousSnapshot = session.snapshot;
660+
mockDispatch.mockImplementation(async (_device, command) => {
661+
if (command !== 'snapshot') return {};
662+
return {
663+
backend: 'xctest',
664+
quality: {
665+
state: 'sparse',
666+
backend: 'private-ax',
667+
reason: 'sparse tree',
668+
reasonCode: 'sparse-tree',
669+
},
670+
nodes: [
671+
{
672+
index: 0,
673+
type: 'Application',
674+
},
675+
],
676+
};
677+
});
678+
679+
const { response } = await runFindClickScenario({
680+
positionals: ['text', 'Never appears', 'wait', '350'],
681+
session,
682+
});
683+
684+
expect(response.ok).toBe(false);
685+
expect(session.snapshot).toBe(previousSnapshot);
686+
expect(!response.ok && response.error).toMatchObject({
687+
code: 'COMMAND_FAILED',
688+
message: 'find could not read the current accessibility tree',
689+
details: {
690+
reason: 'sparse tree',
691+
hint: expect.stringContaining('snapshot quality verdict is sparse'),
692+
},
693+
});
694+
});
695+
696+
test('handleFindCommands wait captures fresh snapshots while polling', async () => {
618697
const { response } = await runFindClickScenario({
619698
positionals: ['text', 'Never appears', 'wait', '350'],
620699
nodes: [{ index: 0, depth: 0, type: 'StaticText', label: 'Other text' }],
@@ -624,5 +703,5 @@ test('handleFindCommands wait reuses the snapshot cache while polling', async ()
624703
if (!response.ok) {
625704
expect(response.error.message).toContain('find wait timed out');
626705
}
627-
expect(mockDispatch).toHaveBeenCalledTimes(1);
706+
expect(mockDispatch).toHaveBeenCalledTimes(2);
628707
});

‎src/daemon/handlers/__tests__/react-native.test.ts‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -641,7 +641,20 @@ test('react-native dismiss-overlay reports verified success after a clean post-d
641641
test('react-native dismiss-overlay reports sparse verdict instead of no overlay detected', async () => {
642642
const sessionName = 'rn-sparse-session';
643643
const sessionStore = makeSessionStore();
644-
sessionStore.set(sessionName, makeSession(sessionName));
644+
const session = makeSession(sessionName);
645+
session.snapshot = {
646+
nodes: [
647+
{
648+
index: 0,
649+
ref: 'e1',
650+
label: 'Previous screen action',
651+
rect: { x: 24, y: 600, width: 180, height: 52 },
652+
},
653+
],
654+
createdAt: Date.now(),
655+
};
656+
const previousSnapshot = session.snapshot;
657+
sessionStore.set(sessionName, session);
645658
mockCaptureSnapshot.mockResolvedValue({
646659
snapshot: {
647660
nodes: [
@@ -676,6 +689,7 @@ test('react-native dismiss-overlay reports sparse verdict instead of no overlay
676689
});
677690

678691
expect(response?.ok).toBe(false);
692+
expect(session.snapshot).toBe(previousSnapshot);
679693
expect(mockDispatchCommand).not.toHaveBeenCalled();
680694
expect(!response?.ok && response?.error).toMatchObject({
681695
code: 'COMMAND_FAILED',

‎src/daemon/handlers/find.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,12 @@ function createFindNodeFetcher(params: {
231231
let snapshot = await capture(scope, interactiveOnly);
232232
if (interactiveOnly && isLegacySparseIosInteractiveSnapshot(snapshot)) {
233233
snapshot = await recoverSparseInteractiveSnapshot({ capture, locator, query, scope });
234+
} else if (
235+
interactiveOnly &&
236+
isSparseSnapshotQualityVerdict(snapshot.snapshotQuality) &&
237+
shouldScopeFind(locator)
238+
) {
239+
snapshot = await recoverSparseVerdictWithQueryScope({ capture, query, snapshot });
234240
}
235241
const snapshotResult = {
236242
nodes: snapshot.nodes,
@@ -248,6 +254,19 @@ function createFindNodeFetcher(params: {
248254
};
249255
}
250256

257+
async function recoverSparseVerdictWithQueryScope(params: {
258+
capture: (scope: string | undefined, interactive: boolean) => Promise<SnapshotState>;
259+
query: string;
260+
snapshot: SnapshotState;
261+
}): Promise<SnapshotState> {
262+
const { capture, query, snapshot } = params;
263+
try {
264+
return await capture(query, false);
265+
} catch {
266+
return snapshot;
267+
}
268+
}
269+
251270
/**
252271
* Legacy iOS runners did not report a structured quality verdict. For those mixed-version
253272
* sessions, the one-node application shape still means the runner could not enumerate the tree:

‎src/daemon/handlers/interaction-snapshot.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import type { SnapshotState } from '../../utils/snapshot.ts';
55
import type { ContextFromFlags } from './interaction-common.ts';
66
import { captureSnapshot } from './snapshot-capture.ts';
77
import { setSessionSnapshot } from '../session-snapshot.ts';
8+
import { isSparseSnapshotQualityVerdict } from '../../utils/snapshot-quality.ts';
89

910
export type CaptureSnapshotForSession = (
1011
session: SessionState,
@@ -39,7 +40,9 @@ export async function captureSnapshotForSession(
3940
logPath: dispatchContext.logPath ?? '',
4041
androidFreshnessMode: options.androidFreshnessMode,
4142
});
42-
setSessionSnapshot(session, snapshot);
43-
sessionStore.set(session.name, session);
43+
if (!isSparseSnapshotQualityVerdict(snapshot.snapshotQuality)) {
44+
setSessionSnapshot(session, snapshot);
45+
sessionStore.set(session.name, session);
46+
}
4447
return snapshot;
4548
}

‎src/daemon/handlers/react-native.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,6 @@ async function verifyReactNativeOverlayDismissal(
168168
params.contextFromFlags,
169169
{ interactiveOnly: true },
170170
);
171-
const overlay = analyzeReactNativeOverlay(verificationSnapshot.nodes);
172171
if (isSparseSnapshotQualityVerdict(verificationSnapshot.snapshotQuality)) {
173172
return {
174173
verified: false,
@@ -177,6 +176,7 @@ async function verifyReactNativeOverlayDismissal(
177176
nextCommand: 'agent-device screenshot',
178177
};
179178
}
179+
const overlay = analyzeReactNativeOverlay(verificationSnapshot.nodes);
180180
if (!overlay.detected) {
181181
return {
182182
verified: true,

0 commit comments

Comments
 (0)