Skip to content

Commit 664b713

Browse files
authored
feat(interaction): opt-in --verify evidence for press/click/fill (#1064)
* feat(interaction): opt-in --verify evidence for press/click/fill (#1047) Adds an opt-in --verify flag that returns cheap post-action evidence (foregroundApp, nodeCount, interactiveNodeCount, digest, changedFromBefore) instead of requiring a full follow-up snapshot to confirm a mutating command had an effect. The digest hashes the (type, label, identifier) multiset of an interactive-only capture, order-independent so it doesn't flip on harmless re-ordering; the node tree is never serialized back to the client, only the digest and counts. Default behavior (no --verify) is byte-identical to today. Implements the approved design from the #1047 issue comment: - src/utils/ax-digest.ts: new standalone digest module. - Pre-action digest reuses the snapshot the resolution path already captures for ref/selector targets (zero extra cost); point targets opt into one extra baseline capture only when --verify is set. - Post-action: one interactive-only capture through the same capture helper, digested and discarded. - --verify threaded through the CLI flag schema, MCP input schema, interactionResultExtra allowlist, and MCP output schemas, following the same plumbing as --double-tap and the #1040 targetHittable precedent. - The native-ref/direct-iOS-selector fast paths are skipped when --verify is set, since they bypass the resolution/capture path evidence depends on. * test: cover press --verify with a provider scenario; classify the flag CI's architecture-progress gate failed on 1 unclassified public flag. --verify drives real device captures, so it belongs in the device-observable list backed by an actual provider scenario rather than the intentionally-outside bucket: the new scenario asserts evidence (changedFromBefore, digest, nodeCount) on press @ref --verify, that the verify capture's tree is never serialized into the response, and that the transcript completes (snapshot -> tap -> verify snapshot; the @ref path reuses the session snapshot as its baseline, so no extra resolution capture entry exists). * fix: type the verify scenario transcript entries * fix: include interaction extras in the fill @ref response branch The ref branch of dispatchFillViaRuntime rebuilt responseData from backendResult/coordinates, dropping interactionResultExtra(result) — so fill @ref --verify returned no evidence even though the post-action capture ran (live E2E gap found in PR #1064 review). Spreading the extras also gives fill @ref the same ref/refLabel/selectorChain (and conditional targetHittable/hint) fields press @ref already returns. Adds daemon tests for fill @ref --verify evidence and for the no-verify path staying evidence-free with no post-action capture.
1 parent 4711bc4 commit 664b713

18 files changed

Lines changed: 980 additions & 12 deletions

File tree

‎scripts/integration-progress-model.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,7 @@ function summarizeProviderScenarioFlagCoverage(files) {
184184
['batchMaxSteps', 'batch max-step guard', ['maxSteps']],
185185
['findFirst', 'find first disambiguation'],
186186
['findLast', 'find last disambiguation'],
187+
['verify', 'post-action evidence capture on press/click/fill'],
187188
];
188189
const sources = files.map((file) => fs.readFileSync(file, 'utf8')).join('\n');
189190
return flagTargets.map(([key, reason, aliases = []]) => {

‎src/cli/parser/cli-flags.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ export type CliFlags = CloudProviderProfileFields &
105105
jitterPx?: number;
106106
pixels?: number;
107107
doubleTap?: boolean;
108+
verify?: boolean;
108109
clickButton?: ClickButton;
109110
backMode?: BackMode;
110111
pauseMs?: number;
@@ -839,6 +840,14 @@ const FLAG_DEFINITIONS: readonly FlagDefinition[] = [
839840
usageLabel: '--double-tap',
840841
usageDescription: 'Use double-tap gesture per press iteration',
841842
},
843+
{
844+
key: 'verify',
845+
names: ['--verify'],
846+
type: 'boolean',
847+
usageLabel: '--verify',
848+
usageDescription:
849+
'Capture cheap post-action evidence (AX digest, node counts, changedFromBefore) instead of a follow-up snapshot',
850+
},
842851
{
843852
key: 'clickButton',
844853
names: ['--button'],

‎src/client/client-normalizers.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -342,6 +342,7 @@ export function buildFlags(options: InternalRequestOptions): CommandFlags {
342342
jitterPx: options.jitterPx,
343343
pixels: options.pixels,
344344
doubleTap: options.doubleTap,
345+
verify: options.verify,
345346
clickButton: options.clickButton,
346347
pauseMs: options.pauseMs,
347348
pattern: options.pattern,

‎src/client/client-types.ts‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -634,12 +634,20 @@ export type ClickOptions = DeviceCommandBaseOptions &
634634
InteractionTarget &
635635
RepeatedPressOptions & {
636636
button?: ClickButton;
637+
/**
638+
* Opt-in (#1047): return cheap post-action evidence (AX digest, node counts,
639+
* changedFromBefore) in the response instead of requiring a follow-up
640+
* snapshot to confirm the action had an effect.
641+
*/
642+
verify?: boolean;
637643
};
638644

639645
export type PressOptions = DeviceCommandBaseOptions &
640646
SelectorSnapshotCommandOptions &
641647
InteractionTarget &
642-
RepeatedPressOptions;
648+
RepeatedPressOptions & {
649+
verify?: boolean;
650+
};
643651

644652
export type LongPressOptions = DeviceCommandBaseOptions &
645653
SelectorSnapshotCommandOptions &
@@ -692,6 +700,7 @@ export type FillOptions = DeviceCommandBaseOptions &
692700
InteractionTarget & {
693701
text: string;
694702
delayMs?: number;
703+
verify?: boolean;
695704
};
696705

697706
export type ScrollOptions = DeviceCommandBaseOptions & {
@@ -918,6 +927,7 @@ type CommandExecutionOptions = Partial<ScreenshotRequestFlags> & {
918927
jitterPx?: number;
919928
pixels?: number;
920929
doubleTap?: boolean;
930+
verify?: boolean;
921931
clickButton?: ClickButton;
922932
pauseMs?: number;
923933
pattern?: SwipePattern;

‎src/commands/interaction/index.ts‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -70,15 +70,15 @@ const interactionCliSchemas = {
7070
usageOverride: 'click <x y|@ref|selector>',
7171
positionalArgs: ['target'],
7272
allowsExtraPositionals: true,
73-
allowedFlags: [...REPEATED_TOUCH_FLAGS, 'clickButton', ...SELECTOR_SNAPSHOT_FLAGS],
73+
allowedFlags: [...REPEATED_TOUCH_FLAGS, 'clickButton', 'verify', ...SELECTOR_SNAPSHOT_FLAGS],
7474
},
7575
press: {
7676
usageOverride: 'press <x y|@ref|selector>',
7777
helpDescription:
7878
'Short press a semantic UI target by ref, selector, or point. For native context menus or hold gestures, use longpress <target> <durationMs> instead of press --hold-ms.',
7979
positionalArgs: ['targetOrX', 'y?'],
8080
allowsExtraPositionals: true,
81-
allowedFlags: [...REPEATED_TOUCH_FLAGS, ...SELECTOR_SNAPSHOT_FLAGS],
81+
allowedFlags: [...REPEATED_TOUCH_FLAGS, 'verify', ...SELECTOR_SNAPSHOT_FLAGS],
8282
},
8383
longpress: {
8484
usageOverride: 'longpress <x y|@ref|selector> [durationMs]',
@@ -114,7 +114,7 @@ const interactionCliSchemas = {
114114
usageOverride: 'fill <x> <y> <text> | fill <@ref|selector> <text>',
115115
positionalArgs: ['targetOrX', 'yOrText', 'text?'],
116116
allowsExtraPositionals: true,
117-
allowedFlags: [...SELECTOR_SNAPSHOT_FLAGS, 'delayMs'],
117+
allowedFlags: [...SELECTOR_SNAPSHOT_FLAGS, 'delayMs', 'verify'],
118118
},
119119
scroll: {
120120
usageOverride: 'scroll <direction|top|bottom> [amount] [--pixels <n>] [--duration-ms <ms>]',
@@ -342,6 +342,7 @@ function toClickOptions(input: ClickInput): ClickOptions {
342342
...toSelectorSnapshotOptions(input),
343343
...toRepeatedOptions(input),
344344
button: input.button,
345+
verify: input.verify,
345346
};
346347
}
347348

@@ -351,6 +352,7 @@ function toPressOptions(input: PressInput): PressOptions {
351352
...toClientInteractionTarget(input.target),
352353
...toSelectorSnapshotOptions(input),
353354
...toRepeatedOptions(input),
355+
verify: input.verify,
354356
};
355357
}
356358

@@ -361,6 +363,7 @@ function toFillOptions(input: FillInput): FillOptions {
361363
...toSelectorSnapshotOptions(input),
362364
text: input.text,
363365
delayMs: input.delayMs,
366+
verify: input.verify,
364367
};
365368
}
366369

‎src/commands/interaction/interactions.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,12 +37,14 @@ export const interactionCliReaders = {
3737
...repeatedInputFromFlags(flags),
3838
target: targetInputFromClientTarget(readInteractionTargetFromPositionals(positionals)),
3939
button: flags.clickButton,
40+
verify: flags.verify,
4041
}),
4142
press: (positionals, flags) => ({
4243
...commonInputFromFlags(flags),
4344
...selectorSnapshotInputFromFlags(flags),
4445
...repeatedInputFromFlags(flags),
4546
target: targetInputFromClientTarget(readInteractionTargetFromPositionals(positionals)),
47+
verify: flags.verify,
4648
}),
4749
longpress: (positionals, flags) => {
4850
const decoded = readLongPressTargetFromPositionals(positionals);
@@ -80,6 +82,7 @@ export const interactionCliReaders = {
8082
target: targetInputFromClientTarget(decoded.target),
8183
text: decoded.text,
8284
delayMs: flags.delayMs,
85+
verify: flags.verify,
8386
};
8487
},
8588
scroll: (positionals, flags) => ({

‎src/commands/interaction/metadata.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,24 +66,32 @@ const interactionCommandDescriptions = {
6666

6767
type InteractionCommandName = keyof typeof interactionCommandDescriptions;
6868

69+
const verifyField = () =>
70+
booleanField(
71+
'Capture cheap post-action evidence (AX digest, node counts, changedFromBefore) instead of a follow-up snapshot.',
72+
);
73+
6974
const clickFields = {
7075
target: requiredField(interactionTargetField()),
7176
button: enumField(CLICK_BUTTONS, 'Pointer button for platforms that support mouse buttons.'),
7277
...selectorSnapshotFields(),
7378
...repeatedFields(),
79+
verify: verifyField(),
7480
};
7581

7682
const pressFields = {
7783
target: requiredField(interactionTargetField()),
7884
...selectorSnapshotFields(),
7985
...repeatedFields(),
86+
verify: verifyField(),
8087
};
8188

8289
const fillFields = {
8390
target: requiredField(interactionTargetField()),
8491
text: requiredField(stringField('Text to enter into the target.')),
8592
delayMs: integerField('Delay between typed characters.', { min: 0 }),
8693
...selectorSnapshotFields(),
94+
verify: verifyField(),
8795
};
8896

8997
const longPressFields = {

‎src/commands/interaction/runtime/interactions.test.ts‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,56 @@ test('runtime click taps an explicit point without requiring a snapshot', async
2727
assert.deepEqual(result, { kind: 'point', point: { x: 10, y: 20 } });
2828
});
2929

30+
test('runtime click with verify captures a baseline for point targets and reports evidence', async () => {
31+
let captureCount = 0;
32+
const device = createInteractionDevice(selectorSnapshot(), {
33+
captureSnapshot: async () => {
34+
captureCount += 1;
35+
if (captureCount === 1) return { snapshot: selectorSnapshot() };
36+
return { snapshot: makeSnapshotState([]) };
37+
},
38+
tap: async () => ({ ok: true }),
39+
});
40+
41+
const result = await device.interactions.click(
42+
{ kind: 'point', x: 10, y: 20 },
43+
{ session: 'default', verify: true },
44+
);
45+
46+
assert.equal(result.kind, 'point');
47+
assert.ok(result.evidence);
48+
assert.equal(result.evidence?.changedFromBefore, true);
49+
assert.equal(result.evidence?.nodeCount, 0);
50+
assert.equal(captureCount, 2);
51+
});
52+
53+
test('runtime click with verify skips the native ref fast path so evidence can be captured', async () => {
54+
const calls: string[] = [];
55+
let captureCount = 0;
56+
const device = createInteractionDevice(selectorSnapshot(), {
57+
platform: 'web',
58+
captureSnapshot: async () => {
59+
captureCount += 1;
60+
return { snapshot: selectorSnapshot() };
61+
},
62+
tapTarget: async (_context, target) => {
63+
calls.push(target.ref);
64+
return { ref: target.ref.replace(/^@/, '') };
65+
},
66+
tap: async () => ({ ok: true }),
67+
});
68+
69+
const result = await device.interactions.click(ref('@e1'), {
70+
session: 'default',
71+
verify: true,
72+
});
73+
74+
assert.deepEqual(calls, []);
75+
assert.equal(result.kind, 'ref');
76+
assert.ok(result.evidence);
77+
assert.ok(captureCount >= 1);
78+
});
79+
3080
test('runtime click uses backend ref primitive without resolving snapshot geometry', async () => {
3181
const calls: string[] = [];
3282
const device = createInteractionDevice(selectorSnapshot(), {
@@ -162,6 +212,127 @@ test('runtime selector interactions fall back to a full snapshot when interactiv
162212
]);
163213
});
164214

215+
test('runtime press without verify omits evidence entirely', async () => {
216+
const device = createInteractionDevice(selectorSnapshot(), {
217+
tap: async () => ({ ok: true }),
218+
});
219+
220+
const result = await device.interactions.press(selector('label=Continue'), {
221+
session: 'default',
222+
});
223+
224+
assert.equal('evidence' in result, false);
225+
});
226+
227+
test('runtime press with verify reports unchanged evidence when the post-action capture matches', async () => {
228+
const device = createInteractionDevice(selectorSnapshot(), {
229+
tap: async () => ({ ok: true }),
230+
});
231+
232+
const result = await device.interactions.press(selector('label=Continue'), {
233+
session: 'default',
234+
verify: true,
235+
});
236+
237+
assert.equal(result.kind, 'selector');
238+
assert.ok(result.evidence);
239+
assert.equal(result.evidence?.changedFromBefore, false);
240+
assert.equal(result.evidence?.nodeCount, 1);
241+
assert.equal(result.evidence?.interactiveNodeCount, 1);
242+
assert.equal(typeof result.evidence?.digest, 'string');
243+
assert.ok(result.evidence?.digest.startsWith('ax1:'));
244+
});
245+
246+
test('runtime press with verify reports changedFromBefore true when the post-action capture differs', async () => {
247+
let captureCount = 0;
248+
const device = createInteractionDevice(selectorSnapshot(), {
249+
captureSnapshot: async () => {
250+
captureCount += 1;
251+
if (captureCount === 1) return { snapshot: selectorSnapshot() };
252+
return {
253+
snapshot: makeSnapshotState([
254+
{
255+
index: 0,
256+
depth: 0,
257+
type: 'Button',
258+
label: 'Continue',
259+
value: 'Continue',
260+
rect: { x: 10, y: 20, width: 100, height: 40 },
261+
hittable: true,
262+
},
263+
{
264+
index: 1,
265+
depth: 0,
266+
type: 'Text',
267+
label: 'Loading…',
268+
rect: { x: 10, y: 80, width: 100, height: 20 },
269+
hittable: true,
270+
},
271+
]),
272+
};
273+
},
274+
tap: async () => ({ ok: true }),
275+
});
276+
277+
const result = await device.interactions.press(selector('label=Continue'), {
278+
session: 'default',
279+
verify: true,
280+
});
281+
282+
assert.equal(result.kind, 'selector');
283+
assert.ok(result.evidence);
284+
assert.equal(result.evidence?.changedFromBefore, true);
285+
assert.equal(result.evidence?.nodeCount, 2);
286+
});
287+
288+
test('runtime fill without verify omits evidence entirely', async () => {
289+
const device = createInteractionDevice(fillableSnapshot(), {
290+
fill: async () => ({ ok: true }),
291+
});
292+
293+
const result = await device.interactions.fill(selector('label=Email'), 'hi', {
294+
session: 'default',
295+
});
296+
297+
assert.equal('evidence' in result, false);
298+
});
299+
300+
test('runtime fill with verify reports evidence and detects a changed post-action capture', async () => {
301+
let captureCount = 0;
302+
const device = createInteractionDevice(fillableSnapshot(), {
303+
captureSnapshot: async () => {
304+
captureCount += 1;
305+
if (captureCount === 1) return { snapshot: fillableSnapshot() };
306+
return {
307+
snapshot: makeSnapshotState([
308+
{
309+
index: 0,
310+
depth: 0,
311+
type: 'XCUIElementTypeTextField',
312+
label: 'Email',
313+
value: 'hi',
314+
rect: { x: 20, y: 10, width: 60, height: 40 },
315+
hittable: true,
316+
},
317+
]),
318+
};
319+
},
320+
fill: async () => ({ ok: true }),
321+
});
322+
323+
const result = await device.interactions.fill(selector('label=Email'), 'hi', {
324+
session: 'default',
325+
verify: true,
326+
});
327+
328+
assert.equal(result.kind, 'selector');
329+
assert.ok(result.evidence);
330+
// Digest is over (type, label, identifier) only, so a value-only change does
331+
// not flip the digest — this is intentional (see ax-digest.ts docs).
332+
assert.equal(result.evidence?.changedFromBefore, false);
333+
assert.equal(result.evidence?.nodeCount, 1);
334+
});
335+
165336
test('runtime click keeps distinct tab button centers when iOS reports the tab bar as hittable', async () => {
166337
const calls: Point[] = [];
167338
const device = createInteractionDevice(iosTabBarSnapshot(), {

0 commit comments

Comments
 (0)