Repository navigation
Expand file tree
/
Copy pathsettle.ts
More file actions
475 lines (449 loc) · 20.2 KB
/
Copy pathsettle.ts
File metadata and controls
475 lines (449 loc) · 20.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
import type { Point, SnapshotNode } from '@agent-device/kernel/snapshot';
import type {
AgentDeviceRuntime,
CommandContext,
CommandSessionRecord,
} from '@agent-device/contracts/runtime-contract';
import { isSparseSnapshotQualityVerdict } from '@agent-device/capture-kit/snapshot-quality-verdict';
import { buildSnapshotDiff } from '@agent-device/capture-kit/snapshot-diff';
import { displayLabel, formatRole } from '@agent-device/capture-kit/snapshot-lines';
import {
collectSettleChromeRefs,
withoutSettleChrome,
} from '@agent-device/capture-kit/snapshot-chrome';
import type {
InteractionEvidence,
PostActionSurfaceChange,
ResolvedInteractionTarget,
SettleObservation,
SettleParams,
SettleTailEntry,
SurfaceScopedNodes,
} from '@agent-device/contracts/interaction';
import type { RuntimeCommand } from '../../runtime-types.ts';
import type { CapturedSnapshot } from './selector-read-shared.ts';
import {
DEFAULT_STABLE_QUIET_MS,
DEFAULT_STABLE_TIMEOUT_MS,
runStableCaptureLoop,
TINY_STABLE_TREE_HINT,
TINY_STABLE_TREE_NODE_COUNT,
} from './stable-capture.ts';
import {
crossSurfaceSettleHint,
resolvePostActionSurfaceChange,
summarizePostActionEvidence,
surfaceScopedNodes,
} from './post-action-surface.ts';
/**
* `--settle` (#1101): after a mutating command, wait for the UI to go quiet
* (wait stable's loop, shared via stable-capture.ts) and return the settled
* DIFF against the pre-action tree in the same response — one round trip
* instead of the act → observe pair.
*
* Two entry points over one engine ({@link settleAfterAction}):
* {@link settleAfterInteraction} for the targeted touch commands, which take
* their baseline and proximity point from the resolution, and
* {@link settleObservationCommand} for the target-less generic route
* (`scroll`/`back`, #1638), which supplies the baseline itself and is reached
* as a runtime command because the daemon may not import `commands/`.
*
* Best-effort by contract: this module never throws. The action already
* succeeded when it runs; observation quality is advisory (same principle as
* `--verify` evidence).
*/
export type SettleOutcome = {
observation: SettleObservation;
/** The final capture; doubles as the `--verify` evidence source. */
settledCapture?: SurfaceScopedNodes;
};
// Changed-lines bound: the settled diff is the response payload, and unbounded
// added/removed lists on a full screen transition would crowd out everything
// else. The summary always carries the true counts.
const MAX_SETTLE_DIFF_LINES = 80;
// Unchanged-interactive-tail bound: same token-budget principle as the diff
// line cap, sized smaller since the tail is a fallback list, not the primary
// payload.
const MAX_SETTLE_TAIL_ENTRIES = 20;
export const NEVER_SETTLED_HINT =
'The UI kept changing for the whole settle budget (animation, carousel, or ticker?), so no settled diff is shown. Raise --timeout, wait for specific content, or take a fresh snapshot.';
const SETTLE_CAPTURE_STALLED_HINT =
'A snapshot capture stalled past the settle budget, so no settled diff is shown. The action itself succeeded; observe with wait stable or snapshot.';
export async function settleAfterInteraction(
runtime: AgentDeviceRuntime,
options: CommandContext,
params: SettleParams & { resolved: ResolvedInteractionTarget },
): Promise<SettleOutcome> {
return await settleAfterAction(runtime, options, {
...params,
baseline: await resolveSettleBaseline(runtime, options, params.resolved),
actionPoint: params.resolved.point,
});
}
export type SettleObservationCommandOptions = CommandContext &
SettleParams & {
/** The pre-action tree the settled diff is taken against, and the surface it describes. */
baseline: SurfaceScopedNodes;
};
/**
* The target-less settle as a RUNTIME COMMAND (#1638), which is how the daemon
* reaches it: `scroll` and `back` run the generic route, and that dispatcher
* may not import the command surface (R2) — it composes an `AgentDeviceRuntime`
* and calls commands through it, exactly as the touch handlers do. Returns the
* observation alone; the target-less path has no `--verify` companion to feed,
* so the settled node list stays internal.
*/
export const settleObservationCommand: RuntimeCommand<
SettleObservationCommandOptions,
SettleObservation
> = async (runtime, options) => (await settleAfterAction(runtime, options, options)).observation;
/**
* The target-less engine (#1638), for mutations that change the screen without
* resolving an element. Same loop, storage, hints, and diff bounds as the
* interaction entry point — only the two things a resolution would have
* supplied come from the caller:
*
* - `baseline` is the diff baseline and the surface it describes. On the generic
* route the nodes are the session's STORED pre-action tree, which may be
* several commands older than the action, so the diff honestly reads "settled
* tree vs the last tree you observed" rather than press's freshly resolved
* pre-action capture.
* - `actionPoint` is absent: with no point there is nothing to self-echo
* against, so the tail's self-echo exclusion simply never fires.
*/
async function settleAfterAction(
runtime: AgentDeviceRuntime,
options: CommandContext,
params: SettleParams & { baseline: SurfaceScopedNodes; actionPoint?: Point },
): Promise<SettleOutcome> {
const quietMs = params.quietMs ?? DEFAULT_STABLE_QUIET_MS;
const timeoutMs = params.timeoutMs ?? DEFAULT_STABLE_TIMEOUT_MS;
const base: SettleObservation = { settled: false, waitedMs: 0, captures: 0, quietMs, timeoutMs };
try {
const outcome = await runStableCaptureLoop(runtime, options, {
quietMs,
timeoutMs,
resetBudgetOnPrivateAxRecovery: true,
transitionBaselineNodes: params.baseline.nodes,
});
return await readSettledOutcome(runtime, options, params, base, outcome);
} catch (error) {
// Never fail the action over the observation: report that settling itself
// broke and let the caller fall back to an explicit snapshot.
return {
observation: {
...base,
hint: `Settle observation unavailable (${error instanceof Error ? error.message : String(error)}). The action itself succeeded; take a snapshot to observe the result.`,
},
};
}
}
/** Turns a finished stable-capture loop into the settled observation payload. */
async function readSettledOutcome(
runtime: AgentDeviceRuntime,
options: CommandContext,
params: { baseline: SurfaceScopedNodes; actionPoint?: Point },
base: SettleObservation,
outcome: Awaited<ReturnType<typeof runStableCaptureLoop>>,
): Promise<SettleOutcome> {
const observation: SettleObservation = {
...base,
settled: outcome.settled,
waitedMs: outcome.waitedMs,
captures: outcome.captures,
};
if (!outcome.lastCapture) {
return {
observation: {
...observation,
hint: outcome.stalled ? SETTLE_CAPTURE_STALLED_HINT : NEVER_SETTLED_HINT,
},
};
}
const { stored, session } = await storeSettledSnapshot(runtime, options, outcome.lastCapture);
const settledCapture = surfaceScopedNodes(outcome.lastCapture.snapshot);
const settledNodes = settledCapture.nodes;
// A settled capture of an in-place system surface (a web sign-in sheet) and a pre-action capture
// of the app describe different surfaces (#2438). The diff below would then be a whole-surface
// replacement presented as change within one surface, refs included, so it is refused and the
// transition is disclosed instead.
const surfaceChange = resolvePostActionSurfaceChange(params.baseline, settledCapture);
return {
observation: {
...observation,
...(surfaceChange ? { surfaceChange } : {}),
// The diff (with its added-line refs) is only attached when the settled
// tree actually became the stored session snapshot: those refs must be
// valid against the tree the next @ref command resolves on. The daemon
// treats `diff` presence as "this response issues refs". Unsettled
// captures are intentionally diff-less: they are not a stable
// observation, so surfacing refs would invite agents to act on
// advisory state.
...(outcome.settled && stored && !surfaceChange
? buildSettleDiffAndTail(
params.baseline.nodes,
settledNodes,
params.actionPoint,
session?.appBundleId,
)
: {}),
...resolveSettleHint(outcome, stored, settledNodes.length, surfaceChange),
},
settledCapture,
};
}
/**
* `--settle --verify` composition: the settle loop's final capture doubles as
* the verify evidence source, so the pair costs zero extra captures. Without a
* final capture there is no evidence — best-effort, like verify itself.
*/
export function settleEvidence(
settledCapture: SurfaceScopedNodes | undefined,
baseline: SurfaceScopedNodes | undefined,
): InteractionEvidence | undefined {
if (!settledCapture) return undefined;
return summarizePostActionEvidence(settledCapture, baseline);
}
async function resolveSettleBaseline(
runtime: AgentDeviceRuntime,
options: CommandContext,
resolved: ResolvedInteractionTarget,
): Promise<SurfaceScopedNodes> {
const session = await runtime.sessions.get(options.session ?? 'default');
// A ref is authorized against the stored ref frame. Keep that visible presentation as the
// transition baseline: a best-effort evidence recapture can recover through private AX and see
// covered background controls that were not actionable when the ref was issued.
// Resolved-target evidence is best-effort at the contracts boundary. The session still owns the
// authoritative ref frame, so reuse it rather than silently turning a missing optional field
// into an empty transition/diff baseline. Fall back to the latest observation for point targets
// and pre-frame sessions.
return (
authorizedRefBaseline(resolved, session) ??
nonEmptyBaseline(resolved.preAction) ??
sessionBaseline(session)
);
}
function authorizedRefBaseline(
resolved: ResolvedInteractionTarget,
session: CommandSessionRecord | undefined,
): SurfaceScopedNodes | undefined {
if (resolved.kind !== 'ref') return undefined;
const frame = session?.refFrameSnapshot;
return frame ? nonEmptyBaseline(surfaceScopedNodes(frame)) : undefined;
}
function sessionBaseline(session: CommandSessionRecord | undefined): SurfaceScopedNodes {
const tree = session?.refFrameSnapshot ?? session?.snapshot;
return tree ? surfaceScopedNodes(tree) : { nodes: [] };
}
function nonEmptyBaseline(
baseline: SurfaceScopedNodes | undefined,
): SurfaceScopedNodes | undefined {
return baseline?.nodes.length ? baseline : undefined;
}
function buildSettleDiff(
baselineNodes: SnapshotNode[],
settledNodes: SnapshotNode[],
appBundleId: string | undefined,
): NonNullable<SettleObservation['diff']> {
// Flattened compare, like `diff -i`: both sides are interactive-flavored
// captures and depth jitter across captures should not read as change. When
// the baseline came from a richer stored tree (ref targets reuse the session
// snapshot), extra baseline-only lines surface as removals — advisory noise,
// the same baseline caveat --verify's changedFromBefore already accepts.
const diff = buildSnapshotDiff(
withoutSettleChrome(baselineNodes, appBundleId),
withoutSettleChrome(settledNodes, appBundleId),
{ flatten: true, withRefs: true },
);
const changed = diff.lines.filter((line) => line.kind !== 'unchanged');
const lines = capSettleDiffLines(changed).map((line) => ({
kind: line.kind as 'added' | 'removed',
text: line.text,
...(line.ref ? { ref: line.ref } : {}),
}));
return {
summary: diff.summary,
lines,
...(changed.length > lines.length ? { truncated: true as const } : {}),
};
}
function buildSettleDiffAndTail(
baselineNodes: SnapshotNode[],
settledNodes: SnapshotNode[],
actionPoint: Point | undefined,
appBundleId: string | undefined,
): Pick<SettleObservation, 'diff' | 'tail' | 'tailTruncated'> {
const diff = buildSettleDiff(baselineNodes, settledNodes, appBundleId);
return { diff, ...buildSettleTail(diff, settledNodes, actionPoint, appBundleId) };
}
/**
* Unchanged interactive refs tail: attached ONLY when the settled diff carries
* zero MEANINGFUL added-line refs (a modal-dismiss/toast-only diff shows
* removals but nothing added, so the next actionable target is otherwise
* invisible). "Meaningful" excludes two ref classes that are provably not a
* NEW target (live-verified on the July 2026 React Navigation benchmark flow):
*
* - Keyboard chrome refs: a fill that summons the keyboard adds the keyboard
* container line, but that is not a next target the caller asked for.
* - Self-echo refs: added lines whose settled node's rect contains the action
* point are re-descriptions of the element the caller JUST acted on (a
* filled field re-labels itself with its new value, and its ancestor
* wrappers inherit that label), not something new to press next.
*
* Refs already present on the diff's added lines (meaningful or not) are also
* excluded from the tail itself so it never repeats what the diff already
* handed the caller.
*/
function buildSettleTail(
diff: NonNullable<SettleObservation['diff']>,
settledNodes: SnapshotNode[],
actionPoint: Point | undefined,
appBundleId: string | undefined,
): Pick<SettleObservation, 'tail' | 'tailTruncated'> {
const addedRefs = new Set(
diff.lines.filter((line) => line.kind === 'added' && line.ref).map((line) => line.ref),
);
const chromeRefs = collectSettleChromeRefs(settledNodes, appBundleId);
const byRef = new Map(settledNodes.filter((node) => node.ref).map((node) => [node.ref, node]));
const hasMeaningfulAddedRef = [...addedRefs].some(
(ref) =>
ref !== undefined && !chromeRefs.has(ref) && !isSelfEchoNode(byRef.get(ref), actionPoint),
);
if (hasMeaningfulAddedRef) return {};
return buildSettleTailEntries(settledNodes, addedRefs, appBundleId);
}
function isSelfEchoNode(node: SnapshotNode | undefined, actionPoint: Point | undefined): boolean {
if (!node?.rect || !actionPoint) return false;
const { x, y, width, height } = node.rect;
return (
actionPoint.x >= x &&
actionPoint.x <= x + width &&
actionPoint.y >= y &&
actionPoint.y <= y + height
);
}
// Structural container roles that survive an interactive-only capture (as a
// lone root or alongside real content) but are never a next actionable
// target: application/window chrome, not a control. `snapshot -i` itself
// still lists these lines, but the tail exists specifically to name pressable
// targets, so it drops them rather than spend budget on chrome.
const STRUCTURAL_TAIL_ROLES = new Set(['application', 'window']);
/**
* The filtering/cap step behind `buildSettleTail`, split out so the dedup
* rule (excludeRefs) is unit-testable independent of the trigger condition
* above.
*
* Inclusion bar matches what `snapshot -i` itself would show for the same
* interactive-only capture: presence in `settledNodes` (already filtered to
* interactive content upstream) IS the interactivity bar, so this does NOT
* additionally require `hittable === true`. A real dismiss-animation capture
* routinely reports transient buttons as `hittable: false`/`undefined` right
* after the dismissing element leaves — requiring `hittable === true` here
* was stricter than `snapshot -i`'s own bar and silently dropped exactly the
* buttons the tail exists to surface (#1167 post-merge benchmark). Structural
* application/window chrome, any keyboard container/chrome subtree (iOS), and
* Android IME/persistent-system chrome (#1198) are excluded on top of that
* bar: never a next actionable target either way.
*/
export function buildSettleTailEntries(
settledNodes: SnapshotNode[],
excludeRefs: ReadonlySet<string | undefined>,
appBundleId?: string,
): Pick<SettleObservation, 'tail' | 'tailTruncated'> {
const chromeRefs = collectSettleChromeRefs(settledNodes, appBundleId);
const candidates = settledNodes.filter(
(node) =>
node.ref &&
node.interactionBlocked !== 'covered' &&
!excludeRefs.has(node.ref) &&
!chromeRefs.has(node.ref) &&
!STRUCTURAL_TAIL_ROLES.has(formatRole(node.type ?? 'Element')),
);
if (candidates.length === 0) return {};
const tail: SettleTailEntry[] = candidates.slice(0, MAX_SETTLE_TAIL_ENTRIES).map((node) => {
const role = formatRole(node.type ?? 'Element');
const label = displayLabel(node, role);
return { ref: node.ref, role, ...(label ? { label } : {}) };
});
return {
tail,
...(candidates.length > tail.length ? { tailTruncated: true as const } : {}),
};
}
/**
* Truncation policy: added lines win. They carry the settled tree's fresh
* refs — the actionable half of the diff — while removals only describe what
* left the screen (the summary still counts them). Relative order within each
* kind is preserved; removals fill whatever budget the additions leave.
*/
function capSettleDiffLines<T extends { kind: string }>(changed: T[]): T[] {
if (changed.length <= MAX_SETTLE_DIFF_LINES) return changed;
const added = changed.filter((line) => line.kind === 'added');
const keptAdded = new Set(added.slice(0, MAX_SETTLE_DIFF_LINES));
let removedBudget = MAX_SETTLE_DIFF_LINES - keptAdded.size;
const kept: T[] = [];
for (const line of changed) {
if (keptAdded.has(line)) {
kept.push(line);
} else if (line.kind === 'removed' && removedBudget > 0) {
kept.push(line);
removedBudget -= 1;
}
}
return kept;
}
/**
* The settled observation's hint. A surface change (#2438) is stated alongside whatever the loop
* itself reports rather than in place of it: the transition explains the missing diff, and the
* loop's own verdict (stalled, never settled, sparse, tiny tree) still explains the capture.
*/
function resolveSettleHint(
outcome: { settled: boolean; stalled: boolean },
stored: boolean,
settledNodeCount: number,
surfaceChange: PostActionSurfaceChange | undefined,
): { hint?: string } {
const loopHint = resolveSettleLoopHint(outcome, stored, settledNodeCount).hint;
if (!surfaceChange) return loopHint === undefined ? {} : { hint: loopHint };
const surfaceHint = crossSurfaceSettleHint(surfaceChange);
return { hint: loopHint === undefined ? surfaceHint : `${surfaceHint} ${loopHint}` };
}
function resolveSettleLoopHint(
outcome: { settled: boolean; stalled: boolean },
stored: boolean,
settledNodeCount: number,
): { hint?: string } {
if (outcome.stalled) return { hint: SETTLE_CAPTURE_STALLED_HINT };
if (!outcome.settled) return { hint: NEVER_SETTLED_HINT };
if (!stored) {
return {
hint: 'Settled on a sparse, unreadable tree — the diff is omitted and its refs/selectors are invalid. Use screenshot as visual truth and coordinate taps; retry snapshot after navigating.',
};
}
// Same weak-readiness signal wait stable reports: a settled-but-tiny tree
// usually means a splash/loading surface, not real content.
if (settledNodeCount < TINY_STABLE_TREE_NODE_COUNT) return { hint: TINY_STABLE_TREE_HINT };
return {};
}
// The settle loop itself captures with updateSession: false (a capture that
// later stalls must not race a session write past the response). The FINAL
// capture is stored so follow-up snapshots/selectors see the latest surface.
// Only settled captures issue a diff/ref payload; an unsettled stored capture
// replaces the observation without issuing refs — it does NOT touch the ref
// frame. Read staleness is frame-derived: the frame expires at a side-effect
// seam (ADR 0014), not because a fresh observation was stored here.
// Sparse-quality captures are not stored (mirroring captureSelectorSnapshot)
// and therefore issue no refs. The fetched session is returned alongside
// `stored` so the caller can read `appBundleId` for settle-chrome scoping
// (#1198) without a second `sessions.get` round trip.
async function storeSettledSnapshot(
runtime: AgentDeviceRuntime,
options: CommandContext,
capture: CapturedSnapshot,
): Promise<{ stored: boolean; session?: CommandSessionRecord }> {
if (isSparseSnapshotQualityVerdict(capture.snapshot.snapshotQuality)) return { stored: false };
const session = await runtime.sessions.get(options.session ?? 'default');
if (!session) return { stored: false };
await runtime.sessions.set({ ...session, snapshot: capture.snapshot });
return { stored: true, session };
}