Repository navigation
Expand file tree
/
Copy pathinteraction.ts
More file actions
558 lines (518 loc) · 20.4 KB
/
Copy pathinteraction.ts
File metadata and controls
558 lines (518 loc) · 20.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
import type {
PresentedSurface,
Point,
SnapshotKeyboardBandFact,
SnapshotNode,
} from '@agent-device/kernel/snapshot';
import type { ResponseCost } from '@agent-device/kernel/contracts';
import type { ClickButton } from './click-button.ts';
import type { FillUnconfirmedVerification } from './fill-evidence.ts';
export type SelectorTarget = {
kind: 'selector';
selector: string;
};
export type RefTarget = {
kind: 'ref';
ref: string;
fallbackLabel?: string;
};
export type ElementTarget = SelectorTarget | RefTarget;
export type PointTarget = {
kind: 'point';
x: number;
y: number;
};
export type InteractionTarget = ElementTarget | PointTarget;
export type ResolvedTarget =
| {
kind: 'selector';
selector: string;
}
| {
kind: 'ref';
ref: string;
};
/** The decisive criterion separating a resolveSelectorChain winner from its strongest runner-up (ADR 0012). */
export type DisambiguationTiebreak =
| 'visible'
| 'deepest'
| 'smallest-area'
| 'structural-equivalence';
/**
* A disambiguation winner or losing alternative. `diagnosticRef` is an opaque,
* non-`@` token — never a snapshot ref, never issued via `refsGeneration`,
* never pinnable or usable as an `@ref` target. Strings are UTF-8 truncated
* to 256 bytes.
*/
export type ResolutionDiagnosticEntry = {
diagnosticRef: string;
role?: string;
label?: string;
};
/**
* ADR 0012 decision 2: pre-action disclosure of how the acting path resolved
* its target, including each endpoint of a target-authored drag. Never ref-issuing.
* `direct-ios`/`not-observed` = the XCTest fast path has no daemon tree to
* report from; `ref`/`label-fallback` = a stale `@ref` recovered via
* first-match label lookup, never exact ref provenance; `alternatives` holds
* at most 5 losing candidates, winner excluded.
*/
export type ResolutionDisclosure =
| { source: 'runtime'; phase: 'pre-action'; kind: 'unique' }
| {
source: 'runtime';
phase: 'pre-action';
kind: 'disambiguated';
matchCount: number;
winnerDiagnostic: ResolutionDiagnosticEntry;
tiebreak: DisambiguationTiebreak;
alternatives: ResolutionDiagnosticEntry[];
}
| { source: 'ref'; phase: 'pre-action'; kind: 'exact' }
| { source: 'ref'; phase: 'pre-action'; kind: 'label-fallback' }
| { source: 'direct-ios'; kind: 'not-observed' };
/**
* #1280 (ADR 0012 decision 3 amendment): recording-only side channel — the
* labeled descendant an identity-empty press container was retargeted to.
* Consumed exclusively at the recording boundary
* (`interaction-touch-response.ts`): the recorded action entry and its
* `target-v1` evidence key off this node/chain/ref-label, while every
* response payload keeps describing the dispatched container. Never
* serialized into a response.
*/
export type RecordingTargetOverride = {
node: SnapshotNode;
selectorChain: string[];
refLabel?: string;
};
/**
* #1654: a target its CALLER already resolved, against a tree the caller
* captured itself, handed to the interaction leaf so the leaf does not resolve
* the same `@ref` a second time.
*
* The sole producer is a mutating `find` (`packages/daemon/src/execution/interaction/internal/find.ts`): it
* captures, matches by locator, promotes to a hittable ancestor, and mints
* `@eN` off the node it chose — then re-enters the interaction leaf. Without
* this channel the leaf repeated an in-memory `@eN` lookup after find had
* already selected the node. No reachable production path is currently known
* to advance the tree in that interval; this type removes the duplicate lookup
* structurally and keeps the selected ref/node/tree provenance together.
*
* What this does NOT skip: the shared guards. Occlusion, hittable-ancestor
* promotion, and the off-screen check still run, on this node, at the same
* symbols the ADR 0011 `runtime-ref` cells name — the pre-resolution replaces
* the LOOKUP, not the guarantees.
*
* In-process only, like `RecordingTargetOverride` above: it travels on the
* daemon-only `internal` request channel (`toDaemonRequest` never copies that
* off the wire), carries live node references, and is never serialized into a
* response.
*/
export type PreresolvedInteractionTarget = {
/** The ref minted for `node`; the consumer validates it against the positional target. */
ref: string;
/** The node the caller resolved, after the caller's own promotion. */
node: SnapshotNode;
/**
* The tree `node` came from, minted by `surfaceScopedNodes` from the caller's capture. The guards
* read its siblings for occlusion/viewport, and the adopting leaf's post-action baseline is this
* value as-is, so the surface (#2438) and presentation (#3565) facts travel with the nodes.
*/
tree: SurfaceScopedNodes;
/**
* The keyboard band the capture of `tree` measured, carried so the adopting consumer's keyboard
* guard reads the same band the tree was captured with instead of re-deriving one (#2660).
*/
keyboard?: SnapshotKeyboardBandFact;
};
/**
* One side of a post-action comparison: the nodes, and the SURFACE the capture they came from
* described (#2438: the bundle id of an in-place iOS system surface such as a web sign-in sheet,
* absent for ordinary app content).
*
* One value, never two channels: a capture of the sheet and a capture of the app describe
* different surfaces, so a `--verify` digest comparison or a `--settle` diff across that boundary
* is not about one presentation. Every boundary that carries a baseline carries this type, so
* nodes cannot arrive without the surface they describe.
*/
export type SurfaceScopedNodes = {
nodes: SnapshotNode[];
/** Bundle id of the in-place iOS system surface; absent for ordinary app content. */
surfaceBundleId?: string;
/**
* Whether the producer that captured these nodes read a modal presentation over the app (#3565).
* The settle baseline is carried in this type, so the fact has to travel with the nodes: a consumer
* that arms #1831's dismissal confirmation reads it here instead of guessing from the tree.
* Absent means the capture's producer published no fact.
*/
presentedSurface?: PresentedSurface;
};
/** Which tree a resolved `@ref`'s node was read from (ADR 0014, "Mutation admission and read behavior"). */
export type RefObservationSource = 'observation' | 'authorized-frame';
export type ResolvedInteractionTarget =
| {
kind: 'point';
point: Point;
preAction?: SurfaceScopedNodes;
}
| {
kind: 'ref';
point?: Point;
target: Extract<ResolvedTarget, { kind: 'ref' }>;
node?: SnapshotNode;
selectorChain?: string[];
refLabel?: string;
targetHittable?: boolean;
hint?: string;
resolution?: ResolutionDisclosure;
recordingTarget?: RecordingTargetOverride;
preAction?: SurfaceScopedNodes;
/**
* ADR 0014, "Mutation admission and read behavior": the source tree of the dispatched node, reported by the
* reconciliation that chose it. `'observation'` means the session's newest capture, so the
* node's rect is a position that capture measured; `'authorized-frame'` means the older tree
* the ref was authorized against, whose rect says nothing about where the target sits now.
* Reported rather than re-derived downstream from node or rect identity, so no caller can
* disagree with the rule after the rule has decided. Absent on a node a caller carried in
* (#1654), which no reconciliation chose.
*/
adoptedFrom?: RefObservationSource;
}
| {
kind: 'selector';
point: Point;
target: Extract<ResolvedTarget, { kind: 'selector' }>;
node: SnapshotNode;
selectorChain: string[];
refLabel?: string;
targetHittable?: boolean;
hint?: string;
resolution?: ResolutionDisclosure;
recordingTarget?: RecordingTargetOverride;
preAction?: SurfaceScopedNodes;
};
/**
* A post-action capture that describes a DIFFERENT surface than the pre-action baseline (#2438): an
* in-place iOS system surface (a web sign-in or Apple Pay sheet, hosted out of the app's process)
* was presented over the app, or left it. `from`/`to` name the two surfaces — a host bundle id, or
* `APP_SURFACE` (`@agent-device/contracts/ios-system-surface`) for ordinary app content.
*
* Its presence IS the refusal of a same-surface claim: the two captures are not one presentation,
* so `--verify` reports `changedFromBefore` from this transition instead of from a digest
* comparison across it, and `--settle` attaches no settled diff (and therefore no refs) across it.
*/
export type PostActionSurfaceChange = {
from: string;
to: string;
/** The one agent-facing sentence for this transition (`@agent-device/contracts/ios-system-surface`). */
disclosure: string;
};
/**
* Opt-in (`--verify`) cheap post-condition evidence for mutating interaction
* commands (#1047). `digest`/`nodeCount`/`interactiveNodeCount` describe a single
* interactive-only capture taken right after the action; `changedFromBefore`
* compares that digest against the pre-action capture the resolution path already
* held, so no extra device round trip is spent beyond the one verify capture.
* `changedFromBefore: false` is evidence, not failure — the command still
* succeeded.
*
* The field is ABSENT when no pre-action baseline was captured, so "no comparison
* was made" stays distinguishable from "nothing changed": a missing baseline must
* not read as an unchanged screen, because a coordinate tap that landed on
* nothing would otherwise report the same verdict as one that changed nothing.
*
* When `surfaceChange` is present the two captures describe different surfaces, so the digest
* comparison is not made at all: `changedFromBefore` then reports that transition, which replaced
* the whole observed surface.
*/
export type InteractionEvidence = {
foregroundApp?: string;
nodeCount: number;
interactiveNodeCount: number;
digest: string;
changedFromBefore?: boolean;
surfaceChange?: PostActionSurfaceChange;
};
export type SettleDiffLine = {
kind: 'added' | 'removed';
text: string;
/**
* Plain ref body (`e12`) for ADDED lines: minted from the settled tree that
* became the stored session snapshot, so it is immediately actionable and
* lets the MCP layer pin it at `refsGeneration`. Removed lines never carry
* one — their refs name nodes of the replaced tree.
*/
ref?: string;
};
/**
* One still-present, actionable element on the settled tree, surfaced by the
* unchanged-interactive tail (see `SettleObservation.tail`).
*/
export type SettleTailEntry = {
ref: string;
role: string;
label?: string;
};
/**
* Opt-in (`--settle`, #1101) post-action settled observation for mutating
* interaction commands. After the action, the daemon re-captures the
* interactive tree with `wait stable`'s quiet-window semantics and returns the
* DIFF against the pre-action tree in the same response, collapsing the
* dominant interact → observe round-trip pair into one.
*
* Best-effort by contract: settling never fails the action. `settled: false`
* means the quiet window was never reached inside the budget (carousel/ticker/
* animation) or a capture stalled; no diff/refs are issued because the
* observation is advisory. `hint` tells callers how to observe explicitly.
*
* Token budget: `diff.lines` carries only added/removed display lines (the
* unchanged bulk rides as `diff.summary.unchanged`), bounded by the daemon; a
* full tree per interaction would invert the snapshot token-budget principle.
*/
/** Tuning for the settle wait; defaults live with the loop (stable-capture.ts). */
export type SettleParams = {
quietMs?: number;
timeoutMs?: number;
};
/**
* The find read-action result, consumed verbatim on both sides of the
* daemon/engine boundary (R2: the shape lives below both zones — a daemon-side
* structural twin drifted into an identical clone before it moved here).
*/
export type FindReadResult =
| { kind: 'found'; found: true; waitedMs?: number }
| { kind: 'text'; ref?: string; text: string; node: SnapshotNode }
| { kind: 'attrs'; ref?: string; node: SnapshotNode }
/** #1625: the read-only inspection surface — every match, never a tap. */
| { kind: 'list'; matches: Array<{ ref?: string; node: SnapshotNode }> };
export type SettleObservation = {
settled: boolean;
waitedMs: number;
captures: number;
quietMs: number;
timeoutMs: number;
/**
* The session's snapshot generation after the settled tree became the stored
* snapshot (#1076 versioned refs). Attached by the daemon response layer
* when `diff` is present: added lines carry refs minted from that tree, so
* the response is ref-issuing — the MCP layer merges per-ref pins from it
* exactly like snapshot/find responses.
*/
refsGeneration?: number;
/**
* Digest response view only: capped added-line refs preserved without the
* verbose diff line text, so MCP can still pin refs when `diff.lines` is
* intentionally omitted.
*/
refs?: Array<{ ref: string }>;
/**
* Present when the settled capture describes a different surface than the pre-action baseline
* (#2438). The settled tree then replaced the whole surface rather than changing within one, so
* `diff` is omitted: its lines (and their refs) would present a surface replacement as an
* in-surface change. `hint` says what to do instead.
*/
surfaceChange?: PostActionSurfaceChange;
/**
* Present only for `settled: true` observations that stored the settled tree, and never across a
* `surfaceChange` — a diff describes change WITHIN one surface.
*/
diff?: {
summary: { additions: number; removals: number; unchanged: number };
lines: SettleDiffLine[];
/** Present (true) when lines were capped to the response bound. */
truncated?: boolean;
};
/**
* Unchanged interactive refs tail: benchmarks (July 2026) showed 27% of
* `--settle` actions were followed by a fallback `snapshot -i` because a
* change-only diff omits refs for elements that did not change — after a
* modal dismiss the diff shows only removals, and the next button to press
* (already on screen, untouched) is absent from the response. `tail` lists
* the settled tree's remaining uncovered interactive elements (excluding
* structural application/window chrome and the keyboard window's chrome)
* so the response stays actionable without that extra round trip. Attached
* ONLY when `diff` carries zero added-line refs naming a NEW target (the
* modal-dismiss/toast-only/fill signature) — a diff whose added refs hand
* the next target already pays its way, so the tail would be pure byte
* cost. Keyboard-chrome refs and self-echo refs (added lines whose node
* contains the action point: the acted-on element re-describing itself,
* e.g. a filled field re-labeled with its new value) do not count as new
* targets. Refs already present on `diff`'s added lines are excluded.
* Capped; `tailTruncated` marks when candidates exceeded the cap.
*/
tail?: SettleTailEntry[];
tailTruncated?: true;
hint?: string;
};
/**
* Public daemon response data shared by press/click/fill/longpress.
* `buildInteractionResponseData` emits this shape (ADR 0011 Layer 2):
* `targetKind` discriminates the resolved target, identity fields are FLAT
* (`ref`, `selector`, `x`, `y`), and per-command extras ride alongside.
*/
type TouchResponseDataBase = {
message?: string;
warning?: string;
x?: number;
y?: number;
referenceWidth?: number;
referenceHeight?: number;
evidence?: InteractionEvidence;
settle?: SettleObservation;
resolution?: ResolutionDisclosure;
cost?: ResponseCost;
/** Direct iOS Maestro coordinate-fallback signals. */
maestroNonHittableCoordinateFallbackAllowed?: boolean;
maestroNonHittableCoordinateFallbackUsed?: boolean;
maestroFallbackReason?: 'non-hittable-coordinate';
};
type TouchResponsePoint = TouchResponseDataBase & {
targetKind: 'point';
x: number;
y: number;
};
type TouchResponseRef = TouchResponseDataBase & {
targetKind: 'ref';
ref: string;
refLabel?: string;
selectorChain?: string[];
targetHittable?: boolean;
hint?: string;
};
type TouchResponseSelector = TouchResponseDataBase & {
targetKind: 'selector';
selector: string;
selectorChain?: string[];
refLabel?: string;
targetHittable?: boolean;
hint?: string;
};
type TouchPressExtras = {
button?: ClickButton;
count?: number;
intervalMs?: number;
holdMs?: number;
jitterPx?: number;
doubleTap?: boolean;
};
export type PressCommandResponseData =
| (TouchResponsePoint & TouchPressExtras)
| (TouchResponseRef & TouchPressExtras)
| (TouchResponseSelector & TouchPressExtras);
export type ClickCommandResponseData = PressCommandResponseData;
type TouchFillExtras = {
text: string;
delayMs?: number;
} & (
| {
verification?: never;
requested?: never;
before?: never;
after?: never;
target?: never;
}
| FillUnconfirmedVerification
);
export type FillCommandResponseData =
| (TouchResponsePoint & TouchFillExtras)
| (TouchResponseRef & TouchFillExtras)
| (TouchResponseSelector & TouchFillExtras);
type TouchLongPressExtras = {
durationMs?: number;
gesture: 'longpress';
};
export type LongPressCommandResponseData =
| (TouchResponsePoint & TouchLongPressExtras)
| (TouchResponseRef & TouchLongPressExtras)
| (TouchResponseSelector & TouchLongPressExtras);
type TouchHoverExtras = {
gesture: 'hover';
};
export type HoverCommandResponseData =
| (TouchResponsePoint & TouchHoverExtras)
| (TouchResponseRef & TouchHoverExtras)
| (TouchResponseSelector & TouchHoverExtras);
/**
* Internal runtime result for press/click. The daemon response layer turns
* this into `PressCommandResponseData` via `buildInteractionResponseData`.
*/
export type PressCommandResult = ResolvedInteractionTarget & {
backendResult?: Record<string, unknown>;
message?: string;
warning?: string;
evidence?: InteractionEvidence;
settle?: SettleObservation;
};
/**
* Internal runtime result for fill. The daemon response layer turns this into
* `FillCommandResponseData` via `buildInteractionResponseData`.
*/
export type FillCommandResult = ResolvedInteractionTarget & {
text: string;
warning?: string;
backendResult?: Record<string, unknown>;
message?: string;
evidence?: InteractionEvidence;
settle?: SettleObservation;
};
/**
* Internal runtime result for longpress. The daemon response layer turns this
* into `LongPressCommandResponseData` via `buildInteractionResponseData`.
*/
export type LongPressCommandResult = ResolvedInteractionTarget & {
durationMs?: number;
backendResult?: Record<string, unknown>;
message?: string;
warning?: string;
settle?: SettleObservation;
};
/**
* Internal runtime result for hover. The daemon response layer turns this
* into `HoverCommandResponseData` via `buildInteractionResponseData`.
*/
export type HoverCommandResult = ResolvedInteractionTarget & {
backendResult?: Record<string, unknown>;
message?: string;
warning?: string;
settle?: SettleObservation;
};
/**
* Daemon response data for the `find` command. Read-only actions (`exists`,
* `wait`, `get_text`, `get_attrs`) may issue a pinnable ref with
* `refsGeneration`; mutating actions (`click`, `fill`, `focus`, `type`) carry
* `ref` as diagnostic pre-action identity and intentionally omit `refsGeneration`
* (ADR 0014). The shape is intentionally a flat, optional-field record because
* the action positional changes which fields are present.
*/
export type FindCommandResponseData = {
ref?: string;
refsGeneration?: number;
found?: true;
waitedMs?: number;
text?: string;
node?: SnapshotNode;
/** Every match of the read-only `list` action (#1625), each ref pinnable at `refsGeneration`. A
* match on content the capture's producer issued no ref for carries none: readable, not pinnable. */
matches?: Array<{ ref?: string; node: SnapshotNode }>;
locator?: string;
query?: string;
x?: number;
y?: number;
message?: string;
settle?: SettleObservation;
cost?: ResponseCost;
};
/**
* Repeated-activation options shared by tap-like interactions: how many
* times, how fast, how long each contact holds, and whether the pair is a
* double-tap. Declared here (below both its consumers) so the backend
* surface and the command-input parsers agree on one shape without either
* importing the other.
*/
export type RepeatedInput = {
count?: number;
intervalMs?: number;
holdMs?: number;
jitterPx?: number;
doubleTap?: boolean;
};