Repository navigation
Expand file tree
/
Copy pathsnapshot.ts
More file actions
813 lines (750 loc) · 33.4 KB
/
Copy pathsnapshot.ts
File metadata and controls
813 lines (750 loc) · 33.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
/**
* Structured quality verdict computed once by a platform snapshot capture/presentation plan.
* The daemon renders it; it never re-derives degradation from node shapes.
*
* Defined here (the foundational snapshot type module) rather than in
* capture-kit's snapshot-quality-verdict.ts so SnapshotNode can reference it without a cyclic
* import. Ownership splits three ways: this module owns the vocabularies below, capture-kit parses
* an untrusted runner payload into them, and contracts re-hydrates a verdict this repo published.
*/
/**
* Which capture STRATEGY produced a snapshot, within one platform's plan —
* distinct from `SnapshotBackend`, which names the platform channel
* (`xctest`/`android`/…). A platform plan may change strategy mid-sequence, and two strategies do
* not return comparable views of one screen (#1569). Android's helper presentation is included
* here so its quality verdict uses the same typed contract as the iOS strategy chain.
*/
export type SnapshotCaptureBackend = 'tree' | 'queries' | 'private-ax' | 'android-helper';
/** Internal backends that evidence probes may select explicitly. */
export type SnapshotPreferredBackend = 'tree' | 'private-ax';
export type SnapshotQualityTiming = {
acquisitionMs: number;
presentationMs: number;
};
/**
* The verdict states a capture plan may stamp. This tuple is the ONE declaration of that
* vocabulary, and `SnapshotQualityVerdict['state']` is its projection; readers hold exhaustive maps
* over the union instead of importing this module, because the eager-closure gate freezes their
* loading shape (#2872). This tuple and the Apple runner's `SnapshotQualityState.allCases` are each
* pinned as a set to `contracts/fixtures/ios-snapshot-quality-states.json`, so a state one side
* renames, adds, or deletes without the other goes red there instead of arriving as a verdict the
* host cannot name — which reads as verdict-absent and drops the disclosure with it.
*/
export const SNAPSHOT_QUALITY_STATES = ['healthy', 'recovered', 'sparse'] as const;
export type SnapshotQualityState = (typeof SNAPSHOT_QUALITY_STATES)[number];
export type SnapshotQualityNote = {
code: 'regular-geometry-clipped';
clippedNodeCount: number;
droppedNodeCount: number;
};
export type SnapshotQualityVerdict = {
state: SnapshotQualityState;
backend: SnapshotCaptureBackend;
reason?: string;
// 'deferred' = the penalty circuit breaker pre-selected a non-XCTest backend; nothing new
// degraded on THIS capture (no repeated warning, no settle budget reset).
// 'requested-backend' = the REQUEST pre-selected it (e.g. `snapshot --actions`,
// which only the private-AX backend can serve). Nothing degraded at all, so it
// must never surface as a degradation — not even through the one-shot latch
// that exists to catch internally-armed penalties.
reasonCode?:
| 'ax-rejected'
| 'sparse-tree'
| 'budget'
| 'no-nodes'
| 'capture-failed'
| 'presentation-failed'
| 'deferred'
| 'requested-backend';
effectiveDepth?: number;
collapsedLeafIndexes?: number[];
/**
* Coverage of an opt-in custom-action pass (`snapshot --actions`): how many
* merged elements were eligible and how many the bounded pass reached. An
* unread element is indistinguishable from one with no actions, so a partial
* pass has to be disclosed rather than left to look complete.
*/
customActions?: { read: number; candidates: number; truncated: number; blocked: boolean };
/** Response-level phase timing for the backend named by `backend`. */
timing?: SnapshotQualityTiming;
};
export type Rect = {
x: number;
y: number;
width: number;
height: number;
};
export type Point = {
x: number;
y: number;
};
export type IosSnapshotObservation = {
mode: 'observe-only';
activationPerformed: false;
/** XCTest's report, not proof of foreground ownership (#2696). */
appState: 'runningForeground';
appStateSource: 'xcuiapplication-state';
};
export type SnapshotOptions = {
observeOnly?: boolean;
interactiveOnly?: boolean;
depth?: number;
scope?: string;
raw?: boolean;
/**
* Internal (never CLI-exposed): capture with this backend first regardless of
* channel health. Evidence comparisons are only valid same-backend (backends
* are not comparable views of a screen), so a corroboration probe must be
* captured the way its baseline was.
*/
preferredBackend?: SnapshotPreferredBackend;
/**
* Read accessibility custom actions for elements that merge their children
* away. Opt-in because each such element costs its own accessibility round
* trip; see `RawSnapshotNode.actions`.
*/
customActions?: boolean;
};
// The snapshot capture family stated ONCE as option key ↔ command-flag key.
//
// The same pair (`customActions` ↔ `snapshotCustomActions`, and its seven
// siblings) used to be re-typed by hand at every seam that carries a snapshot
// request across the option/flag vocabulary line — the CLI reader, the client
// option projection, the daemon capture inputs, the presentation key. Each copy
// restated a fact already stated here and decided nothing, so a new snapshot
// option cost one edit per seam and a missed seam dropped the option silently.
//
// Declared in the kernel because both vocabularies are declared here
// (SnapshotOptions) and above (`CommandFlags` in contracts), so this is
// the lowest point both sides can read. This generalises the shipped
// `screenshotFlagsFromOptions`/`screenshotOptionsFromFlags` pair for the
// screenshot family.
//
// Every projection takes an EXPLICIT key list: a seam admits the options it
// routes and no more, so adding a pair here never silently widens a seam that
// cannot honour it.
//
// Exported only because the exported types below say `typeof` it — this const
// is not itself public API, so the rationale stays a source comment (stripped
// from the published .d.ts) instead of a bundled JSDoc block.
/** The CLI/daemon flag key for each snapshot capture option, by option name. */
export const SNAPSHOT_OPTION_FLAGS = {
interactiveOnly: 'snapshotInteractiveOnly',
depth: 'snapshotDepth',
scope: 'snapshotScope',
raw: 'snapshotRaw',
observeOnly: 'snapshotObserveOnly',
customActions: 'snapshotCustomActions',
forceFull: 'snapshotForceFull',
includeHiddenContentHints: 'snapshotIncludeHiddenContentHints',
preferredBackend: 'snapshotPreferredBackend',
} as const;
export type SnapshotOptionKey = keyof typeof SNAPSHOT_OPTION_FLAGS;
type SnapshotOptionValues = {
observeOnly: boolean;
interactiveOnly: boolean;
depth: number;
scope: string;
raw: boolean;
customActions: boolean;
forceFull: boolean;
includeHiddenContentHints: boolean;
preferredBackend: SnapshotPreferredBackend;
};
/** The option-vocabulary view of the declared pairs, narrowed to `TKeys`. */
export type SnapshotOptionFields<TKeys extends SnapshotOptionKey = SnapshotOptionKey> = {
[TKey in TKeys]?: SnapshotOptionValues[TKey];
};
/** The flag-vocabulary view of the declared pairs, narrowed to `TKeys`. */
export type SnapshotOptionFlagFields<TKeys extends SnapshotOptionKey = SnapshotOptionKey> = {
[TKey in TKeys as (typeof SNAPSHOT_OPTION_FLAGS)[TKey]]?: SnapshotOptionValues[TKey];
};
/**
* Option keys a `snapshot`/`diff` command request carries end to end. `scope` is
* resolved against the session before capture, so seams that resolve it spread
* this projection and then override that one key.
*/
export const SNAPSHOT_COMMAND_OPTION_KEYS = [
'observeOnly',
'interactiveOnly',
'depth',
'scope',
'raw',
'customActions',
'forceFull',
] as const;
/**
* The snapshot capture options a `snapshot`/`diff` request is stated in, in
* every vocabulary that names them: the public SDK type, the internal request
* bag and the command runtime options each reference THIS type instead of
* re-listing the same six keys.
*/
export type SnapshotCommandOptionFields = SnapshotOptionFields<
(typeof SNAPSHOT_COMMAND_OPTION_KEYS)[number]
>;
/** Option keys a daemon runtime capture input carries; `forceFull` is a command-level concern. */
export const SNAPSHOT_CAPTURE_OPTION_KEYS = [
'observeOnly',
'interactiveOnly',
'preferredBackend',
'depth',
'scope',
'raw',
'customActions',
'includeHiddenContentHints',
] as const;
/** Option keys that identify a presentation; see {@link buildSnapshotPresentationKey}. */
export const SNAPSHOT_PRESENTATION_OPTION_KEYS = [
'depth',
'interactiveOnly',
'raw',
'scope',
'customActions',
] as const;
/**
* Reads the declared options out of a flags bag. Every requested key is present
* (possibly `undefined`), matching what the hand-written copies produced.
*/
export function snapshotOptionsFromFlags<const TKeys extends readonly SnapshotOptionKey[]>(
flags: SnapshotOptionFlagFields | undefined,
keys: TKeys,
): SnapshotOptionFields<TKeys[number]> {
return Object.fromEntries(
keys.map((key) => [key, flags?.[SNAPSHOT_OPTION_FLAGS[key]]]),
) as SnapshotOptionFields<TKeys[number]>;
}
/** Writes the declared options back into flag vocabulary, dropping absent values. */
export function snapshotFlagsFromOptions<const TKeys extends readonly SnapshotOptionKey[]>(
options: SnapshotOptionFields | undefined,
keys: TKeys,
): SnapshotOptionFlagFields<TKeys[number]> {
return Object.fromEntries(
keys.flatMap((key) => {
const value = options?.[key];
return value === undefined ? [] : [[SNAPSHOT_OPTION_FLAGS[key], value]];
}),
) as SnapshotOptionFlagFields<TKeys[number]>;
}
export type SnapshotPresentationFlagInput = SnapshotOptionFlagFields<
(typeof SNAPSHOT_PRESENTATION_OPTION_KEYS)[number]
>;
/** The options that decide a capture's node set, and so the refs minted on it. */
export type SnapshotPresentation = SnapshotOptionFields<
(typeof SNAPSHOT_PRESENTATION_OPTION_KEYS)[number]
>;
/**
* The bounded, non-actionable description of the element that occludes a covered node. It carries
* only what the occlusion scan observed — presented role, an optional clipped accessible name, and
* the cover's rect — and deliberately no ref: the cover may be a container the snapshot never
* issues refs for, and naming it is diagnosis, not an invitation to target it.
*/
export type SnapshotCoveringNode = {
/** The cover's presented role, from the same table the text presenter reads (`formatRole`). */
kind: string;
/** The cover's accessible name, trimmed and clipped to {@link SNAPSHOT_COVERING_LABEL_MAX_LENGTH}. */
label?: string;
/** The cover's positive rect as the scan measured it. */
rect?: Rect;
};
/** Upper bound for a covering-node label carried in diagnostics; longer names are clipped. */
export const SNAPSHOT_COVERING_LABEL_MAX_LENGTH = 80;
/** Evidence from an exact English React Native iOS value signature, not proof of its renderer. */
export type IosValueSemantics = {
source: 'react-native-ios-value-heuristic';
encoding: 'en';
role?: string;
checked?: boolean | 'mixed';
expanded?: boolean;
busy?: true;
};
export type RawSnapshotNode = {
index: number;
type?: string;
role?: string;
subrole?: string;
label?: string;
value?: string;
/** Derived RN evidence; value and native type/role remain verbatim. Explicit facts take precedence. */
iosValueSemantics?: IosValueSemantics;
/**
* Android content description when it is not already the `label`. An Android node is
* labelled by its text and falls back to the content description only when it has none,
* so an accessibility label the app set beside visible text (a labelled text view, a
* filled or hinted field) is carried here for consumers that want the accessible name.
*/
contentDescription?: string;
/**
* Android view text when it is not already the `label`: the empty string on a view that has
* a content description and no text, where the description is the label. Absent otherwise,
* so `text ?? label` is the text the view shows.
*/
text?: string;
identifier?: string;
rect?: Rect;
enabled?: boolean;
selected?: boolean;
/** Checked state of a checkable control (switch, checkbox, radio); absent means not checkable or unavailable. */
checked?: boolean;
focused?: boolean;
/** Accessibility heading flag an app set on the node; absent means not a heading or unavailable. */
heading?: boolean;
/** Localized role description an app set beside the native class, verbatim (`Tab`, `Tab List`, `Link`). */
roleDescription?: string;
/** Native accessibility facts; absent means unavailable, not false. */
editable?: boolean;
password?: boolean;
hintShowing?: boolean;
/**
* Placeholder text of a text field (the Android hint), whether or not the field is showing it.
* Absent when the field has none or the producer did not read it.
*/
placeholder?: string;
/** Accessibility selection offsets, never a character count or proof of value equality. */
selectionStart?: number;
selectionEnd?: number;
visibleToUser?: boolean;
/** UIKit `isUserInteractionEnabled`; absent means the producer did not read it, not false. */
userInteractionEnabled?: boolean;
hittable?: boolean;
depth?: number;
parentIndex?: number;
pid?: number;
bundleId?: string;
/**
* Android status/navigation bar subtree membership, retained when ancestors are omitted.
* Shade headers, dialogs, and other SystemUI overlays are not bars. Absent means the
* producer did not establish bar membership.
*/
systemBar?: 'status' | 'navigation';
appName?: string;
windowTitle?: string;
surface?: string;
/** Keyboard tap-surface membership, including hosted app controls; independent of settle ownership.
* Omission means unevaluated. Survives ancestor removal. */
keyboardOwned?: boolean;
hiddenContentAbove?: boolean;
hiddenContentBelow?: boolean;
interactionBlocked?: 'covered';
/**
* Bounded description of the element the geometric occlusion scan found drawing on top of this
* node, published together with `interactionBlocked: 'covered'` so a refusal can name the culprit
* instead of only the fact of coverage. Absent when the covered decision came from a
* backend-policy annotation that reports a boolean, and when the cover the scan found is an
* anonymous container no reader could recognize; both refusals keep the generic recovery rather
* than guessing a culprit. This is a description, not an authorization: it carries no ref and is
* never an actionable target.
*/
coveredBy?: SnapshotCoveringNode;
presentationHints?: string[];
/**
* Backend-minted ref for this node, when the capture backend already assigns a
* stable, actionable ref (e.g. the web/agent-browser backend resolves actions
* against its own `@eN` refs). `attachRefs` preserves this instead of re-minting
* a dense positional ref, so the ref an agent sees in the snapshot is the same
* ref the backend can resolve on the next action. Absent for backends that do
* not mint refs — those fall back to dense `e${index}` numbering.
*/
ref?: string;
/**
* Accessibility custom actions the element exposes (iOS
* `UIAccessibilityCustomAction`, React Native `accessibilityActions`). Merged
* cards publish their real affordances here instead of as child elements, so
* this is often the only evidence that a collapsed node has any. Populated by
* opt-in captures only — see `snapshot --actions`.
*/
actions?: string[];
};
export type HiddenContentHint = {
hiddenContentAbove?: true;
hiddenContentBelow?: true;
};
/**
* Keyboard evidence in the node coordinate space: `visible` measured; `derived` from the shared
* docked-tree policy before projection; `visibility-only` boolean; `tree-rejected` rejected geometry
* against a usable viewport with optional visibility; `unmeasurable` unknown; `absent` explicit absence.
* `treeBound`: read from the capture's own tree (iOS XCTest, Android, docked-tree), not a probe. A full
* pre-projection iOS root scan proves absence; projected or partial nodes and omitted facts never do.
*/
export type SnapshotKeyboardBandFact =
| { kind: 'visible'; frame: Rect; treeBound?: true }
| { kind: 'derived'; derivation: 'docked-tree'; frame: Rect; treeBound?: true }
| { kind: 'visibility-only'; visible: boolean }
| { kind: 'tree-rejected'; derivation: 'docked-tree'; visible?: boolean }
| { kind: 'absent'; treeBound?: true }
| { kind: 'unmeasurable'; reason: string };
/**
* Whether this capture read a modal presentation over the app's own content (#3565).
*
* The capture producer owns this: an interactive capture, a full capture, a depth cut, and a scoped
* capture of ONE screen all answer the same value, because the fact is decided on the acquired tree
* before any of those projections runs. Settle's dismissal confirmation (#1831) arms on this value
* instead of reconstructing a presentation from the shape of a tree it happens to hold.
*
* `'absent'` is a positive reading, not a default: the producer walked every window it captured and
* every window showed only its own presentation container. Anything the producer could not read —
* a tree from a producer that reports no UIKit class facts, an unexplained extra container, a
* truncated capture — answers `'unknown'`, never `'absent'`.
*/
export type PresentedSurface = 'presented' | 'absent' | 'unknown';
/**
* The box a capture's node rects are measured in, published beside them so a reader never has to
* infer it from the tree (#3182). Two dimensions and no origin: the field answers "how big is the
* surface these numbers describe", which is the only question a consumer that places points on the
* tree cannot answer from the tree itself when the tree is empty or sparse.
*
* It is the box of the surface the producer read, which is not always the physical panel. iOS
* reports the app window in the app's orientation space (ADR 0004), which is smaller than the panel
* under iPad Split View and is never the foldable panel `fold` reports (ADR 0025); Android and the
* Apple TV runner report the screen the bounds were measured on. A producer that has no box of its
* own to answer with leaves the field off: the macOS desktop, whose rects are absolute in window
* space and answer to no single frame, and the web and Linux backends, which read a tree without
* reading a screen. A consumer that needs a *gesture* band inside those bounds still reads `keyboard`
* and the app window, which is #1821's remaining scope.
*
* Absent means the producer measured no box. Absence is never `0`: `snapshotViewportSizeFrom` from
* `@agent-device/kernel/rect` is the sole construction path and refuses a box
* `isPositiveFiniteRect` refuses. The brand makes that invariant part of the type: a plain
* `{ width, height }` literal — including one with a zero in it — is not assignable here, so only
* modules that import the brand token from `kernel/rect` can build one, and they build it through
* the guard.
*/
export type SnapshotViewportSize = {
width: number;
height: number;
} & SnapshotViewportSizeBrand;
/** @internal Exported only so `kernel/rect` can mint values of {@link SnapshotViewportSize}. */
export declare const SNAPSHOT_VIEWPORT_SIZE_BRAND: unique symbol;
export type SnapshotViewportSizeBrand = {
readonly [SNAPSHOT_VIEWPORT_SIZE_BRAND]: 'validated';
};
export type SnapshotNode = RawSnapshotNode & {
/**
* The ref an agent addresses this node by, or absent when nothing can address it.
*
* A backend that mints its own refs (web/agent-browser) owns that namespace and can only act on
* the refs it minted, so {@link attachRefs} mints nothing into it — a node the backend left
* unaddressed stays ref-less in the tree, which is how on-screen-but-not-actionable content
* (static text, a status region) reaches a reader without ever being offered as a tap target.
* Backends that mint no refs (Apple, Android, Linux) keep dense `e${index}` numbering, so every
* node of their trees carries a ref, as before.
*
* Every consumer that offers a node as an address must read this as an absence rather than print
* one: a ref-less node is selectable by text and by `find`/`wait`/`get`, and refused as a target.
*
* A ref-less node also carries no `rect`. Producers that mint refs measure geometry for the refs
* they issued (see `RawSnapshotNode.ref`), and producers that mint none get dense numbering, so
* every node a geometry-requiring path can reach — promotion, the touch point, the covered and
* keyboard refusals — has a ref. That is why those paths print `node.ref` without a guard.
*/
ref?: string;
/**
* Normalized role, from the same table the text presenter reads (`formatRole` below). `type`
* stays the raw platform class; `kind` is the presenter's own answer, published so a `--json`
* consumer gets the presented role without re-deriving it from `type` (#2656). `attachRefs` is
* the one production construction path and always sets it; optional here only so the many
* hand-built `SnapshotNode` fixtures across the daemon/selector test suites that predate #2656 and
* never route through `attachRefs` do not all need updating for a field their assertions never
* read. An absent `ref` is a real answer about the node; an absent `kind` is a fixture shortcut.
*/
kind?: string;
/**
* Output-only marker set by client-serialization dedup (see
* ../snapshot/snapshot-label-dedup.ts) when `label`/`identifier` was omitted
* because it string-equals the nearest ancestor's value in the parent chain.
* Never set on the in-daemon session tree used by selectors/wait/replay.
*/
inheritsLabel?: true;
inheritsIdentifier?: true;
};
export type SnapshotAddressableNode = SnapshotNode & { ref: string };
/**
* Platform role vocabulary → the text presenter's normalized label. Owned here (rather than
* capture-kit) so `attachRefs` can publish the same `kind` on every node without capture-kit
* depending on kernel in the wrong direction; capture-kit's snapshot-lines module re-exports
* `formatRole` for its existing callers.
*/
const ROLE_LABELS: Record<string, string> = {
application: 'application',
navigationbar: 'navigation-bar',
tabbar: 'tab-bar',
button: 'button',
imagebutton: 'button',
link: 'link',
cell: 'cell',
statictext: 'text',
checkedtextview: 'text',
textbox: 'text-field',
textfield: 'text-field',
edittext: 'text-field',
textarea: 'text-view',
switch: 'switch',
slider: 'slider',
image: 'image',
imageview: 'image',
webview: 'webview',
framelayout: 'group',
linearlayout: 'group',
relativelayout: 'group',
constraintlayout: 'group',
viewgroup: 'group',
view: 'group',
listview: 'list',
recyclerview: 'list',
collectionview: 'collection',
searchfield: 'search',
heading: 'heading',
activityindicator: 'activity-indicator',
progressindicator: 'progress-indicator',
segmentedcontrol: 'segmented-control',
group: 'group',
window: 'window',
checkbox: 'checkbox',
radio: 'radio',
menuitem: 'menu-item',
toolbar: 'toolbar',
scrollarea: 'scroll-area',
scrollview: 'scroll-area',
nestedscrollview: 'scroll-area',
table: 'table',
};
function lookupRoleLabel(normalized: string): string | undefined {
return Object.prototype.hasOwnProperty.call(ROLE_LABELS, normalized)
? ROLE_LABELS[normalized]
: undefined;
}
export function formatRole(type: string): string {
const raw = type;
let normalized = type.replaceAll(/XCUIElementType/gi, '').toLowerCase();
const isAndroidClass =
raw.includes('.') &&
(raw.startsWith('android.') || raw.startsWith('androidx.') || raw.startsWith('com.'));
if (normalized.includes('.')) {
normalized = normalized
.replace(/^android\.widget\./, '')
.replace(/^android\.view\./, '')
.replace(/^android\.webkit\./, '')
.replace(/^androidx\./, '')
.replace(/^com\.google\.android\./, '')
.replace(/^com\.android\./, '');
if (isAndroidClass && normalized.includes('.')) {
normalized = normalized.slice(normalized.lastIndexOf('.') + 1);
}
}
if (normalized === 'textview') {
return isAndroidClass ? 'text' : 'text-view';
}
return lookupRoleLabel(normalized) || normalized || 'element';
}
/**
* Reasons the Apple runner can stamp when serving a command required re-activating the session app
* (#2682). Mirrors its `activateTarget(bundleId:reason:)` call sites.
*/
export const IOS_TARGET_ACTIVATION_REASONS = [
'bundle_changed',
'stale_target',
'missing_after_wait',
'interaction_foreground_guard',
] as const;
export type IosTargetActivationReason = (typeof IOS_TARGET_ACTIVATION_REASONS)[number];
/** Whether `value` is a reason the runner can stamp; the only gate consumers apply to the field. */
export function isIosTargetActivationReason(value: unknown): value is IosTargetActivationReason {
return (
typeof value === 'string' &&
(IOS_TARGET_ACTIVATION_REASONS as readonly string[]).includes(value)
);
}
/**
* How XCTest reports an app running (`XCUIApplication.State`), in the SDK's raw order: unknown 0,
* notRunning 1, suspended 2, plain background 3, foreground 4 — the SDK declares suspended on
* non-macOS platforms only. This is the one declaration of those names; the `appState` runner
* command answers the session app's state with them, and `RunnerTests+ApplicationStateRawValueTests`
* ties them to the SDK enum. The `appState` path names states, so nothing here assigns a raw value;
* only the activation decoder's raw table does.
*/
export const APPLE_APPLICATION_STATES = [
'unknown',
'notRunning',
'runningBackgroundSuspended',
'runningBackground',
'runningForeground',
] as const;
export type AppleApplicationState = (typeof APPLE_APPLICATION_STATES)[number];
/**
* States an activation could have been needed for: every Apple state except the foreground one,
* which the runner skips `activate()` in and therefore stamps no fact about. Derived from the full
* list so the two cannot drift, and in the SDK's raw order — a state added to the full list lands
* here and must then be pinned natively before the decoder tie accepts it.
*/
export const IOS_TARGET_ACTIVATION_PRIOR_STATES = Object.freeze(
APPLE_APPLICATION_STATES.filter(
(state): state is Exclude<AppleApplicationState, 'runningForeground'> =>
state !== 'runningForeground',
),
);
export type IosTargetActivationPriorState = (typeof IOS_TARGET_ACTIVATION_PRIOR_STATES)[number];
export function isAppleApplicationState(value: unknown): value is AppleApplicationState {
return (
typeof value === 'string' && (APPLE_APPLICATION_STATES as readonly string[]).includes(value)
);
}
/**
* Foreground repair the Apple runner performed while serving one command (#2682). `priorState` is
* the session app's state BEFORE the runner activated it, so the fact describes what was repaired
* rather than what the repair produced. `otherActiveApplicationPid` is present only when exactly one
* application other than the session app held an active accessibility session at that moment: a
* liveness claim and nothing more, since the private AX client reports no ordering of
* `activeApplications`, resolves pids only, and answers no bundle id for an arbitrary app.
*/
export type IosTargetActivation = Readonly<{
reason: IosTargetActivationReason;
priorState: IosTargetActivationPriorState;
otherActiveApplicationPid?: number;
}>;
export type SnapshotUnchanged = {
ageMs: number;
nodeCount: number;
interactiveOnly?: boolean;
scope?: string;
};
export type SnapshotVisibilityReason =
| 'offscreen-nodes'
| 'scroll-hidden-above'
| 'scroll-hidden-below';
export type SnapshotVisibility = {
partial: boolean;
visibleNodeCount: number;
totalNodeCount: number;
reasons: SnapshotVisibilityReason[];
};
export type ScreenshotOverlayRef = {
ref: string;
label?: string;
rect: Rect;
overlayRect: Rect;
center: Point;
};
/**
* How a capture's nodes get the ref an agent addresses them by.
*
* - `dense` — the capture's producer mints no refs, so this layer numbers the tree `e1…eN` and
* every node becomes addressable by its position.
* - `backend-issued` — the producer resolves actions against refs of its own (`e${n}` minted by
* the backend, in tree order and not dense), so a ref this layer invented would name nothing.
* A node the producer did not issue a ref for stays ref-less: present to a reader, absent from
* the addressable namespace.
*/
export type SnapshotRefMinting = 'dense' | 'backend-issued';
/**
* Assign a display ref to every node, in the mode the capture's producer needs (see
* `SnapshotRefMinting`).
*
* A node that already carries a backend-minted `ref` keeps it (see `RawSnapshotNode.ref`) — the
* web/agent-browser backend resolves actions against its own refs, so re-minting a dense positional
* ref here would make the snapshot show one ref while actions act on a different element.
*
* Under `dense` every node ends up addressable, as it always has. Under `backend-issued` only the
* nodes the producer issued a ref for do: the rest stay ref-less, readable and searchable by their
* text and tree position, and never an address — which is what `hasSnapshotRef` and
* `snapshotRefBodies` report, and what a caller publishing an address must check.
*/
export function attachRefs(
nodes: RawSnapshotNode[],
minting: SnapshotRefMinting = 'dense',
): SnapshotNode[] {
return nodes.map((node, idx) => ({
...node,
...(node.ref === undefined && minting !== 'dense' ? {} : { ref: node.ref ?? `e${idx + 1}` }),
kind: formatRole(node.type ?? 'Element'),
}));
}
/**
* Whether a node can be addressed by ref, narrowing the ref for callers that publish it as an
* address (an overlay label, a settle tail entry, a ref pin). A ref-less node is readable and
* searchable, never addressable.
*/
export function hasSnapshotRef(node: SnapshotNode): node is SnapshotAddressableNode {
return node.ref !== undefined;
}
/**
* The refs a set of ref-carrying records can be re-addressed by; a record with no ref contributes no
* evidence. Typed over the field rather than over `SnapshotNode` so every site holding an optional
* ref (a node, a competitor rect, a published match) answers the same question here instead of
* restating the filter.
*/
export function snapshotRefBodies(refCarriers: readonly { ref?: string }[]): string[] {
return refCarriers.flatMap((carrier) => (carrier.ref === undefined ? [] : [carrier.ref]));
}
/**
* Versioned-ref grammar (#1076): a ref argument may carry an optional
* `~s<generation>` suffix pinning it to the session snapshot generation that
* minted it, e.g. `@e12~s3`. Snapshot output stays plain `e12` refs (the tree is
* the most token-expensive artifact agents consume), except the CLI text of a
* `--scope` snapshot, which prints every ref in this pinned form unless `--raw`
* is set (ADR 0014).
* Ref-issuing responses carry the generation ONCE as the additive
* `refsGeneration` field.
*/
const REF_GENERATION_SUFFIX_RE = /^~s(\d+)$/;
export const REF_GRAMMAR_HINT =
'Refs look like @e12, optionally pinned to the snapshot generation that minted them: @e12~s3 (the ref, then "~s" and the refsGeneration reported by the issuing snapshot/find response).';
export type SplitRef = { base: string; generation?: number };
/**
* Split an optional `~s<generation>` suffix off a ref token (`@e12~s3` or bare
* `e12~s3`). `base` keeps the token's `@` prefix (or lack of one). Returns null
* when a `~` is present but the suffix does not match the grammar — callers
* surface INVALID_ARGS with REF_GRAMMAR_HINT.
*/
export function splitRefGenerationSuffix(input: string): SplitRef | null {
const trimmed = input.trim();
const tildeIndex = trimmed.indexOf('~');
if (tildeIndex === -1) return { base: trimmed };
const match = REF_GENERATION_SUFFIX_RE.exec(trimmed.slice(tildeIndex));
if (!match || tildeIndex === 0) return null;
return { base: trimmed.slice(0, tildeIndex), generation: Number(match[1]) };
}
export function normalizeRef(input: string): string | null {
// Node lookup always uses the plain ref; the generation suffix is stripped
// here so every existing parse site accepts the pinned form (#1076).
const split = splitRefGenerationSuffix(input);
if (!split) return null;
const trimmed = split.base;
if (trimmed.startsWith('@')) {
const ref = trimmed.slice(1);
return ref ? ref : null;
}
if (trimmed.startsWith('e')) return trimmed;
return null;
}
export function findNodeByRef(nodes: SnapshotNode[], ref: string): SnapshotNode | null {
return nodes.find((node) => node.ref === ref) ?? null;
}
export function buildSnapshotPresentationKey(flags: SnapshotOptions | undefined): string {
return JSON.stringify({
interactiveOnly: flags?.interactiveOnly === true,
depth: typeof flags?.depth === 'number' ? flags.depth : null,
scope: flags?.scope?.trim() || null,
raw: flags?.raw === true,
// A capture that asked for custom actions is not the same presentation as
// one that did not: without this, 'snapshot' then 'snapshot --actions' on a
// still screen reports 'unchanged' and never delivers what was asked for.
customActions: flags?.customActions === true,
});
}
/**
* Whether two captures read one presentation, so a diff between them compares like with like
* (#4486): a full tree holds nodes an interactive read omits, and a scoped, depth-limited, raw or
* custom-action read holds a different node set than a plain one. An absent presentation is
* the default full tree, as on SnapshotState.
*/
export function isSameSnapshotPresentation(
a: SnapshotPresentation | undefined,
b: SnapshotPresentation | undefined,
): boolean {
return buildSnapshotPresentationKey(a) === buildSnapshotPresentationKey(b);
}
export function centerOfRect(rect: Rect): Point {
return {
x: Math.round(rect.x + rect.width / 2),
y: Math.round(rect.y + rect.height / 2),
};
}