Skip to content

Commit af2f604

Browse files
committed
feat(snapshot): carry the presenter's platform-neutral role on nodes
Every SnapshotNode now carries `kind`, the text presenter's normalized role (button, text-field, text, switch, ...), computed by `formatRole` for the same value the text line prints in brackets. `formatRole` and its role table move from capture-kit to kernel so `attachRefs` — the one construction path every platform, backend, and projection routes through — can set `kind` on every node without a wrong-direction package dependency; capture-kit re-exports it for its existing callers. `type` stays the raw platform class. `kind` is optional on the shared `SnapshotNode` type rather than required like `ref`: `attachRefs` always sets it for real captures, but the type also backs the in-daemon session tree and hundreds of pre-existing hand-built test fixtures across capture-kit/contracts/selectors/maestro that never route through `attachRefs` and do not assert on the field. Adds a conformance test that feeds an iOS and an Android fixture tree through `attachRefs` and the production text presenter and asserts `kind` equals the role printed on each node's line, plus an `attachRefs` unit test covering the XCUI/Android/undefined-type vocabularies. No wire-compat ledger entry: CaptureSnapshotResult/SnapshotNode field shapes are outside test/wire-compat's tracked surface (that ledger only digests transport framing per ADR 0006), confirmed by an unmodified `test/wire-compat` pass. Fixes #2656
1 parent 1433e11 commit af2f604

36 files changed

Lines changed: 466 additions & 109 deletions

‎packages/capture-kit/src/mobile-snapshot-semantics.test.ts‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import {
77
} from './mobile-snapshot-semantics.ts';
88
import { createSnapshotVisibility } from '@agent-device/contracts/snapshot';
99
import type { Rect, SnapshotNode } from '@agent-device/kernel/snapshot';
10+
import { formatRole } from '@agent-device/kernel/snapshot';
1011

1112
test('rootless mobile presentation performs linear geometry reads', () => {
1213
let rectReads = 0;
@@ -15,6 +16,7 @@ test('rootless mobile presentation performs linear geometry reads', () => {
1516
index,
1617
ref: `@e${index}`,
1718
type: 'android.widget.TextView',
19+
kind: formatRole('android.widget.TextView'),
1820
...(index === 0 ? {} : { parentIndex: 0 }),
1921
};
2022
Object.defineProperty(node, 'rect', {
@@ -39,6 +41,7 @@ test('mobile presentation keeps only visible nodes and adds off-screen summary f
3941
index: 0,
4042
depth: 0,
4143
type: 'Window',
44+
kind: formatRole('Window'),
4245
rect: { x: 0, y: 0, width: 390, height: 844 },
4346
},
4447
{
@@ -47,6 +50,7 @@ test('mobile presentation keeps only visible nodes and adds off-screen summary f
4750
depth: 1,
4851
parentIndex: 0,
4952
type: 'XCUIElementTypeButton',
53+
kind: formatRole('XCUIElementTypeButton'),
5054
label: 'Visible action',
5155
rect: { x: 20, y: 140, width: 160, height: 44 },
5256
hittable: true,
@@ -57,6 +61,7 @@ test('mobile presentation keeps only visible nodes and adds off-screen summary f
5761
depth: 1,
5862
parentIndex: 0,
5963
type: 'XCUIElementTypeButton',
64+
kind: formatRole('XCUIElementTypeButton'),
6065
label: 'Later action',
6166
rect: { x: 20, y: 1100, width: 160, height: 44 },
6267
hittable: true,
@@ -78,6 +83,7 @@ test('mobile presentation assigns hidden content hints to visible scroll contain
7883
index: 0,
7984
depth: 0,
8085
type: 'Window',
86+
kind: formatRole('Window'),
8187
rect: { x: 0, y: 0, width: 390, height: 844 },
8288
},
8389
{
@@ -86,6 +92,7 @@ test('mobile presentation assigns hidden content hints to visible scroll contain
8692
depth: 1,
8793
parentIndex: 0,
8894
type: 'android.widget.ScrollView',
95+
kind: formatRole('android.widget.ScrollView'),
8996
label: 'Messages',
9097
rect: { x: 0, y: 140, width: 390, height: 500 },
9198
},
@@ -95,6 +102,7 @@ test('mobile presentation assigns hidden content hints to visible scroll contain
95102
depth: 2,
96103
parentIndex: 1,
97104
type: 'android.widget.Button',
105+
kind: formatRole('android.widget.Button'),
98106
label: 'Older',
99107
rect: { x: 20, y: 60, width: 350, height: 44 },
100108
hittable: true,
@@ -105,6 +113,7 @@ test('mobile presentation assigns hidden content hints to visible scroll contain
105113
depth: 2,
106114
parentIndex: 1,
107115
type: 'android.widget.Button',
116+
kind: formatRole('android.widget.Button'),
108117
label: 'Visible',
109118
rect: { x: 20, y: 260, width: 350, height: 44 },
110119
hittable: true,
@@ -115,6 +124,7 @@ test('mobile presentation assigns hidden content hints to visible scroll contain
115124
depth: 2,
116125
parentIndex: 1,
117126
type: 'android.widget.Button',
127+
kind: formatRole('android.widget.Button'),
118128
label: 'Newer',
119129
rect: { x: 20, y: 680, width: 350, height: 44 },
120130
hittable: true,
@@ -135,6 +145,7 @@ test('mobile presentation keeps fixed bottom controls after long visible scroll
135145
index: 0,
136146
depth: 0,
137147
type: 'Application',
148+
kind: formatRole('Application'),
138149
rect: { x: 0, y: 0, width: 402, height: 874 },
139150
},
140151
{
@@ -143,6 +154,7 @@ test('mobile presentation keeps fixed bottom controls after long visible scroll
143154
depth: 1,
144155
parentIndex: 0,
145156
type: 'Window',
157+
kind: formatRole('Window'),
146158
rect: { x: 0, y: 0, width: 402, height: 874 },
147159
},
148160
{
@@ -151,6 +163,7 @@ test('mobile presentation keeps fixed bottom controls after long visible scroll
151163
depth: 2,
152164
parentIndex: 1,
153165
type: 'ScrollView',
166+
kind: formatRole('ScrollView'),
154167
label: 'Contacts',
155168
rect: { x: 0, y: 116, width: 402, height: 675 },
156169
},
@@ -162,6 +175,7 @@ test('mobile presentation keeps fixed bottom controls after long visible scroll
162175
depth: 3,
163176
parentIndex: 2,
164177
type: 'StaticText',
178+
kind: formatRole('StaticText'),
165179
label: `Contact ${offset}`,
166180
rect: { x: 52, y: 132 + offset * 64, width: 160, height: 18 },
167181
};
@@ -172,6 +186,7 @@ test('mobile presentation keeps fixed bottom controls after long visible scroll
172186
depth: 2,
173187
parentIndex: 1,
174188
type: 'Button',
189+
kind: formatRole('Button'),
175190
label: 'Article, unselected',
176191
identifier: 'article',
177192
rect: { x: 0, y: 791, width: 101, height: 49 },
@@ -183,6 +198,7 @@ test('mobile presentation keeps fixed bottom controls after long visible scroll
183198
depth: 2,
184199
parentIndex: 1,
185200
type: 'Button',
201+
kind: formatRole('Button'),
186202
label: 'Contacts, selected',
187203
identifier: 'contacts',
188204
rect: { x: 201, y: 791, width: 101, height: 49 },
@@ -195,6 +211,7 @@ test('mobile presentation keeps fixed bottom controls after long visible scroll
195211
depth: 2,
196212
parentIndex: 1,
197213
type: 'Button',
214+
kind: formatRole('Button'),
198215
label: 'Albums, unselected',
199216
identifier: 'albums',
200217
rect: { x: 302, y: 791, width: 100, height: 49 },
@@ -219,6 +236,7 @@ test('mobile presentation handles zero-width viewport gracefully', () => {
219236
index: 0,
220237
depth: 0,
221238
type: 'Window',
239+
kind: formatRole('Window'),
222240
rect: { x: 0, y: 0, width: 0, height: 844 },
223241
},
224242
{
@@ -227,6 +245,7 @@ test('mobile presentation handles zero-width viewport gracefully', () => {
227245
depth: 1,
228246
parentIndex: 0,
229247
type: 'XCUIElementTypeButton',
248+
kind: formatRole('XCUIElementTypeButton'),
230249
label: 'Action',
231250
rect: { x: 20, y: 140, width: 160, height: 44 },
232251
hittable: true,
@@ -245,6 +264,7 @@ test('mobile presentation handles zero-height viewport gracefully', () => {
245264
index: 0,
246265
depth: 0,
247266
type: 'Window',
267+
kind: formatRole('Window'),
248268
rect: { x: 0, y: 0, width: 390, height: 0 },
249269
},
250270
{
@@ -253,6 +273,7 @@ test('mobile presentation handles zero-height viewport gracefully', () => {
253273
depth: 1,
254274
parentIndex: 0,
255275
type: 'XCUIElementTypeButton',
276+
kind: formatRole('XCUIElementTypeButton'),
256277
label: 'Action',
257278
rect: { x: 20, y: 140, width: 160, height: 44 },
258279
hittable: true,
@@ -270,6 +291,7 @@ test('mobile presentation handles nodes with negative coordinates', () => {
270291
index: 0,
271292
depth: 0,
272293
type: 'Window',
294+
kind: formatRole('Window'),
273295
rect: { x: 0, y: 0, width: 390, height: 844 },
274296
},
275297
{
@@ -278,6 +300,7 @@ test('mobile presentation handles nodes with negative coordinates', () => {
278300
depth: 1,
279301
parentIndex: 0,
280302
type: 'XCUIElementTypeButton',
303+
kind: formatRole('XCUIElementTypeButton'),
281304
label: 'Off left',
282305
rect: { x: -200, y: 200, width: 160, height: 44 },
283306
hittable: true,
@@ -288,6 +311,7 @@ test('mobile presentation handles nodes with negative coordinates', () => {
288311
depth: 1,
289312
parentIndex: 0,
290313
type: 'XCUIElementTypeButton',
314+
kind: formatRole('XCUIElementTypeButton'),
291315
label: 'Visible',
292316
rect: { x: 20, y: 200, width: 160, height: 44 },
293317
hittable: true,
@@ -306,6 +330,7 @@ test('visibility is true for node at exact viewport boundary', () => {
306330
index: 0,
307331
depth: 0,
308332
type: 'Window',
333+
kind: formatRole('Window'),
309334
rect: { x: 0, y: 0, width: 390, height: 844 },
310335
},
311336
{
@@ -314,6 +339,7 @@ test('visibility is true for node at exact viewport boundary', () => {
314339
depth: 1,
315340
parentIndex: 0,
316341
type: 'XCUIElementTypeButton',
342+
kind: formatRole('XCUIElementTypeButton'),
317343
label: 'Edge',
318344
// Bottom edge of node touches top of viewport — 1 px overlap
319345
rect: { x: 20, y: -43, width: 160, height: 44 },
@@ -332,6 +358,7 @@ test('visibility is false for node just outside viewport boundary', () => {
332358
index: 0,
333359
depth: 0,
334360
type: 'Window',
361+
kind: formatRole('Window'),
335362
rect: { x: 0, y: 0, width: 390, height: 844 },
336363
},
337364
{
@@ -340,6 +367,7 @@ test('visibility is false for node just outside viewport boundary', () => {
340367
depth: 1,
341368
parentIndex: 0,
342369
type: 'XCUIElementTypeButton',
370+
kind: formatRole('XCUIElementTypeButton'),
343371
label: 'Outside',
344372
// Entirely above viewport — y + height = -1, which is above the viewport top
345373
rect: { x: 20, y: -45, width: 160, height: 44 },
@@ -356,6 +384,7 @@ function windowRoot(): SnapshotNode {
356384
index: 0,
357385
depth: 0,
358386
type: 'Window',
387+
kind: formatRole('Window'),
359388
rect: { x: 0, y: 0, width: 400, height: 800 },
360389
};
361390
}
@@ -369,6 +398,7 @@ test('classifyOffscreenScrollDirection names the reveal direction for a fully sc
369398
depth: 1,
370399
parentIndex: 0,
371400
type: 'Button',
401+
kind: formatRole('Button'),
372402
label: 'Far below',
373403
rect: { x: 20, y: 1200, width: 120, height: 44 },
374404
hittable: true,
@@ -387,6 +417,7 @@ test('classifyOffscreenScrollDirection names the reveal direction for a fully sc
387417
depth: 1,
388418
parentIndex: 0,
389419
type: 'Button',
420+
kind: formatRole('Button'),
390421
label: 'Far above',
391422
rect: { x: 20, y: -100, width: 120, height: 44 },
392423
hittable: true,
@@ -404,6 +435,7 @@ test('classifyOffscreenScrollDirection returns null for an on-screen node', () =
404435
depth: 1,
405436
parentIndex: 0,
406437
type: 'Button',
438+
kind: formatRole('Button'),
407439
label: 'Visible',
408440
rect: { x: 20, y: 40, width: 120, height: 44 },
409441
hittable: true,
@@ -424,6 +456,7 @@ test('classifyOffscreenScrollDirection resolves a partial clip whose center is p
424456
depth: 1,
425457
parentIndex: 0,
426458
type: 'Button',
459+
kind: formatRole('Button'),
427460
label: 'Straddling the bottom edge',
428461
rect: { x: 20, y: 790, width: 120, height: 44 },
429462
hittable: true,
@@ -449,6 +482,7 @@ test('classifyOffscreenScrollDirection resolves a child inside an off-screen scr
449482
depth: 1,
450483
parentIndex: 0,
451484
type: 'ScrollView',
485+
kind: formatRole('ScrollView'),
452486
label: 'Drawer',
453487
rect: { x: -400, y: 0, width: 400, height: 800 },
454488
},
@@ -458,6 +492,7 @@ test('classifyOffscreenScrollDirection resolves a child inside an off-screen scr
458492
depth: 2,
459493
parentIndex: 1,
460494
type: 'Button',
495+
kind: formatRole('Button'),
461496
label: 'Drawer action',
462497
rect: { x: -380, y: 300, width: 200, height: 44 },
463498
hittable: true,
@@ -478,6 +513,7 @@ test('mobile presentation infers hidden content from vertical scroll indicator v
478513
index: 0,
479514
depth: 0,
480515
type: 'Window',
516+
kind: formatRole('Window'),
481517
rect: { x: 0, y: 0, width: 390, height: 844 },
482518
},
483519
{
@@ -486,6 +522,7 @@ test('mobile presentation infers hidden content from vertical scroll indicator v
486522
depth: 1,
487523
parentIndex: 0,
488524
type: 'CollectionView',
525+
kind: formatRole('CollectionView'),
489526
label: 'Vertical scroll bar, 2 pages',
490527
rect: { x: 0, y: 80, width: 390, height: 680 },
491528
},
@@ -495,6 +532,7 @@ test('mobile presentation infers hidden content from vertical scroll indicator v
495532
depth: 2,
496533
parentIndex: 1,
497534
type: 'Other',
535+
kind: formatRole('Other'),
498536
label: 'Vertical scroll bar, 2 pages',
499537
value: '0%',
500538
rect: { x: 360, y: 96, width: 20, height: 640 },
@@ -514,6 +552,7 @@ test('mobile presentation infers hidden content from vertical scroll indicator v
514552
index: 0,
515553
depth: 0,
516554
type: 'Window',
555+
kind: formatRole('Window'),
517556
rect: { x: 0, y: 0, width: 390, height: 844 },
518557
},
519558
{
@@ -522,6 +561,7 @@ test('mobile presentation infers hidden content from vertical scroll indicator v
522561
depth: 1,
523562
parentIndex: 0,
524563
type: 'CollectionView',
564+
kind: formatRole('CollectionView'),
525565
label: 'Vertical scroll bar, 2 pages',
526566
rect: { x: 0, y: 80, width: 390, height: 680 },
527567
},
@@ -531,6 +571,7 @@ test('mobile presentation infers hidden content from vertical scroll indicator v
531571
depth: 2,
532572
parentIndex: 1,
533573
type: 'Other',
574+
kind: formatRole('Other'),
534575
label: 'Vertical scroll bar, 2 pages',
535576
value: '48%',
536577
rect: { x: 360, y: 96, width: 20, height: 640 },
@@ -550,6 +591,7 @@ test('mobile presentation does not let contradictory scroll indicator add hidden
550591
index: 0,
551592
depth: 0,
552593
type: 'Window',
594+
kind: formatRole('Window'),
553595
rect: { x: 0, y: 0, width: 402, height: 874 },
554596
},
555597
{
@@ -558,6 +600,7 @@ test('mobile presentation does not let contradictory scroll indicator add hidden
558600
depth: 1,
559601
parentIndex: 0,
560602
type: 'ScrollView',
603+
kind: formatRole('ScrollView'),
561604
label: 'List of chat messages',
562605
rect: { x: 0, y: 135, width: 402, height: 617 },
563606
},
@@ -567,6 +610,7 @@ test('mobile presentation does not let contradictory scroll indicator add hidden
567610
depth: 2,
568611
parentIndex: 1,
569612
type: 'Button',
613+
kind: formatRole('Button'),
570614
label: 'Older message',
571615
rect: { x: 0, y: -120, width: 402, height: 56 },
572616
},
@@ -576,6 +620,7 @@ test('mobile presentation does not let contradictory scroll indicator add hidden
576620
depth: 2,
577621
parentIndex: 1,
578622
type: 'Button',
623+
kind: formatRole('Button'),
579624
label: 'Latest visible message',
580625
rect: { x: 0, y: 696, width: 402, height: 56 },
581626
},
@@ -585,6 +630,7 @@ test('mobile presentation does not let contradictory scroll indicator add hidden
585630
depth: 2,
586631
parentIndex: 1,
587632
type: 'Other',
633+
kind: formatRole('Other'),
588634
label: 'Vertical scroll bar, 3 pages',
589635
value: '0%',
590636
rect: { x: 369, y: 135, width: 30, height: 617 },

0 commit comments

Comments
 (0)