Skip to content

Commit 744c8a3

Browse files
committed
refactor(settle): one contracts iOS keyboard-role table; assistant hosts are markers
The assistant view and its page view are keyboard hosts the APP can also fill (UITextField.inputAssistantItem bar buttons ride them), so they join the inputView placeholder as markers: chrome themselves, claiming no subtree. Subtree claims stay with the system-only hosts — the key plane (UIKeyboardLayoutStar) and the prediction bar — where no app content can ever sit. Surface roles are chrome per node. With no subtree walk able to reach app content, the accessory re-join guard and the hosted editable-text carve-out are dead weight and are removed. The role tables move to @agent-device/contracts/ios-keyboard-ownership, the iOS twin of android-input-ownership, shared by the settle classifier and the tap-occlusion surface set; the prediction cell stays out of the surface table so QuickType geometry never enters the occlusion band. Accepts two inert noise survivors in the flattened capture (floating toolbar, QuickType scroll wrapper) — noise fails loudly, a subtree claim through an app-fillable host would not. Also reverts the broad-replacement right parameter to non-optional: the memo is gone and the only caller always passes a defined signal.
1 parent 41c8a82 commit 744c8a3

8 files changed

Lines changed: 193 additions & 146 deletions

File tree

‎packages/capture-kit/src/snapshot-chrome.test.ts‎

Lines changed: 88 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -234,11 +234,15 @@ test('the real interactive keyboard capture collapses to one [Keyboard] diff lin
234234
}
235235

236236
// The stripped set collapses the ~50 keyboard-surface nodes; what survives is
237-
// app content plus the lone keyboard plane line.
237+
// app content plus the keyboard plane line and two accepted residues: the
238+
// floating home-indicator Toolbar (31) and the inert QuickType UIScrollView
239+
// (35) under the assistant markers, which declare no keyboard-owned role and
240+
// are not under a system-only subtree host. Noise lines fail loudly; a
241+
// subtree claim through the assistant host could swallow app controls.
238242
assert.equal(survivors.filter((node) => node.type === 'Key').length, 0);
239243
assert.deepEqual(
240244
survivors.filter((node) => node.index >= 31).map((node) => node.type),
241-
['Toolbar', 'Keyboard'],
245+
['Toolbar', 'ScrollView', 'Keyboard'],
242246
);
243247
// App content never rides along: the search field the fill targeted survives.
244248
assert.equal(chromeRefs.has(refFor(nodes, 'Search products')), false);
@@ -289,9 +293,10 @@ test('the inputView host frame is stripped without swallowing app input-view con
289293
);
290294
});
291295

292-
// An app that hosts editable text inside a keyboard-owned subtree (its own
293-
// composer riding the input-assistant host) keeps the field it is typing into.
294-
test('editable text hosted inside a keyboard-owned subtree survives classification', () => {
296+
// An app that hosts editable text under a keyboard MARKER host (its own composer
297+
// riding the input-assistant host) keeps the field it is typing into: the marker
298+
// is chrome itself and owns no subtree.
299+
test('editable text hosted under a keyboard marker host survives classification', () => {
295300
const nodes = attachRefs([
296301
{
297302
index: 0,
@@ -404,3 +409,81 @@ test('app inputAssistantItem buttons under the assistant view keep the accessory
404409
assert.equal(chromeRefs.has(refFor(nodes, 'Assistant Bar')), true);
405410
assert.equal(chromeRefs.has(refFor(nodes, 'q')), true);
406411
});
412+
413+
// The route #3469 targets: the interactive projection has NO Window anchor, so the
414+
// app-accessory exemption is empty and only the marker rule protects app content. An
415+
// app Button under the assistant view survives on its own; a prediction cell beside
416+
// it, chrome by its own declared role, still collapses.
417+
test('an app control under the assistant host survives on the Window-less interactive shape', () => {
418+
const nodes = attachRefs([
419+
{
420+
index: 0,
421+
depth: 0,
422+
type: 'Application',
423+
label: 'Example',
424+
rect: { x: 0, y: 0, width: 402, height: 874 },
425+
},
426+
{
427+
index: 1,
428+
depth: 1,
429+
parentIndex: 0,
430+
type: 'TextField',
431+
label: 'Search',
432+
rect: { x: 18, y: 100, width: 366, height: 44 },
433+
hittable: true,
434+
},
435+
{
436+
index: 2,
437+
depth: 1,
438+
parentIndex: 0,
439+
type: 'Other',
440+
role: 'TUISystemInputAssistantView',
441+
rect: { x: 0, y: 546, width: 402, height: 44 },
442+
},
443+
{
444+
index: 3,
445+
depth: 2,
446+
parentIndex: 2,
447+
type: 'Button',
448+
label: 'Paste',
449+
rect: { x: 8, y: 550, width: 60, height: 36 },
450+
hittable: true,
451+
},
452+
{
453+
index: 4,
454+
depth: 2,
455+
parentIndex: 2,
456+
type: 'Button',
457+
role: 'TUIPredictionViewCell',
458+
label: 'keep',
459+
rect: { x: 200, y: 550, width: 60, height: 36 },
460+
hittable: true,
461+
},
462+
{
463+
index: 5,
464+
depth: 1,
465+
parentIndex: 0,
466+
type: 'Keyboard',
467+
role: 'UIKeyboardAutomatic',
468+
rect: { x: 0, y: 606, width: 402, height: 268 },
469+
},
470+
{
471+
index: 6,
472+
depth: 2,
473+
parentIndex: 5,
474+
type: 'Key',
475+
role: 'UIAccessibilityElementKBKey',
476+
label: 'q',
477+
rect: { x: 0, y: 620, width: 39, height: 54 },
478+
},
479+
]);
480+
const chromeRefs = collectSettleChromeRefs(nodes, undefined);
481+
482+
assert.equal(
483+
chromeRefs.has(refFor(nodes, 'Paste')),
484+
false,
485+
'the assistant host is a marker: it claims no subtree, so an app control under it survives',
486+
);
487+
assert.equal(chromeRefs.has(refFor(nodes, 'keep')), true);
488+
assert.equal(chromeRefs.has(refFor(nodes, 'Search')), false);
489+
});

‎packages/capture-kit/src/snapshot-chrome.ts‎

Lines changed: 34 additions & 131 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,13 @@ import {
44
hasAndroidSystemChromeProvenance,
55
isAndroidSystemChromeWindowResourceId,
66
} from '@agent-device/contracts/android-system-chrome';
7+
import {
8+
hasIosKeyboardRole,
9+
IOS_KEYBOARD_CHROME_MARKER_ROLES,
10+
IOS_KEYBOARD_SETTLE_CHROME_ROLES,
11+
IOS_KEYBOARD_SUBTREE_CHROME_ROLES,
12+
IOS_KEYBOARD_SURFACE_ROLES,
13+
} from '@agent-device/contracts/ios-keyboard-ownership';
714
import type { SnapshotNode } from '@agent-device/kernel/snapshot';
815
import { normalizeType } from '@agent-device/contracts/snapshot';
916

@@ -74,65 +81,30 @@ type SettleChrome = {
7481

7582
const EMPTY_SETTLE_CHROME: SettleChrome = { strippedIndexes: new Set(), refs: new Set() };
7683

77-
/**
78-
* UIKit classes the keyboard owns whose nodes the regular projection can flatten beside app
79-
* content — the key plane and a key when no `[Keyboard]` container survives, the system
80-
* input-assistant (QuickType) bar, and the dock's "Next keyboard" / "Dictate" buttons. Once the
81-
* projection has dropped the keyboard's own window, neither the container-subtree nor the
82-
* window walk can reach them, but each node still carries its producing class in `role`.
83-
*
84-
* Exact class names only, so a near miss fails closed rather than widening what counts as
85-
* chrome (the same discipline `geometry-policy.ts` applies to private UIKit roles). Classes are
86-
* locale-independent, unlike every label these nodes carry, and private to the keyboard
87-
* surfaces — an app cannot host one in its own view tree.
88-
*/
89-
const IOS_KEYBOARD_OWNED_ROLES: ReadonlySet<string> = new Set([
90-
'UIKeyboardLayoutStar',
91-
'UIAccessibilityElementKBKey',
92-
'UIKeyboardDockItemButton',
93-
'TUISystemInputAssistantView',
94-
'TUISystemInputAssistantPageView',
95-
'TUIPredictionView',
96-
'TUIPredictionViewCell',
97-
]);
98-
99-
/**
100-
* The system `inputView` host frame: chrome itself, but it OWNS NO SUBTREE — an app's custom
101-
* `UITextField.inputView` content (a drawn keypad, a composer) renders under this placeholder and
102-
* stays a control the agent must see. Stripping only the marker keeps the keyboard's appearance
103-
* at one visible line without swallowing app content; if the class is ever renamed, the inert
104-
* marker line returns as noise rather than app content vanishing — noise fails loudly.
105-
*/
106-
const IOS_KEYBOARD_CHROME_MARKER_ROLES: ReadonlySet<string> = new Set([
107-
'_UIRemoteKeyboardPlaceholderView',
108-
]);
109-
110-
function hasRole(node: SnapshotNode, roles: ReadonlySet<string>): boolean {
111-
return node.role !== undefined && roles.has(node.role);
112-
}
113-
114-
function isKeyboardOwnedRoleNode(node: SnapshotNode): boolean {
115-
return hasRole(node, IOS_KEYBOARD_OWNED_ROLES);
116-
}
117-
11884
function collectKeyboardChrome(nodes: SnapshotNode[]): SettleChrome {
11985
const byIndex = new Map(nodes.map((node) => [node.index, node]));
12086
const containerIndexes = new Set(
12187
nodes.filter((node) => normalizeType(node.type ?? '') === 'keyboard').map((node) => node.index),
12288
);
123-
const ownedSubtreeRootIndexes = new Set(
124-
nodes.filter(isKeyboardOwnedRoleNode).map((node) => node.index),
89+
// Role-declared keyboard surfaces, reached because the interactive projection flattens them
90+
// beside app content: chrome PER NODE (surface roles), chrome by SUBTREE (system-only hosts),
91+
// or chrome AS A MARKER with no subtree (hosts the app can also fill — its custom inputView,
92+
// its inputAssistantItem bar buttons — so the app content under a marker survives).
93+
const surfaceIndexes = new Set(
94+
nodes
95+
.filter(
96+
(node) =>
97+
hasIosKeyboardRole(node, IOS_KEYBOARD_SURFACE_ROLES) ||
98+
hasIosKeyboardRole(node, IOS_KEYBOARD_SETTLE_CHROME_ROLES),
99+
)
100+
.map((node) => node.index),
125101
);
126102
const markerIndexes = new Set(
127103
nodes
128-
.filter((node) => hasRole(node, IOS_KEYBOARD_CHROME_MARKER_ROLES))
104+
.filter((node) => hasIosKeyboardRole(node, IOS_KEYBOARD_CHROME_MARKER_ROLES))
129105
.map((node) => node.index),
130106
);
131-
if (
132-
containerIndexes.size === 0 &&
133-
ownedSubtreeRootIndexes.size === 0 &&
134-
markerIndexes.size === 0
135-
) {
107+
if (containerIndexes.size === 0 && surfaceIndexes.size === 0 && markerIndexes.size === 0) {
136108
return EMPTY_SETTLE_CHROME;
137109
}
138110
const containerSubtreeIndexes = collectSubtreeIndexes(nodes, byIndex, containerIndexes);
@@ -143,36 +115,31 @@ function collectKeyboardChrome(nodes: SnapshotNode[]): SettleChrome {
143115
}
144116
// Preserve app-owned accessory controls hosted above the keyboard: they were
145117
// swept in by the whole-window pass but are not keyboard chrome.
146-
const accessoryExemptIndexes = collectKeyboardAccessoryIndexes(
118+
for (const index of collectKeyboardAccessoryIndexes(
147119
nodes,
148120
byIndex,
149121
windowIndexes,
150122
containerIndexes,
151123
containerSubtreeIndexes,
152-
);
153-
for (const index of accessoryExemptIndexes) {
124+
)) {
154125
chromeIndexes.delete(index);
155126
}
156-
// Keyboard-owned classes the projection flattened out of every reachable subtree are chrome
157-
// by their own declaration — but the walk may NOT outrank the app-accessory exemption above:
158-
// an app's `UITextField.inputAssistantItem` bar buttons render UNDER the system assistant
159-
// view, above the keys, and the window pass just exempted them; re-claiming them through the
160-
// assistant's subtree would hide real app controls from the settle diff. An exempted node
161-
// only re-joins chrome when IT declares a keyboard-owned role (a prediction cell is
162-
// system-owned wherever it sits); an app node under an owned host stays exempt. In the
163-
// flattened interactive shape there is no window anchor, the exemption is empty, and the
164-
// walk claims the key plane exactly as the container-descendant walk always did.
165-
for (const index of collectOwnedKeyboardChromeIndexes(
127+
// No role claim may reach app content: surface roles are chrome per node, subtree claims only
128+
// start at system-only hosts the app can never fill, and markers own no subtree. That is why
129+
// these passes need no interaction with the accessory exemption or an editable-text guard —
130+
// an app node under the assistant view or the inputView placeholder is simply never claimed.
131+
for (const index of surfaceIndexes) chromeIndexes.add(index);
132+
for (const index of collectSubtreeIndexes(
166133
nodes,
167134
byIndex,
168-
ownedSubtreeRootIndexes,
169-
containerSubtreeIndexes,
170-
accessoryExemptIndexes,
135+
new Set(
136+
nodes
137+
.filter((node) => hasIosKeyboardRole(node, IOS_KEYBOARD_SUBTREE_CHROME_ROLES))
138+
.map((node) => node.index),
139+
),
171140
)) {
172141
chromeIndexes.add(index);
173142
}
174-
// The system inputView host frame is itself chrome, but owning no subtree: an app's custom
175-
// input view content under it survives classification.
176143
for (const index of markerIndexes) chromeIndexes.add(index);
177144
const refs = new Set(
178145
nodes.filter((node) => node.ref && chromeIndexes.has(node.index)).map((node) => node.ref),
@@ -185,70 +152,6 @@ function collectKeyboardChrome(nodes: SnapshotNode[]): SettleChrome {
185152
return { strippedIndexes, refs };
186153
}
187154

188-
/**
189-
* The subtree under each keyboard-owned-class root, minus two app-content guards: indexes the
190-
* app-accessory exemption already granted (an app's `inputAssistantItem` bar buttons ride the
191-
* system assistant view above the keys), and editable text the app hosts inside an owned root
192-
* outside the keyboard container, with its subtree. A keyboard's own `Key` nodes trip neither —
193-
* they sit inside the container subtree, which the accessory pass never exempts — and the
194-
* flattened interactive shape carries no window anchor, so the exemption is empty there and the
195-
* walk claims the whole key plane exactly as the container-descendant walk always did.
196-
*/
197-
function collectOwnedKeyboardChromeIndexes(
198-
nodes: SnapshotNode[],
199-
byIndex: Map<number, SnapshotNode>,
200-
ownedSubtreeRootIndexes: ReadonlySet<number>,
201-
containerSubtreeIndexes: ReadonlySet<number>,
202-
accessoryExemptIndexes: ReadonlySet<number>,
203-
): Set<number> {
204-
const chromeIndexes = collectSubtreeIndexes(nodes, byIndex, ownedSubtreeRootIndexes);
205-
// An exempted node re-joins chrome only when IT declares a keyboard-owned role: a system
206-
// prediction cell is chrome wherever it sits, while an app's `inputAssistantItem` bar button
207-
// under the assistant view declares no owned role and keeps its exemption.
208-
for (const index of accessoryExemptIndexes) {
209-
if (!ownedSubtreeRootIndexes.has(index)) chromeIndexes.delete(index);
210-
}
211-
for (const index of collectHostedEditableTextIndexes(
212-
nodes,
213-
byIndex,
214-
chromeIndexes,
215-
containerSubtreeIndexes,
216-
)) {
217-
chromeIndexes.delete(index);
218-
}
219-
return chromeIndexes;
220-
}
221-
222-
/**
223-
* Editable text the app hosts under an owned root outside the keyboard container, and the
224-
* chrome claimed beneath it — the inputAccessoryView guard in disguise: an app that rides its
225-
* own composer on the assistant host keeps the field it is typing into.
226-
*/
227-
function collectHostedEditableTextIndexes(
228-
nodes: SnapshotNode[],
229-
byIndex: Map<number, SnapshotNode>,
230-
chromeIndexes: ReadonlySet<number>,
231-
containerSubtreeIndexes: ReadonlySet<number>,
232-
): Set<number> {
233-
const dropped = new Set<number>();
234-
const hostedTextIndexes = new Set(
235-
nodes
236-
.filter(
237-
(node) =>
238-
chromeIndexes.has(node.index) &&
239-
EDITABLE_TEXT_TYPES.has(normalizeType(node.type ?? '')) &&
240-
!containerSubtreeIndexes.has(node.index),
241-
)
242-
.map((node) => node.index),
243-
);
244-
for (const node of nodes) {
245-
if (!chromeIndexes.has(node.index)) continue;
246-
if (hasAncestorIn(node, byIndex, hostedTextIndexes)) dropped.add(node.index);
247-
}
248-
for (const index of hostedTextIndexes) dropped.add(index);
249-
return dropped;
250-
}
251-
252155
/**
253156
* Indexes of app-owned accessory content in a keyboard window: non-keyboard
254157
* nodes whose center sits ABOVE the keyboard container's top edge (an

‎packages/contracts/package.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,10 @@
272272
"types": "./src/interaction-guarantees.ts",
273273
"default": "./src/interaction-guarantees.ts"
274274
},
275+
"./ios-keyboard-ownership": {
276+
"types": "./src/ios-keyboard-ownership.ts",
277+
"default": "./src/ios-keyboard-ownership.ts"
278+
},
275279
"./ios-snapshot": {
276280
"types": "./src/ios-snapshot.ts",
277281
"default": "./src/ios-snapshot.ts"
Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
import type { RawSnapshotNode } from '@agent-device/kernel/snapshot';
2+
3+
/**
4+
* iOS keyboard ownership — the iOS twin of `android-input-ownership.ts`, shared by the tap
5+
* occlusion band (`tap-keyboard-occlusion.ts`) and the settle chrome classifier
6+
* (`@agent-device/capture-kit/snapshot-chrome`).
7+
*
8+
* The interactive projection flattens the keyboard surfaces beside app content, so ancestry
9+
* cannot reach them; each node still carries its producing UIKit class in `role`. The table is
10+
* exact class names only, so a near miss fails closed rather than widening what counts as
11+
* keyboard-owned (the same discipline the snapshot geometry policy applies to private UIKit
12+
* roles). Classes are locale-independent, unlike every label these nodes carry, and private to
13+
* the keyboard surfaces — an app cannot host one in its own view tree.
14+
*
15+
* Three ownership shapes, because "produced by the keyboard" is not one question:
16+
* - {@link IOS_KEYBOARD_SURFACE_ROLES}: nodes the keyboard owns PER NODE, safe for geometry
17+
* (occlusion band).
18+
* - {@link IOS_KEYBOARD_SETTLE_CHROME_ROLES}: per-node chrome for the settle classifier only —
19+
* the QuickType prediction cell. Kept out of the surface table so a QuickType candidate never
20+
* joins the occlusion band's geometry.
21+
* - {@link IOS_KEYBOARD_SUBTREE_CHROME_ROLES}: system-only hosts whose WHOLE subtree is
22+
* keyboard chrome — the key plane and the prediction bar. An app can never fill these, so a
23+
* subtree claim here cannot swallow app content.
24+
* - {@link IOS_KEYBOARD_CHROME_MARKER_ROLES}: hosts the app CAN also fill (its custom
25+
* `inputView`, its `inputAssistantItem` bar buttons ride the assistant view). The marker is
26+
* chrome itself and owns NO subtree: app content under it survives classification.
27+
*/
28+
29+
/** Keyboard-owned per node; contributes geometry wherever the projection places it. */
30+
export const IOS_KEYBOARD_SURFACE_ROLES: ReadonlySet<string> = new Set([
31+
'UIKeyboardDockItemButton',
32+
'UIAccessibilityElementKBKey',
33+
]);
34+
35+
/** Per-node settle chrome beyond the surface table; never occlusion geometry. */
36+
export const IOS_KEYBOARD_SETTLE_CHROME_ROLES: ReadonlySet<string> = new Set([
37+
'TUIPredictionViewCell',
38+
]);
39+
40+
/** System-only keyboard hosts; their entire subtree is keyboard chrome. */
41+
export const IOS_KEYBOARD_SUBTREE_CHROME_ROLES: ReadonlySet<string> = new Set([
42+
'UIKeyboardLayoutStar',
43+
'TUIPredictionView',
44+
]);
45+
46+
/** Keyboard hosts the app can also fill: chrome itself, subtree left to the app. */
47+
export const IOS_KEYBOARD_CHROME_MARKER_ROLES: ReadonlySet<string> = new Set([
48+
'_UIRemoteKeyboardPlaceholderView',
49+
'TUISystemInputAssistantView',
50+
'TUISystemInputAssistantPageView',
51+
]);
52+
53+
/** True when this node's own UIKit class is one of `roles`. */
54+
export function hasIosKeyboardRole(node: RawSnapshotNode, roles: ReadonlySet<string>): boolean {
55+
return node.role !== undefined && roles.has(node.role);
56+
}

0 commit comments

Comments
 (0)