Skip to content

Commit dfdeaba

Browse files
committed
fix: preserve app-owned keyboard accessory controls in chrome filter
The iOS classifier treated an entire keyboard WINDOW as chrome, so a button-only inputAccessoryView/toolbar (e.g. a "Send" button) the app hosts in that window was wrongly stripped from divergence screen.refs — hiding a control the agent must heal against. Narrow it structurally: the keyboard's own chrome (keys, shift/Emoji/return, Next keyboard/Dictate) renders within the keyboard container's frame at the bottom of the screen, while an inputAccessoryView renders as a bar ABOVE the keys, so non-keyboard nodes above the keyboard's top edge survive classification. Structural spine nodes that contain the keyboard are never exempted, so genuine key-only keyboard windows classify exactly as before. Android scope unchanged (app dialog / unmarked SystemUI controls already kept). Adds core classifier unit tests (iOS accessory survives, genuine keyboard unchanged, Android app/SystemUI controls kept) plus a divergence-level regression that an app inputAccessoryView control stays in screen.refs.
1 parent 6029951 commit dfdeaba

3 files changed

Lines changed: 413 additions & 2 deletions

File tree

Lines changed: 202 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,202 @@
1+
import { test } from 'vitest';
2+
import assert from 'node:assert/strict';
3+
import { attachRefs, type RawSnapshotNode, type SnapshotNode } from '../../kernel/snapshot.ts';
4+
import { collectSettleChromeRefs } from '../snapshot-chrome.ts';
5+
6+
function refFor(nodes: SnapshotNode[], label: string): string {
7+
const node = nodes.find((candidate) => candidate.label === label);
8+
assert.ok(node?.ref, `expected a node labelled "${label}" with a ref`);
9+
return node.ref;
10+
}
11+
12+
// iOS keyboard window (iPhone 17 Pro shape): the [Keyboard] container holds the
13+
// keys, and the "Next keyboard"/"Dictate" assistant buttons render inside the
14+
// keyboard's own frame at the bottom of the screen. An inputAccessoryView
15+
// toolbar the app hosts in the same window renders as a bar ABOVE the keys.
16+
function iosKeyboardWindowNodes(options: { withAccessory: boolean }): RawSnapshotNode[] {
17+
const keyboardTop = 360;
18+
const keys = Array.from({ length: 26 }, (_, key) => ({
19+
index: 100 + key,
20+
depth: 4,
21+
parentIndex: 5,
22+
type: 'Key',
23+
label: String.fromCharCode(97 + key),
24+
rect: {
25+
x: (key % 10) * 40,
26+
y: keyboardTop + 20 + Math.floor(key / 10) * 54,
27+
width: 39,
28+
height: 54,
29+
},
30+
}));
31+
return [
32+
{
33+
index: 0,
34+
depth: 0,
35+
type: 'Application',
36+
label: 'Example',
37+
rect: { x: 0, y: 0, width: 402, height: 874 },
38+
hittable: true,
39+
},
40+
{
41+
index: 1,
42+
depth: 1,
43+
parentIndex: 0,
44+
type: 'Window',
45+
label: 'Keyboard Window',
46+
rect: { x: 0, y: 300, width: 402, height: 574 },
47+
},
48+
...(options.withAccessory
49+
? [
50+
{
51+
index: 2,
52+
depth: 2,
53+
parentIndex: 1,
54+
type: 'Other',
55+
label: 'Accessory Bar',
56+
rect: { x: 0, y: 300, width: 402, height: 44 },
57+
},
58+
{
59+
index: 3,
60+
depth: 3,
61+
parentIndex: 2,
62+
type: 'Button',
63+
label: 'Send',
64+
identifier: 'composer-send',
65+
rect: { x: 300, y: 306, width: 90, height: 36 },
66+
hittable: true,
67+
},
68+
]
69+
: []),
70+
{
71+
index: 4,
72+
depth: 2,
73+
parentIndex: 1,
74+
type: 'Other',
75+
label: 'Keyboard Host',
76+
rect: { x: 0, y: keyboardTop, width: 402, height: 514 },
77+
},
78+
{
79+
index: 5,
80+
depth: 3,
81+
parentIndex: 4,
82+
type: 'Keyboard',
83+
label: 'Padding-Left',
84+
rect: { x: 0, y: keyboardTop, width: 402, height: 514 },
85+
},
86+
...keys,
87+
{
88+
index: 200,
89+
depth: 3,
90+
parentIndex: 4,
91+
type: 'Button',
92+
label: 'Next keyboard',
93+
rect: { x: 8, y: 806, width: 68, height: 44 },
94+
hittable: true,
95+
},
96+
{
97+
index: 201,
98+
depth: 3,
99+
parentIndex: 4,
100+
type: 'Button',
101+
label: 'Dictate',
102+
rect: { x: 325, y: 805, width: 68, height: 44 },
103+
},
104+
];
105+
}
106+
107+
test('iOS app-owned inputAccessoryView controls above the keyboard survive chrome classification', () => {
108+
const nodes = attachRefs(iosKeyboardWindowNodes({ withAccessory: true }));
109+
const chromeRefs = collectSettleChromeRefs(nodes, undefined);
110+
111+
// Keyboard keys and the system assistant buttons ARE chrome.
112+
assert.equal(chromeRefs.has(refFor(nodes, 'q')), true);
113+
assert.equal(chromeRefs.has(refFor(nodes, 'Next keyboard')), true);
114+
assert.equal(chromeRefs.has(refFor(nodes, 'Dictate')), true);
115+
// The app's accessory "Send" button (rendered above the keys) is NOT chrome.
116+
assert.equal(chromeRefs.has(refFor(nodes, 'Send')), false);
117+
});
118+
119+
test('iOS genuine key-only keyboard window still classifies every keyboard control as chrome', () => {
120+
const nodes = attachRefs(iosKeyboardWindowNodes({ withAccessory: false }));
121+
const chromeRefs = collectSettleChromeRefs(nodes, undefined);
122+
123+
assert.equal(chromeRefs.has(refFor(nodes, 'q')), true);
124+
assert.equal(chromeRefs.has(refFor(nodes, 'Next keyboard')), true);
125+
assert.equal(chromeRefs.has(refFor(nodes, 'Dictate')), true);
126+
// The keyboard window node itself is chrome (structural spine, never an app target).
127+
assert.equal(chromeRefs.has(refFor(nodes, 'Keyboard Window')), true);
128+
});
129+
130+
const LATIN_IME = 'com.google.android.inputmethod.latin';
131+
132+
test('Android classifier filters IME keys but keeps app dialog and unmarked SystemUI controls', () => {
133+
const nodes = attachRefs([
134+
{
135+
index: 0,
136+
depth: 0,
137+
type: 'android.widget.FrameLayout',
138+
bundleId: 'com.example.app',
139+
rect: { x: 0, y: 0, width: 1080, height: 2400 },
140+
},
141+
{
142+
index: 1,
143+
depth: 1,
144+
parentIndex: 0,
145+
type: 'android.widget.Button',
146+
label: 'Submit',
147+
bundleId: 'com.example.app',
148+
rect: { x: 40, y: 200, width: 1000, height: 140 },
149+
hittable: true,
150+
},
151+
// System dialog button (package `android`) — a legitimate actionable control.
152+
{
153+
index: 2,
154+
depth: 1,
155+
parentIndex: 0,
156+
type: 'android.widget.Button',
157+
label: 'Allow',
158+
bundleId: 'android',
159+
identifier: 'android:id/button1',
160+
rect: { x: 600, y: 1200, width: 400, height: 140 },
161+
hittable: true,
162+
},
163+
// Unmarked SystemUI overlay control (volume panel), NOT status/nav-bar chrome.
164+
{
165+
index: 3,
166+
depth: 1,
167+
parentIndex: 0,
168+
type: 'android.widget.ImageButton',
169+
label: 'Volume',
170+
bundleId: 'com.android.systemui',
171+
identifier: 'com.android.systemui:id/volume_dialog_button',
172+
rect: { x: 980, y: 900, width: 80, height: 80 },
173+
hittable: true,
174+
},
175+
// IME container + a key: both IME-owned, both chrome.
176+
{
177+
index: 4,
178+
depth: 1,
179+
parentIndex: 0,
180+
type: 'android.inputmethodservice.SoftInputWindow',
181+
bundleId: LATIN_IME,
182+
rect: { x: 0, y: 1600, width: 1080, height: 800 },
183+
},
184+
{
185+
index: 5,
186+
depth: 2,
187+
parentIndex: 4,
188+
type: 'android.inputmethodservice.Keyboard$Key',
189+
label: 'q',
190+
bundleId: LATIN_IME,
191+
rect: { x: 0, y: 1700, width: 108, height: 160 },
192+
},
193+
]);
194+
const chromeRefs = collectSettleChromeRefs(nodes, 'com.example.app');
195+
196+
// IME key IS filtered.
197+
assert.equal(chromeRefs.has(refFor(nodes, 'q')), true);
198+
// Legitimate app / system-dialog / unmarked-SystemUI controls are NOT filtered.
199+
assert.equal(chromeRefs.has(refFor(nodes, 'Submit')), false);
200+
assert.equal(chromeRefs.has(refFor(nodes, 'Allow')), false);
201+
assert.equal(chromeRefs.has(refFor(nodes, 'Volume')), false);
202+
});

‎src/core/snapshot-chrome.ts‎

Lines changed: 108 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,19 @@ const EDITABLE_TEXT_TYPES = new Set([
4646
* inputAccessoryView content (e.g. a messaging composer) in the keyboard
4747
* window, and hiding the field the user is typing into would be worse than
4848
* leaking chrome. Such windows fall back to the container-descendant walk.
49+
*
50+
* App-owned accessory guard: even when whole-window classification IS applied
51+
* (a button-only accessory has no editable text to trip the guard above), an
52+
* `inputAccessoryView` / formatting / send toolbar the app hosts in the
53+
* keyboard window is a legitimate control the agent must still see. All of the
54+
* keyboard's OWN chrome — keys, shift/Emoji/return, and the "Next keyboard" /
55+
* "Dictate" assistant buttons — renders within the keyboard container's own
56+
* frame (bottom of the screen), while an inputAccessoryView renders as a bar
57+
* ABOVE the keys. So any non-keyboard node in the keyboard window whose center
58+
* sits above the keyboard container's top edge is app-owned and survives
59+
* classification (`collectKeyboardAccessoryIndexes`). Structural spine nodes
60+
* (the window and wrappers that CONTAIN the keyboard) are never exempted, so a
61+
* genuine key-only keyboard window is classified exactly as before.
4962
*/
5063
type SettleChrome = {
5164
/** Chrome node indexes EXCLUDING the containers: stripped from the diff. */
@@ -62,11 +75,23 @@ function collectKeyboardChrome(nodes: SnapshotNode[]): SettleChrome {
6275
);
6376
if (containerIndexes.size === 0) return EMPTY_SETTLE_CHROME;
6477
const byIndex = new Map(nodes.map((node) => [node.index, node]));
65-
const chromeIndexes = collectSubtreeIndexes(nodes, byIndex, containerIndexes);
66-
const windowIndexes = resolveKeyboardWindowIndexes(nodes, byIndex, chromeIndexes);
78+
const containerSubtreeIndexes = collectSubtreeIndexes(nodes, byIndex, containerIndexes);
79+
const chromeIndexes = new Set(containerSubtreeIndexes);
80+
const windowIndexes = resolveKeyboardWindowIndexes(nodes, byIndex, containerSubtreeIndexes);
6781
for (const index of collectSubtreeIndexes(nodes, byIndex, windowIndexes)) {
6882
chromeIndexes.add(index);
6983
}
84+
// Preserve app-owned accessory controls hosted above the keyboard: they were
85+
// swept in by the whole-window pass but are not keyboard chrome.
86+
for (const index of collectKeyboardAccessoryIndexes(
87+
nodes,
88+
byIndex,
89+
windowIndexes,
90+
containerIndexes,
91+
containerSubtreeIndexes,
92+
)) {
93+
chromeIndexes.delete(index);
94+
}
7095
const refs = new Set(
7196
nodes.filter((node) => node.ref && chromeIndexes.has(node.index)).map((node) => node.ref),
7297
);
@@ -78,6 +103,87 @@ function collectKeyboardChrome(nodes: SnapshotNode[]): SettleChrome {
78103
return { strippedIndexes, refs };
79104
}
80105

106+
/**
107+
* Indexes of app-owned accessory content in a keyboard window: non-keyboard
108+
* nodes whose center sits ABOVE the keyboard container's top edge (an
109+
* `inputAccessoryView` toolbar renders above the keys). Structural spine nodes
110+
* that CONTAIN the keyboard (the window and its wrappers) are excluded — their
111+
* frames span the keys, and exempting them would leak the keyboard itself.
112+
*/
113+
function collectKeyboardAccessoryIndexes(
114+
nodes: SnapshotNode[],
115+
byIndex: Map<number, SnapshotNode>,
116+
windowIndexes: ReadonlySet<number>,
117+
containerIndexes: ReadonlySet<number>,
118+
containerSubtreeIndexes: ReadonlySet<number>,
119+
): Set<number> {
120+
const exempt = new Set<number>();
121+
if (windowIndexes.size === 0) return exempt;
122+
const keyboardTopByWindow = collectKeyboardTopByWindow(byIndex, windowIndexes, containerIndexes);
123+
if (keyboardTopByWindow.size === 0) return exempt;
124+
const spineIndexes = collectContainerAncestorIndexes(byIndex, containerIndexes);
125+
for (const node of nodes) {
126+
const eligible =
127+
!containerSubtreeIndexes.has(node.index) && !spineIndexes.has(node.index) && node.rect;
128+
if (eligible && isAboveKeyboard(node, byIndex, keyboardTopByWindow)) exempt.add(node.index);
129+
}
130+
return exempt;
131+
}
132+
133+
/** True when the node renders above its keyboard window's key area (an accessory bar). */
134+
function isAboveKeyboard(
135+
node: SnapshotNode,
136+
byIndex: Map<number, SnapshotNode>,
137+
keyboardTopByWindow: ReadonlyMap<number, number>,
138+
): boolean {
139+
if (!node.rect) return false;
140+
const window = findNearestWindowAncestor(node, byIndex);
141+
if (window === undefined) return false;
142+
const keyboardTop = keyboardTopByWindow.get(window);
143+
return keyboardTop !== undefined && node.rect.y + node.rect.height / 2 < keyboardTop;
144+
}
145+
146+
/** Per keyboard window, the top edge (min `rect.y`) of its keyboard container(s). */
147+
function collectKeyboardTopByWindow(
148+
byIndex: Map<number, SnapshotNode>,
149+
windowIndexes: ReadonlySet<number>,
150+
containerIndexes: ReadonlySet<number>,
151+
): Map<number, number> {
152+
const keyboardTopByWindow = new Map<number, number>();
153+
for (const containerIndex of containerIndexes) {
154+
const container = byIndex.get(containerIndex);
155+
if (!container?.rect) continue;
156+
const window = findNearestWindowAncestor(container, byIndex);
157+
if (window === undefined || !windowIndexes.has(window)) continue;
158+
const currentTop = keyboardTopByWindow.get(window);
159+
if (currentTop === undefined || container.rect.y < currentTop) {
160+
keyboardTopByWindow.set(window, container.rect.y);
161+
}
162+
}
163+
return keyboardTopByWindow;
164+
}
165+
166+
/** Every ancestor of every keyboard container (the structural spine that hosts it). */
167+
function collectContainerAncestorIndexes(
168+
byIndex: Map<number, SnapshotNode>,
169+
containerIndexes: ReadonlySet<number>,
170+
): Set<number> {
171+
const ancestors = new Set<number>();
172+
for (const containerIndex of containerIndexes) {
173+
const container = byIndex.get(containerIndex);
174+
let current =
175+
container && typeof container.parentIndex === 'number'
176+
? byIndex.get(container.parentIndex)
177+
: undefined;
178+
while (current) {
179+
ancestors.add(current.index);
180+
current =
181+
typeof current.parentIndex === 'number' ? byIndex.get(current.parentIndex) : undefined;
182+
}
183+
}
184+
return ancestors;
185+
}
186+
81187
/** The root indexes plus every node whose parent chain passes through one. */
82188
function collectSubtreeIndexes(
83189
nodes: SnapshotNode[],

0 commit comments

Comments
 (0)