Skip to content

Commit ef98513

Browse files
committed
fix: classify the whole iOS keyboard window as settle chrome
Live-device review of the first Bug B fix found "Next keyboard" and "Dictate" still leaking into the settled diff as added refs: on a real iPhone 17 Pro simulator they live in a SIBLING subtree of the [Keyboard] container (the candidate bar), not under it, so the container-descendant walk missed them. A raw hierarchy capture shows the software keyboard in its own dedicated window hosting both the container and the candidate bar, so the structural rule is now: every node inside a window that has a [Keyboard] descendant is keyboard chrome. Conservative guard: a window also hosting an editable text node outside the container (iOS puts inputAccessoryView composers in the keyboard window) is never window-classified — those fall back to the container-descendant walk. The live run also showed the filled field re-labeling itself with its new value (ancestor wrappers inherit it), which produced added refs that suppressed the tail even with chrome fixed. The trigger now also ignores self-echo refs — added lines whose settled node rect contains the action point — since they re-describe the acted-on element, not a new target. The fill-keyboard provider fixture is now a trimmed REAL capture from the benchmark flow (sibling candidate bar, main app window absent from the interactive settled capture, self-echo relabels) instead of a hand-built tree that hid the sibling-branch shape.
1 parent 347ea1f commit ef98513

4 files changed

Lines changed: 664 additions & 163 deletions

File tree

‎src/commands/interaction/runtime/settle.test.ts‎

Lines changed: 263 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -313,53 +313,118 @@ test('the settled diff line list is bounded with a truncation marker', async ()
313313
assert.equal(diff.truncated, true);
314314
});
315315

316-
test('keyboard Key nodes and chrome buttons never spend the settled diff budget', async () => {
317-
const before = buttonSnapshot();
318-
// A fill-style settled tree: a summoned keyboard (container + keys + real
319-
// XCUIElementTypeButton chrome like shift/Emoji/return/Dictate/Next
320-
// keyboard — NOT Key nodes) plus the content change the agent actually
321-
// cares about.
322-
const keyboardTree = makeSnapshotState([
316+
// Keyboard chrome fixtures mirror the LIVE capture shape (iPhone 17 Pro sim,
317+
// iOS 26, July 2026, React Navigation playground): the keyboard renders in
318+
// its own dedicated window; the [Keyboard] container (keys + shift/Emoji/
319+
// return buttons) and the "Next keyboard"/"Dictate" candidate-bar buttons are
320+
// SIBLING subtrees under that window — the candidate-bar buttons are NOT
321+
// descendants of the container, which is exactly what an earlier hand-built
322+
// fixture got wrong (a container-descendant walk alone misses them).
323+
function keyboardWindowNodes(startIndex: number, parentIndex?: number) {
324+
const kb = startIndex;
325+
return [
323326
{
324-
index: 0,
325-
depth: 0,
326-
type: 'StaticText',
327-
label: 'Results for alpenglow',
328-
rect: { x: 0, y: 0, width: 200, height: 20 },
327+
index: kb,
328+
depth: 1,
329+
...(parentIndex !== undefined ? { parentIndex } : {}),
330+
type: 'Window',
331+
label: 'Next keyboard',
332+
rect: { x: 0, y: 0, width: 402, height: 874 },
329333
hittable: true,
330334
},
331335
{
332-
index: 1,
333-
depth: 0,
334-
type: 'Keyboard',
335-
label: 'keyboard',
336-
rect: { x: 0, y: 500, width: 400, height: 300 },
336+
index: kb + 1,
337+
depth: 3,
338+
parentIndex: kb,
339+
type: 'Other',
340+
label: 'Next keyboard',
341+
rect: { x: 0, y: 566, width: 402, height: 308 },
337342
hittable: true,
338343
},
344+
{
345+
index: kb + 2,
346+
depth: 4,
347+
parentIndex: kb + 1,
348+
type: 'Other',
349+
label: 'Padding-Left',
350+
rect: { x: 0, y: 583, width: 402, height: 233 },
351+
},
352+
{
353+
index: kb + 3,
354+
depth: 5,
355+
parentIndex: kb + 2,
356+
type: 'Keyboard',
357+
label: 'Padding-Left',
358+
rect: { x: 0, y: 583, width: 402, height: 233 },
359+
},
339360
...Array.from({ length: 26 }, (_, key) => ({
340-
index: key + 2,
341-
depth: 1,
342-
parentIndex: 1,
361+
index: kb + 4 + key,
362+
depth: 7,
363+
parentIndex: kb + 3,
343364
type: 'Key',
344365
label: String.fromCharCode(97 + key),
345-
rect: { x: key * 10, y: 520, width: 10, height: 40 },
346-
hittable: true,
366+
rect: { x: (key % 10) * 40, y: 590 + Math.floor(key / 10) * 54, width: 39, height: 54 },
347367
})),
348-
...['shift', 'Emoji', 'return', 'Dictate', 'Next keyboard'].map((label, extra) => ({
349-
index: extra + 28,
350-
depth: 1,
351-
parentIndex: 1,
368+
...['shift', 'Emoji', 'return'].map((label, extra) => ({
369+
index: kb + 30 + extra,
370+
depth: 7,
371+
parentIndex: kb + 3,
352372
type: 'Button',
353373
label,
354-
rect: { x: extra * 40, y: 780, width: 40, height: 40 },
355-
hittable: true,
374+
rect: { x: 5 + extra * 50, y: 752, width: 49, height: 54 },
356375
})),
357-
...['Next', 'Back', 'Home'].map((label, extra) => ({
358-
index: extra + 33,
376+
// Candidate-bar chrome: siblings of the [Keyboard] container's wrapper,
377+
// not descendants of the container itself.
378+
{
379+
index: kb + 33,
380+
depth: 5,
381+
parentIndex: kb + 1,
382+
type: 'Button',
383+
label: 'Next keyboard',
384+
rect: { x: 8, y: 806, width: 68, height: 69 },
385+
hittable: true,
386+
},
387+
{
388+
index: kb + 34,
389+
depth: 5,
390+
parentIndex: kb + 1,
391+
type: 'Button',
392+
label: 'Dictate',
393+
rect: { x: 325, y: 805, width: 68, height: 69 },
394+
},
395+
];
396+
}
397+
398+
test('keyboard keys, chrome buttons, and candidate-bar siblings never spend the settled diff budget', async () => {
399+
const before = buttonSnapshot();
400+
// A fill-style settled tree: the summoned keyboard window plus the content
401+
// change the agent actually cares about.
402+
const keyboardTree = makeSnapshotState([
403+
{
404+
index: 0,
359405
depth: 0,
406+
type: 'Application',
407+
label: 'Example',
408+
rect: { x: 0, y: 0, width: 402, height: 874 },
409+
hittable: true,
410+
},
411+
...keyboardWindowNodes(1, 0),
412+
{
413+
index: 36,
414+
depth: 1,
415+
parentIndex: 0,
416+
type: 'StaticText',
417+
label: 'Results for alpenglow',
418+
rect: { x: 0, y: 0, width: 200, height: 20 },
419+
hittable: true,
420+
},
421+
...['Next', 'Back', 'Home'].map((label, extra) => ({
422+
index: extra + 37,
423+
depth: 1,
424+
parentIndex: 0,
360425
type: 'Button',
361426
label,
362-
rect: { x: 0, y: 40 + extra * 40, width: 100, height: 40 },
427+
rect: { x: 0, y: 240 + extra * 40, width: 100, height: 40 },
363428
hittable: true,
364429
})),
365430
]);
@@ -381,15 +446,17 @@ test('keyboard Key nodes and chrome buttons never spend the settled diff budget'
381446
assert.ok(diff);
382447
const texts = diff.lines.map((line) => line.text).join('\n');
383448
assert.match(texts, /Results for alpenglow/);
384-
assert.match(texts, /keyboard/);
385-
// The container line survives as the keyboard signal; individual keys and
386-
// chrome buttons (real XCUIElementTypeButton nodes, not Key nodes) do not.
449+
assert.match(texts, /\[keyboard\]/);
450+
// The container line survives as the keyboard signal; keys, chrome buttons
451+
// under the container, the candidate-bar siblings (Next keyboard/Dictate),
452+
// and the keyboard window's own wrappers all collapse.
387453
assert.ok(!diff.lines.some((line) => /\[key\]/.test(line.text)));
388454
assert.ok(!diff.lines.some((line) => /shift|Emoji|return|Dictate|Next keyboard/.test(line.text)));
389-
assert.equal(diff.summary.additions, 5);
455+
// application + keyboard container + StaticText + 3 screen buttons.
456+
assert.equal(diff.summary.additions, 6);
390457
});
391458

392-
test('keyboard-only changes (container + chrome) do not suppress the settle tail trigger', async () => {
459+
test('keyboard-only changes (window + container + chrome) do not suppress the settle tail trigger', async () => {
393460
// The field and its screen buttons are unchanged by the fill itself; the
394461
// keyboard summoning is the only tree change. Before #1167's fix, the
395462
// chrome buttons' fresh added-line refs (or, after the diff-only fix, the
@@ -423,32 +490,75 @@ test('keyboard-only changes (container + chrome) do not suppress the settle tail
423490
},
424491
];
425492
const before = makeSnapshotState(controls);
426-
const settledTree = makeSnapshotState([
427-
...controls,
493+
const settledTree = makeSnapshotState([...controls, ...keyboardWindowNodes(3)]);
494+
let captures = 0;
495+
const device = createSettleDevice({
496+
stored: before,
497+
captureSnapshot: () => {
498+
captures += 1;
499+
return { snapshot: captures === 1 ? before : settledTree };
500+
},
501+
});
502+
503+
const result = await device.interactions.fill(selector('label=Search'), 'hello', {
504+
session: 'default',
505+
settle: {},
506+
});
507+
508+
const settle = result.settle;
509+
assert.ok(settle);
510+
// Only the keyboard container line is added; the window, its wrappers, the
511+
// chrome buttons, and the candidate-bar siblings all collapse.
512+
assert.equal(settle.diff?.summary.additions, 1);
513+
assert.ok(
514+
!settle.diff?.lines.some((line) => /shift|return|Dictate|Next keyboard/i.test(line.text)),
515+
);
516+
assert.ok(settle.tail, 'keyboard-only additions must not suppress the tail');
517+
assert.deepEqual(
518+
settle.tail?.map((entry) => entry.label),
519+
['Search', 'Cancel', 'Search now'],
520+
);
521+
});
522+
523+
test('a filled field relabeling itself (self-echo added refs) does not suppress the settle tail', async () => {
524+
// Live-verified shape: filling a field re-labels the field line with its
525+
// new value (and ancestor wrappers inherit it), so the diff carries added
526+
// refs even though nothing NEW appeared on screen. Those self-echo refs
527+
// (settled node rect contains the action point) must not suppress the
528+
// tail — the next targets after a fill are the screen's unchanged controls.
529+
const before = makeSnapshotState([
428530
{
429-
index: 3,
531+
index: 0,
430532
depth: 0,
431-
type: 'Keyboard',
432-
label: 'keyboard',
433-
rect: { x: 0, y: 500, width: 400, height: 300 },
533+
type: 'TextField',
534+
rect: { x: 12, y: 129, width: 377, height: 41 },
434535
hittable: true,
435536
},
436537
{
437-
index: 4,
438-
depth: 1,
439-
parentIndex: 3,
538+
index: 1,
539+
depth: 0,
440540
type: 'Button',
441-
label: 'shift',
442-
rect: { x: 0, y: 520, width: 30, height: 40 },
541+
label: 'Submit',
542+
rect: { x: 12, y: 182, width: 378, height: 40 },
443543
hittable: true,
444544
},
545+
]);
546+
const settledTree = makeSnapshotState([
445547
{
446-
index: 5,
447-
depth: 1,
448-
parentIndex: 3,
548+
index: 0,
549+
depth: 0,
550+
type: 'TextField',
551+
label: 'hello',
552+
value: 'hello',
553+
rect: { x: 12, y: 129, width: 377, height: 41 },
554+
hittable: true,
555+
},
556+
{
557+
index: 1,
558+
depth: 0,
449559
type: 'Button',
450-
label: 'return',
451-
rect: { x: 350, y: 520, width: 50, height: 40 },
560+
label: 'Submit',
561+
rect: { x: 12, y: 182, width: 378, height: 40 },
452562
hittable: true,
453563
},
454564
]);
@@ -461,21 +571,113 @@ test('keyboard-only changes (container + chrome) do not suppress the settle tail
461571
},
462572
});
463573

464-
const result = await device.interactions.fill(selector('label=Search'), 'hello', {
574+
const result = await device.interactions.fill(ref('@e1'), 'hello', {
465575
session: 'default',
466576
settle: {},
467577
});
468578

469579
const settle = result.settle;
470580
assert.ok(settle);
471-
// Only the keyboard container line is added; its chrome buttons collapse.
472-
assert.equal(settle.diff?.summary.additions, 1);
473-
assert.ok(!settle.diff?.lines.some((line) => /shift|return/i.test(line.text)));
474-
assert.ok(settle.tail, 'keyboard-only additions must not suppress the tail');
475-
assert.deepEqual(
476-
settle.tail?.map((entry) => entry.label),
477-
['Search', 'Cancel', 'Search now'],
478-
);
581+
// The relabeled field IS an added line (with a ref) in the diff...
582+
const added = settle.diff?.lines.filter((line) => line.kind === 'added') ?? [];
583+
assert.equal(added.length, 1);
584+
assert.match(added[0]?.text ?? '', /hello/);
585+
assert.ok(added[0]?.ref);
586+
// ...but it re-describes the acted-on element, so the tail still fires
587+
// with the actual next target.
588+
assert.deepEqual(settle.tail, [{ ref: 'e2', role: 'button', label: 'Submit' }]);
589+
});
590+
591+
test('a keyboard window hosting an app composer field is never window-classified as chrome', async () => {
592+
// iOS hosts inputAccessoryView content (e.g. a messaging composer) in the
593+
// keyboard window. The conservative guard keeps such windows out of the
594+
// whole-window chrome rule: the composer field/send button must stay
595+
// visible even though candidate-bar chrome then leaks through.
596+
const before = buttonSnapshot();
597+
const settledTree = makeSnapshotState([
598+
{
599+
index: 0,
600+
depth: 0,
601+
type: 'Application',
602+
label: 'Chat',
603+
rect: { x: 0, y: 0, width: 402, height: 874 },
604+
hittable: true,
605+
},
606+
{
607+
index: 1,
608+
depth: 1,
609+
parentIndex: 0,
610+
type: 'Window',
611+
rect: { x: 0, y: 0, width: 402, height: 874 },
612+
hittable: true,
613+
},
614+
// inputAccessoryView composer: an editable field OUTSIDE the [Keyboard]
615+
// container but INSIDE the keyboard window.
616+
{
617+
index: 2,
618+
depth: 2,
619+
parentIndex: 1,
620+
type: 'TextField',
621+
label: 'Message',
622+
rect: { x: 0, y: 500, width: 320, height: 44 },
623+
hittable: true,
624+
},
625+
{
626+
index: 3,
627+
depth: 2,
628+
parentIndex: 1,
629+
type: 'Button',
630+
label: 'Send',
631+
rect: { x: 330, y: 500, width: 60, height: 44 },
632+
hittable: true,
633+
},
634+
{
635+
index: 4,
636+
depth: 2,
637+
parentIndex: 1,
638+
type: 'Keyboard',
639+
label: 'keyboard',
640+
rect: { x: 0, y: 566, width: 402, height: 308 },
641+
},
642+
{
643+
index: 5,
644+
depth: 3,
645+
parentIndex: 4,
646+
type: 'Key',
647+
label: 'q',
648+
rect: { x: 5, y: 590, width: 39, height: 54 },
649+
},
650+
{
651+
index: 6,
652+
depth: 3,
653+
parentIndex: 4,
654+
type: 'Button',
655+
label: 'shift',
656+
rect: { x: 5, y: 698, width: 51, height: 54 },
657+
},
658+
]);
659+
let captures = 0;
660+
const device = createSettleDevice({
661+
stored: before,
662+
captureSnapshot: () => {
663+
captures += 1;
664+
return { snapshot: captures === 1 ? before : settledTree };
665+
},
666+
});
667+
668+
const result = await device.interactions.press(selector('label=Continue'), {
669+
session: 'default',
670+
settle: {},
671+
});
672+
673+
const diff = result.settle?.diff;
674+
assert.ok(diff);
675+
const texts = diff.lines.map((line) => line.text).join('\n');
676+
// Composer content survives; container-descendant chrome still collapses.
677+
assert.match(texts, /Message/);
678+
assert.match(texts, /Send/);
679+
assert.match(texts, /\[keyboard\]/);
680+
assert.ok(!diff.lines.some((line) => /\[key\]|shift/.test(line.text)));
479681
});
480682

481683
test('added lines win diff-budget slots over removals under truncation', async () => {

0 commit comments

Comments
 (0)