Skip to content

Commit bcc1950

Browse files
okwasniewskiokwasniewski
committed
fix(snapshot): exempt a toolbar host only over its own presentation
A sheet host enclosing a stacked cover's bars, or the screen beneath, now keeps covering, so @ref press no longer reaches controls under a fullScreenCover. Scroll evidence requires a scroll container, not an indicator.
1 parent bd142d4 commit bcc1950

2 files changed

Lines changed: 224 additions & 32 deletions

File tree

‎packages/capture-kit/src/snapshot-occlusion.ts‎

Lines changed: 60 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,7 @@ import type { RawSnapshotNode, Rect } from '@agent-device/kernel/snapshot';
22
import { centerOfRect } from '@agent-device/kernel/snapshot';
33
import { areRectsApproximatelyEqual, normalizeRect } from '@agent-device/kernel/rect-center';
44
import { containsPoint, rectContains } from '@agent-device/kernel/rect';
5-
import {
6-
isScrollableNodeLike,
7-
normalizeType,
8-
isViewportRootNode,
9-
} from '@agent-device/contracts/snapshot';
5+
import { normalizeType, isViewportRootNode } from '@agent-device/contracts/snapshot';
106

117
const COVERED_PRESENTATION_HINT = 'covered';
128
const OVERLAY_KIND_FRAGMENTS = [
@@ -23,6 +19,7 @@ const OVERLAY_KIND_FRAGMENTS = [
2319
];
2420
const VIEWPORT_CHROME_KIND_FRAGMENTS = ['tabbar', 'toolbar', 'navigationbar'];
2521
const TOOLBAR_HOST_KIND_FRAGMENTS = ['toolbar'];
22+
const SCROLL_CONTAINER_TYPES = new Set(['collectionview', 'table', 'scrollview']);
2623
const SEMANTIC_TOUCH_KIND_FRAGMENTS = [
2724
'button',
2825
'link',
@@ -280,7 +277,7 @@ function isOverlayLikeNode(
280277
if (isViewportRootNode(node)) return false;
281278
if (isFullViewportChromeContainer(node, neighbourhood.byIndex)) return false;
282279
if (isSiblingSizedChromeContainer(node, neighbourhood)) return false;
283-
if (isSiblingEnclosingToolbarHost(node, neighbourhood)) return false;
280+
if (isOwnPresentationToolbarHost(node, neighbourhood)) return false;
284281
// This is a presentation-order heuristic: only known floating UI chrome should cover
285282
// later targets. Generic hittable containers can appear later without being visually on top.
286283
return (
@@ -304,6 +301,12 @@ function isOverlayLikeNode(
304301
* pass-through host from an opaque bar of the same frame, and chrome siblings that share one frame
305302
* excuse each other. Both captures show the match only where content was being refused, and the kind
306303
* gate keeps a `dialog`/`sheet`/`alert` sized to the content around it covering.
304+
*
305+
* Accepted limit for stacked presentations: the interactive projection lists them as siblings of one
306+
* root. iOS drops what sits beneath a sheet from the capture, but keeps what sits beneath a
307+
* full-screen cover, and the cover's toolbar host is viewport-sized, so the full-viewport exemption
308+
* lets it through. Beneath a cover, only the cover's own bars and a lower presentation's
309+
* non-exempt toolbar host still cover.
307310
*/
308311
// Mutation-lane note: the `typeof parentIndex` and `!rect` guards below are provably
309312
// redundant — with no parent index the sibling lookup finds nothing and answers "not
@@ -318,45 +321,70 @@ function isSiblingSizedChromeContainer(
318321
if (typeof node.parentIndex !== 'number') return false;
319322
const rect = positiveRect(node.rect);
320323
if (!rect) return false;
321-
const siblings = neighbourhood.childrenByParent.get(node.parentIndex) ?? [];
324+
const { siblings, position } = listedSiblings(node, neighbourhood);
322325
return siblings.some(
323-
(sibling) => sibling.index !== node.index && areRectsApproximatelyEqual(rect, sibling.rect),
326+
(sibling, siblingPosition) =>
327+
siblingPosition !== position && areRectsApproximatelyEqual(rect, sibling.rect),
324328
);
325329
}
326330

327331
/**
328-
* Exempt a toolbar host that encloses a listed sibling bar or scroll region. A bar does not enclose
329-
* another bar or a list; SwiftUI's toolbar host does: it spans the presentation it belongs to (a
330-
* sheet, the navigation container), so its frame wraps the navigation bar and the list listed beside
331-
* it, while the bar it really draws is its own descendants.
332-
*
333-
* The sibling-sized rule cannot see this host once a projection reshapes the tree: pruning drops the
334-
* pass-through wrappers that share the host's frame, and the scroll-indicator viewport rewrites the
335-
* list's frame to its visible band, so no sibling keeps the host's frame. Captured in `snapshot -i`
336-
* on an iOS 27 simulator, this refused every control of a SwiftUI sheet, with or without a visible
337-
* navigation bar beside the list.
332+
* Exempt a toolbar host that encloses only its own presentation's bars and scroll regions. SwiftUI's
333+
* toolbar host spans the presentation it belongs to while drawing only its descendants; once
334+
* projection pruning reshapes its siblings, the sibling-sized rule no longer sees it.
338335
*
339-
* Only the toolbar kind hosts this way: a navigation bar enclosing a same-kind sibling is a stacked
340-
* presentation drawn over the one beneath it. The enclosed sibling must be chrome or a scroll region,
341-
* since a host enclosing a plain content view cannot be told from a bar painted over that view.
336+
* Presentations are listed in z-order, each ending with its toolbar host, so the host's own
337+
* presentation is the run of siblings after the previous toolbar sibling. An enclosed bar or scroll
338+
* region outside that run belongs to a presentation stacked under or over the host, and the host
339+
* keeps covering.
342340
*/
343-
function isSiblingEnclosingToolbarHost(
341+
function isOwnPresentationToolbarHost(
344342
node: RawSnapshotNode,
345343
neighbourhood: NeighbourhoodIndex,
346344
): boolean {
347345
if (!nodeKindIncludesAny(node, TOOLBAR_HOST_KIND_FRAGMENTS)) return false;
348-
if (typeof node.parentIndex !== 'number') return false;
349346
const rect = positiveRect(node.rect);
350347
if (!rect) return false;
351-
const siblings = neighbourhood.childrenByParent.get(node.parentIndex) ?? [];
352-
return siblings.some((sibling) => {
353-
if (sibling.index === node.index) return false;
354-
const siblingRect = positiveRect(sibling.rect);
355-
if (!siblingRect || !rectContains(rect, siblingRect)) return false;
356-
return (
357-
nodeKindIncludesAny(sibling, VIEWPORT_CHROME_KIND_FRAGMENTS) || isScrollableNodeLike(sibling)
358-
);
359-
});
348+
const { siblings, position } = listedSiblings(node, neighbourhood);
349+
const ownStart = ownPresentationStart(siblings, position);
350+
const enclosed = siblings.flatMap((sibling, siblingPosition) =>
351+
siblingPosition !== position && isEnclosedHostedRegion(rect, sibling) ? [siblingPosition] : [],
352+
);
353+
return (
354+
enclosed.length > 0 &&
355+
enclosed.every((siblingPosition) => siblingPosition >= ownStart && siblingPosition < position)
356+
);
357+
}
358+
359+
/** Where the presentation ending at `position` begins: just after the previous toolbar sibling. */
360+
function ownPresentationStart(siblings: readonly RawSnapshotNode[], position: number): number {
361+
let start = 0;
362+
for (const [siblingPosition, sibling] of siblings.entries()) {
363+
if (siblingPosition >= position) break;
364+
if (nodeKindIncludesAny(sibling, TOOLBAR_HOST_KIND_FRAGMENTS)) start = siblingPosition + 1;
365+
}
366+
return start;
367+
}
368+
369+
/** A bar or a scroll container (by type, so a scroll indicator does not count) inside `rect`. */
370+
function isEnclosedHostedRegion(rect: Rect, node: RawSnapshotNode): boolean {
371+
const isHostedRegion =
372+
nodeKindIncludesAny(node, VIEWPORT_CHROME_KIND_FRAGMENTS) ||
373+
SCROLL_CONTAINER_TYPES.has(normalizeType(node.type ?? ''));
374+
const nodeRect = positiveRect(node.rect);
375+
return Boolean(isHostedRegion && nodeRect && rectContains(rect, nodeRect));
376+
}
377+
378+
/** The siblings listed under the node's parent in document order, and the node's position among them. */
379+
function listedSiblings(
380+
node: RawSnapshotNode,
381+
neighbourhood: NeighbourhoodIndex,
382+
): { siblings: readonly RawSnapshotNode[]; position: number } {
383+
const siblings =
384+
typeof node.parentIndex === 'number'
385+
? (neighbourhood.childrenByParent.get(node.parentIndex) ?? [])
386+
: [];
387+
return { siblings, position: siblings.findIndex((sibling) => sibling.index === node.index) };
360388
}
361389

362390
function isFullViewportChromeContainer(

‎packages/capture-kit/src/snapshot-sibling-sized-chrome.test.ts‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -538,3 +538,167 @@ test('a toolbar host enclosing a plain content sibling still covers it, so the e
538538

539539
assert.deepEqual(coveredLabels(nodes), ['Other', 'Barless primary']);
540540
});
541+
542+
// `snapshot -i` of a SwiftUI `fullScreenCover` presented from a `.large` sheet over the Home tab on an
543+
// iOS 27 simulator, cells trimmed. All three presentations are listed as siblings of the root, each
544+
// ending with its toolbar host. The sheet's host encloses Home's bar and list and the cover's.
545+
const COVER_OVER_SHEET_ROWS: CapturedNode[] = [
546+
[0, undefined, 'Other', 'SwiftUIApplication', 'SheetRepro', [0, 0, 402, 874]],
547+
[1, 0, 'NavigationBar', 'SwiftUI.UIKitNavigationBar', undefined, [0, 62, 402, 106]],
548+
[2, 1, 'Button', 'SwiftUI.AccessibilityNode', 'Edit', [328.67, 66, 53.33, 36]],
549+
[4, 0, 'CollectionView', 'SwiftUI.UpdateCoalescingCollectionView', undefined, [0, 116, 402, 675]],
550+
[6, 4, 'Button', 'SwiftUI.AccessibilityNode', 'Open medium sheet', [16, 168, 370, 52]],
551+
[16, 0, 'Toolbar', '_UIFloatingBarContainerView', 'Toolbar', [0, 0, 402, 874]],
552+
[17, 0, 'TabBar', 'UITabBar', 'Tab Bar', [0, 791, 402, 83]],
553+
[18, 17, 'Button', '_UITabButton', 'Home', [68, 795, 94, 54]],
554+
[21, 0, 'NavigationBar', 'SwiftUI.UIKitNavigationBar', undefined, [0, 78, 402, 106]],
555+
[22, 21, 'Button', 'SwiftUI.AccessibilityNode', 'Close', [20, 82, 67, 36]],
556+
[
557+
26,
558+
0,
559+
'CollectionView',
560+
'SwiftUI.UpdateCoalescingCollectionView',
561+
undefined,
562+
[0, 132, 402, 680],
563+
],
564+
[28, 26, 'Button', 'SwiftUI.AccessibilityNode', 'Primary action', [16, 184, 370, 52]],
565+
[43, 26, 'Button', 'SwiftUI.AccessibilityNode', 'Row 1', [16, 733, 370, 52]],
566+
[46, 0, 'Toolbar', '_UIFloatingBarContainerView', 'Toolbar', [0, 62, 402, 812]],
567+
[47, 46, 'Button', 'SwiftUI.AccessibilityNode', 'Bottom', [162, 804, 78, 36]],
568+
[48, 0, 'NavigationBar', 'SwiftUI.UIKitNavigationBar', undefined, [0, 62, 402, 54]],
569+
[49, 48, 'Button', 'SwiftUI.AccessibilityNode', 'Upper close', [20, 66, 116, 36]],
570+
[51, 48, 'Button', 'SwiftUI.AccessibilityNode', 'Upper save', [268, 66, 114, 36]],
571+
[
572+
52,
573+
0,
574+
'CollectionView',
575+
'SwiftUI.UpdateCoalescingCollectionView',
576+
undefined,
577+
[0, 116, 402, 696],
578+
],
579+
[54, 52, 'Button', 'SwiftUI.AccessibilityNode', 'Upper primary', [16, 151, 370, 52]],
580+
[57, 52, 'Button', 'SwiftUI.AccessibilityNode', 'Upper row 0', [16, 255, 370, 52]],
581+
[68, 0, 'Toolbar', '_UIFloatingBarContainerView', 'Toolbar', [0, 0, 402, 874]],
582+
[69, 68, 'Button', 'SwiftUI.AccessibilityNode', 'Upper bottom', [136.67, 804, 128.67, 36]],
583+
];
584+
585+
const HOME_ROW_INDEXES = new Set([1, 2, 4, 6, 16, 17, 18]);
586+
const COVER_ROW_INDEXES = new Set([48, 49, 51, 52, 54, 57, 68, 69]);
587+
const COVER_CONTROLS = [
588+
'Upper close',
589+
'Upper save',
590+
'Upper primary',
591+
'Upper row 0',
592+
'Upper bottom',
593+
];
594+
595+
test('a full-screen cover over a sheet leaves the sheet beneath covered and the cover pressable', () => {
596+
const covered = coveredLabels(capturedTree(COVER_OVER_SHEET_ROWS));
597+
598+
for (const label of ['Edit', 'Open medium sheet', 'Close', 'Primary action', 'Row 1']) {
599+
assert.ok(covered.includes(label), `${label} sits beneath the cover`);
600+
}
601+
for (const label of COVER_CONTROLS) {
602+
assert.ok(!covered.includes(label), `${label} is on the cover`);
603+
}
604+
});
605+
606+
test('a toolbar host enclosing the bars of a presentation stacked over it keeps covering', () => {
607+
// The capture without Home: only the cover's bar and list, listed after the sheet's host, tell it
608+
// apart from a lone sheet.
609+
const rows = COVER_OVER_SHEET_ROWS.filter(([index]) => !HOME_ROW_INDEXES.has(index));
610+
const covered = coveredLabels(capturedTree(rows));
611+
612+
assert.ok(covered.includes('Primary action'));
613+
assert.ok(covered.includes('Row 1'));
614+
for (const label of COVER_CONTROLS) assert.ok(!covered.includes(label));
615+
});
616+
617+
test('a toolbar host enclosing the bars of a presentation beneath it keeps covering', () => {
618+
// The capture without the cover: a sheet host enclosing Home's bar and list, which sit before Home's
619+
// own host, covers Home. It covers the sheet too, failing closed; iOS publishes no presentation
620+
// beneath a sheet, so this shape is not captured live.
621+
const rows = COVER_OVER_SHEET_ROWS.filter(([index]) => !COVER_ROW_INDEXES.has(index));
622+
const covered = coveredLabels(capturedTree(rows));
623+
624+
assert.ok(covered.includes('Edit'));
625+
assert.ok(covered.includes('Open medium sheet'));
626+
});
627+
628+
test.for([
629+
{
630+
name: 'a sheet over a sheet',
631+
rows: [
632+
[0, undefined, 'Other', 'SwiftUIApplication', 'SheetRepro', [0, 0, 402, 874]],
633+
[1, 0, 'NavigationBar', 'SwiftUI.UIKitNavigationBar', undefined, [0, 88, 402, 54]],
634+
[2, 1, 'Button', 'SwiftUI.AccessibilityNode', 'Upper close', [20, 92, 116, 36]],
635+
[4, 1, 'Button', 'SwiftUI.AccessibilityNode', 'Upper save', [268, 92, 114, 36]],
636+
[
637+
5,
638+
0,
639+
'CollectionView',
640+
'SwiftUI.UpdateCoalescingCollectionView',
641+
undefined,
642+
[0, 142, 402, 670],
643+
],
644+
[7, 5, 'Button', 'SwiftUI.AccessibilityNode', 'Upper primary', [16, 177, 370, 52]],
645+
[20, 5, 'Button', 'SwiftUI.AccessibilityNode', 'Upper row 5', [16, 541, 370, 52]],
646+
[21, 0, 'Toolbar', '_UIFloatingBarContainerView', 'Toolbar', [0, 72, 402, 802]],
647+
[22, 21, 'Button', 'SwiftUI.AccessibilityNode', 'Upper bottom', [136.67, 804, 128.67, 36]],
648+
],
649+
},
650+
{
651+
name: 'a sheet over a plain screen',
652+
rows: [
653+
[0, undefined, 'Other', 'SwiftUIApplication', 'SheetRepro', [0, 0, 402, 874]],
654+
[1, 0, 'NavigationBar', 'SwiftUI.UIKitNavigationBar', undefined, [0, 78, 402, 54]],
655+
[2, 1, 'Button', 'SwiftUI.AccessibilityNode', 'Upper close', [20, 82, 116, 36]],
656+
[4, 1, 'Button', 'SwiftUI.AccessibilityNode', 'Upper save', [268, 82, 114, 36]],
657+
[
658+
5,
659+
0,
660+
'CollectionView',
661+
'SwiftUI.UpdateCoalescingCollectionView',
662+
undefined,
663+
[0, 132, 402, 680],
664+
],
665+
[7, 5, 'Button', 'SwiftUI.AccessibilityNode', 'Upper primary', [16, 167, 370, 52]],
666+
[20, 5, 'Button', 'SwiftUI.AccessibilityNode', 'Upper row 5', [16, 531, 370, 52]],
667+
[21, 0, 'Toolbar', '_UIFloatingBarContainerView', 'Toolbar', [0, 62, 402, 812]],
668+
[22, 21, 'Button', 'SwiftUI.AccessibilityNode', 'Upper bottom', [136.67, 804, 128.67, 36]],
669+
],
670+
},
671+
] satisfies { name: string; rows: CapturedNode[] }[])(
672+
'the captured iOS 27 $name publishes only the top sheet, and none of it is covered',
673+
({ rows }) => {
674+
// iOS drops the presentation beneath a sheet from the capture, so no lower control gets a ref.
675+
assert.deepEqual(coveredLabels(capturedTree(rows)), []);
676+
},
677+
);
678+
679+
test('a toolbar host enclosing only a scroll indicator still covers, so the scroll evidence is a container', () => {
680+
const nodes = capturedTree([
681+
[0, undefined, 'Other', 'SwiftUIApplication', 'SheetRepro', [0, 0, 402, 874]],
682+
[1, 0, 'Other', 'UIView', undefined, [0, 100, 402, 712]],
683+
[3, 1, 'Button', 'SwiftUI.AccessibilityNode', 'Barless primary', [16, 100, 370, 49]],
684+
[4, 0, 'ScrollBar', '_UIScrollViewScrollIndicator', undefined, [396, 100, 3, 712]],
685+
[17, 0, 'Toolbar', '_UIFloatingBarContainerView', 'Toolbar', [0, 62, 402, 812]],
686+
]);
687+
688+
assert.ok(coveredLabels(nodes).includes('Barless primary'));
689+
});
690+
691+
test('a control the exempt toolbar host really hosts still covers the row it overlaps', () => {
692+
// The host is exempt for the list it encloses, but a bar-kind strip it hosts is judged by its own
693+
// rect: it covers the row beneath it and leaves the row above it alone.
694+
const nodes = capturedTree([
695+
[0, undefined, 'Other', 'SwiftUIApplication', 'SheetRepro', [0, 0, 402, 874]],
696+
[1, 0, 'CollectionView', 'SwiftUI.CollectionView', undefined, [0, 100, 402, 712]],
697+
[3, 1, 'Button', 'SwiftUI.AccessibilityNode', 'Barless primary', [16, 100, 370, 49]],
698+
[16, 1, 'Button', 'SwiftUI.AccessibilityNode', 'Barless row 12', [16, 773, 370, 39]],
699+
[17, 0, 'Toolbar', '_UIFloatingBarContainerView', 'Toolbar', [0, 62, 402, 812]],
700+
[18, 17, 'Toolbar', '_UIBarPlatterView', 'Bottom bar', [16, 760, 370, 60]],
701+
]);
702+
703+
assert.deepEqual(coveredLabels(nodes), ['Barless row 12']);
704+
});

0 commit comments

Comments
 (0)