Skip to content

Commit 18ec27e

Browse files
committed
fix: preserve compact snapshot action labels
1 parent 07d5280 commit 18ec27e

9 files changed

Lines changed: 182 additions & 5 deletions

File tree

‎src/daemon/snapshot-presentation/ios/index.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,9 @@ const IOS_PRESENTATION_RULES: Array<
1313
> = [
1414
// Semantic representatives must be collected before duplicate-label suppression.
1515
collectIosWebSemanticPresentation,
16-
collectIosPresentationNoiseSuppression,
1716
collectIosImplicitScrollableActions,
1817
collectIosRowPresentation,
18+
collectIosPresentationNoiseSuppression,
1919
];
2020

2121
export function presentIosInteractiveSnapshot(nodes: RawSnapshotNode[]): RawSnapshotNode[] {
Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
import type { RawSnapshotNode } from '@agent-device/kernel/snapshot';
2+
3+
const roomRowRect = { x: 0, y: 400, width: 402, height: 96 };
4+
5+
export const elementClassicRoomListNodes: RawSnapshotNode[] = [
6+
{ index: 0, depth: 0, type: 'Application', label: 'Element Classic' },
7+
{ index: 1, depth: 1, parentIndex: 0, type: 'Table', label: 'HomeVCTableView' },
8+
{
9+
index: 2,
10+
depth: 2,
11+
parentIndex: 1,
12+
type: 'Cell',
13+
identifier: 'RecentTableViewCell',
14+
label: 'Vertical scroll bar, 1 page',
15+
rect: roomRowRect,
16+
},
17+
{ index: 3, depth: 3, parentIndex: 2, type: 'StaticText', label: 'Book Club' },
18+
{
19+
index: 4,
20+
depth: 3,
21+
parentIndex: 2,
22+
type: 'StaticText',
23+
label: 'Dave: Loving it so far, the worldbuilding is great.',
24+
},
25+
{ index: 5, depth: 3, parentIndex: 2, type: 'StaticText', label: '03:55' },
26+
{
27+
index: 6,
28+
depth: 2,
29+
parentIndex: 1,
30+
type: 'Cell',
31+
identifier: 'RecentTableViewCell',
32+
label: 'Vertical scroll bar, 1 page',
33+
rect: { ...roomRowRect, y: 496 },
34+
},
35+
{ index: 7, depth: 3, parentIndex: 6, type: 'StaticText', label: 'Team Standup' },
36+
{ index: 8, depth: 3, parentIndex: 6, type: 'StaticText', label: 'Dave: No blockers from me.' },
37+
{ index: 9, depth: 3, parentIndex: 6, type: 'StaticText', label: '03:55' },
38+
];
39+
40+
export const legitimatelyLabeledCellNodes: RawSnapshotNode[] = [
41+
{ index: 0, depth: 0, type: 'Application', label: 'Example' },
42+
{
43+
index: 1,
44+
depth: 1,
45+
parentIndex: 0,
46+
type: 'Cell',
47+
label: 'Stem Cell',
48+
rect: roomRowRect,
49+
},
50+
{ index: 2, depth: 2, parentIndex: 1, type: 'StaticText', label: 'Research details' },
51+
];
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
import { expect, test } from 'vitest';
2+
import { elementClassicRoomListNodes, legitimatelyLabeledCellNodes } from './rows.fixtures.ts';
3+
import { presentIosInteractiveSnapshot } from './index.ts';
4+
5+
test('iOS row presentation associates generic room cells with their descendant titles', () => {
6+
const nodes = presentIosInteractiveSnapshot(elementClassicRoomListNodes);
7+
8+
expect(nodes.filter((node) => node.type === 'Cell').map((node) => node.label)).toEqual([
9+
'Book Club',
10+
'Team Standup',
11+
]);
12+
expect(nodes.filter((node) => node.label === 'Book Club')).toHaveLength(1);
13+
expect(nodes.filter((node) => node.label === 'Team Standup')).toHaveLength(1);
14+
});
15+
16+
test('iOS row presentation preserves a legitimate cell label', () => {
17+
const nodes = presentIosInteractiveSnapshot(legitimatelyLabeledCellNodes);
18+
19+
expect(nodes.find((node) => node.type === 'Cell')?.label).toBe('Stem Cell');
20+
});

‎src/daemon/snapshot-presentation/ios/rows.ts‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import type { RawSnapshotNode } from '@agent-device/kernel/snapshot';
22
import { normalizeType } from '@agent-device/contracts/snapshot';
3+
import { isSystemScrollIndicatorLabel } from '../../../utils/scroll-indicator.ts';
34
import {
45
areRectsApproximatelyEqual,
56
collectDescendants,
@@ -15,14 +16,51 @@ export function collectIosRowPresentation(
1516
): void {
1617
for (let position = 0; position < nodes.length; position += 1) {
1718
const row = nodes[position];
18-
const rowLabel = row?.label?.trim();
19+
const rowLabel = row ? resolveIosRowLabel(nodes, position, row, context) : undefined;
1920
if (!row?.rect || !rowLabel) {
2021
continue;
2122
}
2223
collectIosRowPresentationForNode(nodes, position, row, rowLabel, context);
2324
}
2425
}
2526

27+
function resolveIosRowLabel(
28+
nodes: RawSnapshotNode[],
29+
position: number,
30+
row: RawSnapshotNode,
31+
context: SnapshotTreeRuleContext,
32+
): string | undefined {
33+
const rowLabel = row.label?.trim();
34+
if (
35+
normalizeType(row.type ?? '') !== 'cell' ||
36+
(rowLabel && !isImplementationCellLabel(rowLabel) && !isSystemScrollIndicatorLabel(rowLabel))
37+
) {
38+
return rowLabel;
39+
}
40+
41+
const titleNode = collectDescendants(nodes, position).find(isIosRowTitleCandidate);
42+
const title = titleNode?.label?.trim();
43+
if (!titleNode || !title) {
44+
return rowLabel;
45+
}
46+
mergeReplacement(context.replacements, row, { label: title });
47+
context.suppressedIndexes.add(titleNode.index);
48+
return title;
49+
}
50+
51+
function isImplementationCellLabel(label: string): boolean {
52+
return label !== 'Cell' && /^[A-Z_$][A-Za-z0-9_$]*Cell$/.test(label);
53+
}
54+
55+
function isIosRowTitleCandidate(node: RawSnapshotNode): boolean {
56+
const label = node.label?.trim();
57+
if (!label) {
58+
return false;
59+
}
60+
const type = normalizeType(node.type ?? '');
61+
return type === 'statictext' || type === 'text' || type === 'textview';
62+
}
63+
2664
function collectIosRowPresentationForNode(
2765
nodes: RawSnapshotNode[],
2866
position: number,

‎src/daemon/snapshot-presentation/ios/scroll.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@ export function collectIosScrollIndicatorPresentation(
1515
context: SnapshotTreeRuleContext,
1616
): void {
1717
for (const node of nodes) {
18-
if (!isIosScrollIndicatorNode(node)) {
18+
const presentedNode = context.replacements.get(node.index) ?? node;
19+
if (!isIosScrollIndicatorNode(presentedNode)) {
1920
continue;
2021
}
2122
collectIosScrollIndicatorNodePresentation(node, context.sourceNodesByIndex, context);

‎src/platforms/android/__tests__/ui-hierarchy.test.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,21 @@ test('parseUiHierarchy keeps visible Android nodes with meaningful test identifi
8787
);
8888
});
8989

90+
test('interactive Android snapshots keep Compose semantic views inside actionable parents', () => {
91+
const xml = `<hierarchy>
92+
<node class="android.view.View" package="com.example.app" bounds="[923,158][1049,284]" clickable="true" visible-to-user="true">
93+
<node class="android.view.View" package="com.example.app" content-desc="Start a call" bounds="[954,189][1017,252]" visible-to-user="true"/>
94+
<node class="android.widget.Button" package="com.example.app" bounds="[933,168][1038,273]" visible-to-user="true"/>
95+
</node>
96+
</hierarchy>`;
97+
98+
const result = parseUiHierarchy(xml, 800, { interactiveOnly: true });
99+
100+
assert.equal(result.nodes[0]?.hittable, true);
101+
assert.equal(result.nodes[1]?.label, 'Start a call');
102+
assert.equal(result.nodes[1]?.parentIndex, 0);
103+
});
104+
90105
test('interactive Android snapshots keep a fixed sibling outside filtered scroll content (#1377)', () => {
91106
const xml = `<hierarchy>
92107
<node class="android.widget.FrameLayout" bounds="[0,0][400,800]" visible-to-user="true">

‎src/platforms/android/ui-hierarchy.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -805,7 +805,10 @@ function shouldIncludeInteractiveProxyNode(
805805
): boolean {
806806
if (!info.hasMeaningfulText && !info.hasMeaningfulId) return false;
807807
if (info.isVisual) return false;
808-
if (info.isStructural && !ancestorCollection) return false;
808+
// Compose commonly places the app-owned content description on a passive
809+
// `android.view.View` inside the clickable container. Keep that semantic
810+
// proxy so presentation can associate the label with the action.
811+
if (info.isStructural && !ancestorCollection && !ancestorHittable) return false;
809812
return ancestorHittable || descendantHittable || ancestorCollection;
810813
}
811814

‎src/utils/__tests__/output.test.ts‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -645,6 +645,38 @@ test('formatSnapshotText promotes Android helper unlabeled action rows', () => {
645645
assert.match(raw, /"Mobile, Wi-Fi, hotspot"/);
646646
});
647647

648+
test('formatSnapshotText promotes Android Compose leaf-view semantics to their action', () => {
649+
const text = withNoColor(() =>
650+
formatSnapshotText({
651+
nodes: [
652+
{
653+
ref: 'e1',
654+
index: 0,
655+
depth: 0,
656+
type: 'android.view.View',
657+
rect: { x: 923, y: 158, width: 126, height: 126 },
658+
hittable: true,
659+
},
660+
{
661+
ref: 'e2',
662+
index: 1,
663+
depth: 1,
664+
parentIndex: 0,
665+
type: 'android.view.View',
666+
label: 'Start a call',
667+
rect: { x: 954, y: 189, width: 63, height: 63 },
668+
},
669+
],
670+
truncated: false,
671+
androidSnapshot: { backend: 'android-helper' },
672+
}),
673+
);
674+
675+
assert.match(text, /Snapshot: 1 visible nodes \(2 total\)/);
676+
assert.match(text, /@e1 \[group\] "Start a call"/);
677+
assert.doesNotMatch(text, /@e2/);
678+
});
679+
648680
test('formatSnapshotText keeps passive row descendants that were not promoted', () => {
649681
const nodes = [
650682
{

‎src/utils/android-helper-presentation/structural-noise.ts‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ function collectPromotableRowContent(
116116
for (const descendant of descendants) {
117117
if (
118118
removed.has(descendant.index) ||
119-
!isPassiveRowContent(descendant) ||
119+
!isPromotableRowContent(descendant, descendants) ||
120120
!isRectContainedBy(descendant.rect, parent.rect)
121121
) {
122122
continue;
@@ -131,6 +131,23 @@ function collectPromotableRowContent(
131131
return { label: labels.join(', '), removableIndexes };
132132
}
133133

134+
function isPromotableRowContent(node: SnapshotNode, descendants: SnapshotNode[]): boolean {
135+
if (isPassiveRowContent(node)) return true;
136+
137+
// Jetpack Compose can expose a content description through a passive leaf
138+
// `View` rather than TextView/ImageView. It is safe to associate only the
139+
// app-provided label of a leaf with its actionable containing row: promoting
140+
// a structural subtree could otherwise hide independently actionable nodes.
141+
const type = (node.type ?? '').toLowerCase();
142+
return (
143+
node.hittable !== true &&
144+
!isEditableNode(node) &&
145+
type.includes('view') &&
146+
visibleNodeLabel(node).trim().length > 0 &&
147+
!descendants.some((candidate) => candidate.parentIndex === node.index)
148+
);
149+
}
150+
134151
function isPassiveRowContent(node: SnapshotNode): boolean {
135152
if (node.hittable === true || isEditableNode(node)) return false;
136153
const type = (node.type ?? '').toLowerCase();

0 commit comments

Comments
 (0)