Skip to content

Commit 6a9a93c

Browse files
committed
refactor(snapshot): preserve normalized node attributes through presentation
1 parent 8c0edab commit 6a9a93c

13 files changed

Lines changed: 195 additions & 103 deletions

‎apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotModels.swift‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ public struct SnapshotRect: Codable, Equatable {
2929
}
3030

3131
public struct RawAXNode: Equatable {
32-
public let index: Int
32+
public private(set) var index: Int
3333
public let type: String
3434
public let label: String?
3535
public let identifier: String?
@@ -42,10 +42,10 @@ public struct RawAXNode: Equatable {
4242
public let selected: Bool?
4343
/// Geometric actionability; `nil` when the capture has no viewport box to decide it against.
4444
public var hittable: Bool?
45-
public let depth: Int
46-
public let parentIndex: Int?
47-
public let hiddenContentAbove: Bool?
48-
public let hiddenContentBelow: Bool?
45+
public private(set) var depth: Int
46+
public private(set) var parentIndex: Int?
47+
public private(set) var hiddenContentAbove: Bool?
48+
public private(set) var hiddenContentBelow: Bool?
4949
public var actions: [String]?
5050

5151
public init(
@@ -91,6 +91,21 @@ public struct RawAXNode: Equatable {
9191
return updated
9292
}
9393

94+
func reindexed(index: Int, depth: Int, parentIndex: Int?) -> RawAXNode {
95+
var updated = self
96+
updated.index = index
97+
updated.depth = depth
98+
updated.parentIndex = parentIndex
99+
return updated
100+
}
101+
102+
func addingHiddenContentHints(above: Bool, below: Bool) -> RawAXNode {
103+
var updated = self
104+
updated.hiddenContentAbove = hiddenContentAbove == true || above ? true : nil
105+
updated.hiddenContentBelow = hiddenContentBelow == true || below ? true : nil
106+
return updated
107+
}
108+
94109
public var hasSemanticContent: Bool {
95110
[label, identifier, value].contains {
96111
!($0?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty ?? true)

‎apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentationProjection.swift‎

Lines changed: 2 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -134,23 +134,10 @@ extension SnapshotPresentation {
134134
return nodes.enumerated().map { offset, node in
135135
let raw = node.raw
136136
return SnapshotPresentationNode(
137-
raw: RawAXNode(
137+
raw: raw.reindexed(
138138
index: offset,
139-
type: raw.type,
140-
label: raw.label,
141-
identifier: raw.identifier,
142-
value: raw.value,
143-
placeholder: raw.placeholder,
144-
rect: raw.rect,
145-
enabled: raw.enabled,
146-
focused: raw.focused,
147-
selected: raw.selected,
148-
hittable: raw.hittable,
149139
depth: max(0, raw.depth - depthOffset),
150-
parentIndex: raw.parentIndex.flatMap { indexMap[$0] },
151-
hiddenContentAbove: raw.hiddenContentAbove,
152-
hiddenContentBelow: raw.hiddenContentBelow,
153-
actions: raw.actions
140+
parentIndex: raw.parentIndex.flatMap { indexMap[$0] }
154141
),
155142
effectiveRect: node.effectiveRect
156143
)

‎apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotVisibilityFold.swift‎

Lines changed: 10 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -191,33 +191,18 @@ public enum SnapshotVisibilityFold {
191191
if decision.isIncluded {
192192
let outIndex = kept.count
193193
let outDepth = keptDepth + 1
194+
var projected = node.reindexed(index: outIndex, depth: outDepth, parentIndex: keptIndex)
195+
projected.hittable = node.parentIndex == nil
196+
? false
197+
: clippedHittability(
198+
source: node.hittable,
199+
enabled: node.enabled,
200+
clippedFrame: decision.effectiveFrame,
201+
viewport: viewport
202+
)
194203
kept.append(
195204
SnapshotPresentationNode(
196-
raw: RawAXNode(
197-
index: outIndex,
198-
type: node.type,
199-
label: node.label,
200-
identifier: node.identifier,
201-
value: node.value,
202-
placeholder: node.placeholder,
203-
rect: node.rect,
204-
enabled: node.enabled,
205-
focused: node.focused,
206-
selected: node.selected,
207-
hittable: node.parentIndex == nil
208-
? false
209-
: clippedHittability(
210-
source: node.hittable,
211-
enabled: node.enabled,
212-
clippedFrame: decision.effectiveFrame,
213-
viewport: viewport
214-
),
215-
depth: outDepth,
216-
parentIndex: keptIndex,
217-
hiddenContentAbove: node.hiddenContentAbove,
218-
hiddenContentBelow: node.hiddenContentBelow,
219-
actions: node.actions
220-
),
205+
raw: projected,
221206
effectiveRect: SnapshotGeometry.snapshotRect(
222207
from: decision.effectiveFrame,
223208
reportedFrame: rect

‎apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotVisibilityFoldProjection.swift‎

Lines changed: 1 addition & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -73,24 +73,7 @@ extension SnapshotVisibilityFold {
7373
let node = presentationNode.raw
7474
guard let hint = hints[node.index] else { return presentationNode }
7575
return SnapshotPresentationNode(
76-
raw: RawAXNode(
77-
index: node.index,
78-
type: node.type,
79-
label: node.label,
80-
identifier: node.identifier,
81-
value: node.value,
82-
placeholder: node.placeholder,
83-
rect: node.rect,
84-
enabled: node.enabled,
85-
focused: node.focused,
86-
selected: node.selected,
87-
hittable: node.hittable,
88-
depth: node.depth,
89-
parentIndex: node.parentIndex,
90-
hiddenContentAbove: node.hiddenContentAbove == true || hint.above ? true : nil,
91-
hiddenContentBelow: node.hiddenContentBelow == true || hint.below ? true : nil,
92-
actions: node.actions
93-
),
76+
raw: node.addingHiddenContentHints(above: hint.above, below: hint.below),
9477
effectiveRect: presentationNode.effectiveRect
9578
)
9679
}
Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
import AgentDeviceSnapshotPresentation
2+
import XCTest
3+
4+
final class SnapshotPresentationProjectionTests: XCTestCase {
5+
func testRawScopeReindexesWithoutDroppingCapturedAttributes() throws {
6+
let scoped = RawAXNode(
7+
index: 7, type: "TextField", label: "Account", identifier: "account-input", value: "saved",
8+
placeholder: "Enter account", rect: SnapshotRect(x: 12, y: 23, width: 34, height: 45),
9+
enabled: false, focused: true, selected: false, hittable: nil,
10+
depth: 3, parentIndex: 6, hiddenContentAbove: false, hiddenContentBelow: true,
11+
actions: ["Clear", "Inspect"]
12+
)
13+
let child = RawAXNode(
14+
index: 8, type: "Button", label: "Clear", identifier: nil, value: nil,
15+
rect: SnapshotRect(x: 13, y: 24, width: 10, height: 11),
16+
enabled: true, focused: nil, selected: nil, hittable: false,
17+
depth: 4, parentIndex: 7, hiddenContentAbove: nil, hiddenContentBelow: nil
18+
)
19+
let options = PresentationOptions(interactiveOnly: false, depth: nil, scope: "Account", raw: true)
20+
let result = SnapshotPresentation.presentRaw(
21+
SnapshotAcquisition(
22+
hint: SnapshotPresentation.captureHint(for: options), nodes: [scoped, child],
23+
truncated: false, effectiveDepth: nil, viewport: .missing(reason: .notProvided)
24+
),
25+
options: options
26+
)
27+
28+
XCTAssertEqual(result.nodes.count, 2)
29+
let presented = try XCTUnwrap(result.nodes.first)
30+
XCTAssertEqual(presented.index, 0)
31+
XCTAssertEqual(presented.depth, 0)
32+
XCTAssertNil(presented.parentIndex)
33+
XCTAssertEqual(result.nodes[1].index, 1)
34+
XCTAssertEqual(result.nodes[1].depth, 1)
35+
XCTAssertEqual(result.nodes[1].parentIndex, 0)
36+
XCTAssertEqual(presented.type, scoped.type)
37+
XCTAssertEqual(presented.label, scoped.label)
38+
XCTAssertEqual(presented.identifier, scoped.identifier)
39+
XCTAssertEqual(presented.value, scoped.value)
40+
XCTAssertEqual(presented.placeholder, scoped.placeholder)
41+
XCTAssertEqual(presented.rect, scoped.rect)
42+
XCTAssertEqual(presented.enabled, scoped.enabled)
43+
XCTAssertEqual(presented.focused, scoped.focused)
44+
XCTAssertEqual(presented.selected, scoped.selected)
45+
XCTAssertEqual(presented.hittable, scoped.hittable)
46+
XCTAssertEqual(presented.hiddenContentAbove, scoped.hiddenContentAbove)
47+
XCTAssertEqual(presented.hiddenContentBelow, scoped.hiddenContentBelow)
48+
XCTAssertEqual(presented.actions, scoped.actions)
49+
}
50+
}
Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
import AgentDeviceSnapshotPresentation
2+
import CoreGraphics
3+
import XCTest
4+
5+
final class SnapshotVisibilityFoldTests: XCTestCase {
6+
func testFoldPreservesCapturedAttributesWhileReparentingClippingAndMergingHints() throws {
7+
let root = node(0, "Application", y: 0, height: 100, depth: 0, parent: nil)
8+
let dropped = node(1, "Other", y: 120, height: 10, depth: 1, parent: 0)
9+
let scroll = RawAXNode(
10+
index: 2, type: "ScrollView", label: "Results", identifier: "search-results", value: "loaded",
11+
placeholder: "Search", rect: SnapshotRect(x: 80, y: 20, width: 40, height: 60),
12+
enabled: true, focused: false, selected: true, hittable: nil,
13+
depth: 2, parentIndex: 1, hiddenContentAbove: true, hiddenContentBelow: false,
14+
actions: ["Refresh", "Jump"]
15+
)
16+
17+
for (hiddenY, expectedBelow) in [(0.0, nil as Bool?), (90.0, true as Bool?)] {
18+
let hidden = node(3, "Button", y: hiddenY, height: 10, depth: 3, parent: 2)
19+
let folded = SnapshotVisibilityFold.fold(
20+
[root, dropped, scroll, hidden],
21+
viewport: .reported(box: CGRect(x: 0, y: 0, width: 100, height: 100)),
22+
interactiveOnly: false,
23+
policy: .cursorProjected
24+
)
25+
26+
XCTAssertEqual(folded.count, 2)
27+
let presentation = try XCTUnwrap(folded.last)
28+
let raw = presentation.raw
29+
XCTAssertEqual(raw.index, 1)
30+
XCTAssertEqual(raw.depth, 1)
31+
XCTAssertEqual(raw.parentIndex, 0)
32+
XCTAssertEqual(raw.hittable, true)
33+
XCTAssertEqual(folded[0].raw.hittable, false)
34+
XCTAssertEqual(raw.hiddenContentAbove, true)
35+
XCTAssertEqual(raw.hiddenContentBelow, expectedBelow)
36+
XCTAssertEqual(presentation.effectiveRect, SnapshotRect(x: 80, y: 20, width: 20, height: 60))
37+
XCTAssertEqual(raw.rect, scroll.rect)
38+
XCTAssertEqual(raw.type, scroll.type)
39+
XCTAssertEqual(raw.label, scroll.label)
40+
XCTAssertEqual(raw.identifier, scroll.identifier)
41+
XCTAssertEqual(raw.value, scroll.value)
42+
XCTAssertEqual(raw.placeholder, scroll.placeholder)
43+
XCTAssertEqual(raw.enabled, scroll.enabled)
44+
XCTAssertEqual(raw.focused, scroll.focused)
45+
XCTAssertEqual(raw.selected, scroll.selected)
46+
XCTAssertEqual(raw.actions, scroll.actions)
47+
}
48+
}
49+
50+
private func node(
51+
_ index: Int, _ type: String, y: Double, height: Double, depth: Int, parent: Int?
52+
) -> RawAXNode {
53+
RawAXNode(
54+
index: index, type: type, label: nil, identifier: nil, value: nil,
55+
rect: SnapshotRect(x: 80, y: y, width: 20, height: height),
56+
enabled: true, focused: nil, selected: nil, hittable: false,
57+
depth: depth, parentIndex: parent, hiddenContentAbove: nil, hiddenContentBelow: nil
58+
)
59+
}
60+
}

‎packages/platform-android/src/__tests__/ui-hierarchy-field-metadata.test.ts‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,15 @@ test.each([false, true])(
2222

2323
test('selection offsets survive on a read-only selectable node (independent of editable)', () => {
2424
const tree = parseUiHierarchyTree(`<hierarchy><node class="android.widget.TextView"
25-
resource-id="field" text="Read-only text" bounds="[0,0][200,100]" visible-to-user="true"
25+
resource-id="field" text="Read-only text" content-desc="Read-only announcement"
26+
hint="Label prompt" bounds="[0,0][200,100]" visible-to-user="true"
2627
editable="false" selection-start="2" selection-end="5" /></hierarchy>`);
2728
const { nodes } = buildUiHierarchySnapshot(tree, undefined, { raw: true });
2829
expect(nodes.find((node) => node.identifier === 'field')).toMatchObject({
30+
label: 'Read-only text',
31+
value: 'Read-only text',
32+
contentDescription: 'Read-only announcement',
33+
placeholder: 'Label prompt',
2934
editable: false,
3035
selectionStart: 2,
3136
selectionEnd: 5,
@@ -44,4 +49,11 @@ test('missing native field metadata remains unknown instead of becoming false or
4449
expect(field?.hintShowing).toBeUndefined();
4550
expect(field?.selectionStart).toBeUndefined();
4651
expect(field?.selectionEnd).toBeUndefined();
52+
const serialized = JSON.parse(JSON.stringify(field));
53+
expect(serialized).not.toHaveProperty('value');
54+
expect(serialized).not.toHaveProperty('editable');
55+
expect(serialized).not.toHaveProperty('password');
56+
expect(serialized).not.toHaveProperty('hintShowing');
57+
expect(serialized).not.toHaveProperty('selectionStart');
58+
expect(serialized).not.toHaveProperty('selectionEnd');
4759
});

‎packages/platform-android/src/__tests__/ui-hierarchy-inclusion.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ function node(overrides: Partial<AndroidNode>): AndroidNode {
5656
type: 'android.widget.Button',
5757
label: null,
5858
value: null,
59+
passiveFacts: {},
5960
identifier: null,
6061
packageName: null,
6162
rect: { x: 0, y: 0, width: 20, height: 20 },

‎packages/platform-android/src/ui-hierarchy-builder.ts‎

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -347,22 +347,11 @@ function createAndroidRawSnapshotNode(
347347
type: node.type ?? undefined,
348348
label: node.label ?? undefined,
349349
value: node.value ?? undefined,
350-
contentDescription: node.contentDescription,
350+
...node.passiveFacts,
351351
identifier: node.identifier ?? undefined,
352352
bundleId: node.packageName ?? undefined,
353353
rect: node.rect,
354-
enabled: node.enabled,
355354
focused: node.focused,
356-
selected: node.selected,
357-
heading: node.heading,
358-
roleDescription: node.roleDescription,
359-
checked: node.checked,
360-
editable: node.editable,
361-
password: node.password,
362-
hintShowing: node.hintShowing,
363-
placeholder: node.placeholder,
364-
selectionStart: node.selectionStart,
365-
selectionEnd: node.selectionEnd,
366355
visibleToUser: node.visibleToUser,
367356
hittable: isAgentTarget(node) || undefined,
368357
depth: compactedAndroidNodeDepth(state.nodes, parentIndex),

‎packages/platform-android/src/ui-hierarchy-node.ts‎

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,21 @@
1-
import type { Rect } from '@agent-device/kernel/snapshot';
1+
import type { RawSnapshotNode, Rect } from '@agent-device/kernel/snapshot';
2+
3+
/** Normalized attributes carried unchanged into the public node, independent of presentation. */
4+
type AndroidPassiveNodeFacts = Pick<
5+
RawSnapshotNode,
6+
| 'contentDescription'
7+
| 'enabled'
8+
| 'selected'
9+
| 'heading'
10+
| 'roleDescription'
11+
| 'checked'
12+
| 'editable'
13+
| 'password'
14+
| 'hintShowing'
15+
| 'placeholder'
16+
| 'selectionStart'
17+
| 'selectionEnd'
18+
>;
219

320
/**
421
* One normalized Android accessibility node presented to the snapshot engine. Helper/API-specific
@@ -8,24 +25,12 @@ export type AndroidUiHierarchy = {
825
type: string | null;
926
label: string | null;
1027
value: string | null;
11-
/** The content description when the label is the text rather than it; see `RawSnapshotNode`. */
12-
contentDescription?: string;
28+
passiveFacts: AndroidPassiveNodeFacts;
1329
identifier: string | null;
1430
packageName: string | null;
1531
rect?: Rect;
16-
enabled?: boolean;
1732
visibleToUser?: boolean;
1833
focused?: boolean;
19-
selected?: boolean;
20-
heading?: boolean;
21-
roleDescription?: string;
22-
checked?: boolean;
23-
editable?: boolean;
24-
password?: boolean;
25-
hintShowing?: boolean;
26-
placeholder?: string;
27-
selectionStart?: number;
28-
selectionEnd?: number;
2934
// Two independent facts, never collapsed, and never undefined: the helper omits false attributes
3035
// while stock UiAutomator writes them out, so reading an absent attribute as a value gave two
3136
// encodings of one control opposite answers.

0 commit comments

Comments
 (0)