Skip to content

Commit c06d8ec

Browse files
committed
feat(ios): publish the presented surface on every iOS capture (#3565)
Have both iOS capture routes answer the fact beside the tree they return. The engine classifies the acquired tree once and publishes the answer on the acquisition's publication, so it travels through the runtime projection and reaches a raw capture, an interactive capture, a depth-cut capture, and a scoped capture of one screen as the same value. Completeness comes from the producer's own truncation statement and is never inferred from the tree: a producer that reports nothing about truncation has stated nothing, and the classifier may not read a silent field as "complete". The XCTest runner classifies its pre-presentation acquisition and stamps the answer on the payload it returns, which is the only reading a runner-declared capture can carry — that tier publishes no UIKit class names, so its honest answer is `presented` or `unknown` and never `absent`.
1 parent 777d9b7 commit c06d8ec

6 files changed

Lines changed: 515 additions & 13 deletions

File tree

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -86,19 +86,29 @@ struct SnapshotBackendCapture {
8686
/// tree tier reads the keyboard, so only that tier has one; the daemon reads a missing band as "this
8787
/// producer could not measure", which is exactly what the query-sweep and private-AX tiers did.
8888
var keyboardBand: KeyboardBandFactPayload? = nil
89+
/// Whether this capture read a modal presentation over the app (#3565), classified on the
90+
/// acquisition this capture presents. A tier that classified nothing carries nothing, and the host
91+
/// reads that as no evidence rather than as absence.
92+
var presentedSurface: PresentedSurface? = nil
8993
}
9094

9195
extension RunnerTests {
96+
/// Assembles what one tier answered, from the tree it presented and the acquisition it read. The
97+
/// presented-surface fact is classified on the acquisition rather than the presentation, because
98+
/// interactive filtering, depth and scope are all cuts into what the tier actually saw (#3565): a
99+
/// capture of a presented screen that kept four nodes still read the presentation.
92100
static func makeSnapshotBackendCapture(
93-
from result: AgentDeviceSnapshotPresentation.SnapshotPresentationResult
101+
from result: AgentDeviceSnapshotPresentation.SnapshotPresentationResult,
102+
acquired acquisition: SnapshotAcquisition
94103
) -> SnapshotBackendCapture {
95104
SnapshotBackendCapture(
96105
payload: DataPayload(nodes: result.nodes, truncated: result.truncated),
97106
effectiveDepth: result.effectiveDepth,
98107
customActions: result.customActions,
99108
qualityPayload: result.qualityNodes.map {
100109
DataPayload(nodes: $0, truncated: result.truncated)
101-
}
110+
},
111+
presentedSurface: SnapshotPresentedSurface.classify(acquisition)
102112
)
103113
}
104114

@@ -576,7 +586,7 @@ extension RunnerTests {
576586
acquired: normalizedAcquisition.hint.projection
577587
)
578588
}
579-
return Self.makeSnapshotBackendCapture(from: result)
589+
return Self.makeSnapshotBackendCapture(from: result, acquired: normalizedAcquisition)
580590
}
581591
} catch let failure as SnapshotPresentationFailure {
582592
return SnapshotBackendAttempt(
@@ -754,6 +764,7 @@ extension RunnerTests {
754764
return SnapshotQualityPayload(nodes: nodes, truncated: quality.truncated == true)
755765
},
756766
snapshotQuality: quality,
767+
presentedSurface: capture.presentedSurface,
757768
keyboard: capture.keyboardBand,
758769
runnerFatal: payload.runnerFatal,
759770
runnerFatalReason: payload.runnerFatalReason

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift‎

Lines changed: 85 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -269,24 +269,99 @@ extension RunnerTests {
269269
XCTAssertNil(payload.qualityPayload?.scope)
270270
}
271271

272+
/// The classification has to happen on the acquisition a tier read, not on the presentation the tier
273+
/// returns, and survive onto the response envelope. This drives the production assembly function with a
274+
/// tree that keeps only what the runner's own AX tier publishes — no UIKit class names, an element type
275+
/// the runner's name table cannot spell — and asks for a capture that dropped every node the acquisition
276+
/// held. Deleting the classification, or reading the presented tree instead of the acquired one, fails
277+
/// it (#3565).
278+
func testSnapshotCaptureAssemblyClassifiesTheAcquisitionItPresented() throws {
279+
let options = PresentationOptions(interactiveOnly: true, depth: nil, scope: nil, raw: false)
280+
let evidenceIndex = 2
281+
let acquisition = SnapshotAcquisition(
282+
hint: SnapshotPresentation.captureHint(for: options),
283+
nodes: [
284+
planTestRawNode(index: 0, type: "Application", parentIndex: nil),
285+
planTestRawNode(index: 1, type: "Window", parentIndex: 0),
286+
planTestRawNode(index: evidenceIndex, type: "Element(7)", parentIndex: 1),
287+
],
288+
truncated: false,
289+
effectiveDepth: nil,
290+
viewport: .reported(box: CGRect(x: 0, y: 0, width: 402, height: 874))
291+
)
292+
let presented = try SnapshotPresentation.presentRegular(acquisition, options: options)
293+
// The premise the assertion below turns on, pinned rather than assumed: this is the node the
294+
// evidence sits on, and the tier's own presentation does not carry it. Were it retained, the two
295+
// trees would agree and the next line would stop proving which one was read.
296+
XCTAssertFalse(
297+
presented.nodes.contains { $0.index == evidenceIndex },
298+
"the presentation has to drop the evidence node, or this test cannot tell the two trees apart"
299+
)
300+
301+
let capture = Self.makeSnapshotBackendCapture(from: presented, acquired: acquisition)
302+
XCTAssertEqual(capture.presentedSurface, .presented, "positive evidence below the presented cut")
303+
304+
// The same presentation assembled against an acquisition that holds no evidence must answer from
305+
// the acquisition. If the classifier ever read the presented tree instead, this pair collapses to
306+
// the same answer and the assertion above stops proving anything.
307+
let cleared = acquisition.replacingNodes([
308+
planTestRawNode(index: 0, type: "Application", parentIndex: nil)
309+
])
310+
XCTAssertEqual(
311+
Self.makeSnapshotBackendCapture(from: presented, acquired: cleared).presentedSurface,
312+
.unknown,
313+
"the fact follows the acquisition, not the presentation it produced"
314+
)
315+
316+
let payload = stampedSnapshotPayload(capture, backend: .recursiveTree, state: .healthy, reason: nil)
317+
XCTAssertEqual(payload.presentedSurface, .presented)
318+
// A tier that assembled no fact publishes nothing: silence is how the host reads "this producer
319+
// published no fact", and it must not be reachable by a default.
320+
XCTAssertNil(
321+
stampedSnapshotPayload(
322+
SnapshotBackendCapture(payload: capture.payload, effectiveDepth: nil),
323+
backend: .recursiveTree,
324+
state: .healthy,
325+
reason: nil
326+
).presentedSurface
327+
)
328+
}
329+
330+
private func planTestRawNode(index: Int, type: String, parentIndex: Int?) -> RawAXNode {
331+
RawAXNode(
332+
index: index,
333+
type: type,
334+
label: nil,
335+
identifier: nil,
336+
value: nil,
337+
rect: SnapshotRect(x: 0, y: 0, width: 10, height: 10),
338+
enabled: true,
339+
focused: nil,
340+
selected: nil,
341+
hittable: nil,
342+
depth: parentIndex == nil ? 0 : 1,
343+
parentIndex: parentIndex,
344+
hiddenContentAbove: nil,
345+
hiddenContentBelow: nil
346+
)
347+
}
348+
272349
func testDirectPresentationDoesNotClaimPlanTiming() {
273350
let options = PresentationOptions(
274351
interactiveOnly: false,
275352
depth: nil,
276353
scope: nil,
277354
raw: true
278355
)
279-
let result = SnapshotPresentation.presentRaw(
280-
SnapshotAcquisition(
281-
hint: SnapshotPresentation.captureHint(for: options),
282-
nodes: [],
283-
truncated: false,
284-
effectiveDepth: nil,
285-
viewport: .reported(box: CGRect(x: 0, y: 0, width: 402, height: 874))
286-
),
287-
options: options
356+
let acquisition = SnapshotAcquisition(
357+
hint: SnapshotPresentation.captureHint(for: options),
358+
nodes: [],
359+
truncated: false,
360+
effectiveDepth: nil,
361+
viewport: .reported(box: CGRect(x: 0, y: 0, width: 402, height: 874))
288362
)
289-
let capture = Self.makeSnapshotBackendCapture(from: result)
363+
let result = SnapshotPresentation.presentRaw(acquisition, options: options)
364+
let capture = Self.makeSnapshotBackendCapture(from: result, acquired: acquisition)
290365

291366
let payload = stampedSnapshotPayload(
292367
capture,
Lines changed: 236 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,236 @@
1+
import Foundation
2+
3+
/// What a capture's producer claims about its own completeness (#3565), stated by the caller rather
4+
/// than inferred from the tree: `complete` when the producer reports the capture is not capped, `capped`
5+
/// when it reports a cap, `unstated` when the producer observes nothing either way. A presented-surface
6+
/// classification may not read a silent field as "complete".
7+
public enum PresentedSurfaceCompleteness: Equatable {
8+
case complete
9+
case capped
10+
case unstated
11+
}
12+
13+
/// One node as the presented-surface rule reads it: the element type every iOS producer publishes, and
14+
/// the UIKit class name only some of them report. A `nil` `className` states a fact about the capture —
15+
/// its producer reports no class names at all, which is true of every XCTest runner tier — so the rule
16+
/// knows not to read window structure it was never given.
17+
public struct PresentedSurfaceNode {
18+
public let index: Int
19+
public let parentIndex: Int?
20+
public let elementType: String
21+
public let className: String?
22+
23+
public init(index: Int, parentIndex: Int?, elementType: String, className: String?) {
24+
self.index = index
25+
self.parentIndex = parentIndex
26+
self.elementType = elementType
27+
self.className = className
28+
}
29+
}
30+
31+
extension PresentedSurfaceNode {
32+
/// One acquired node. An accessibility-tree node carries no UIKit class name, which is what makes a
33+
/// runner capture unable to earn `absent` rather than merely undecided.
34+
public init(_ node: RawAXNode) {
35+
self.init(
36+
index: node.index,
37+
parentIndex: node.parentIndex,
38+
elementType: node.type,
39+
className: nil
40+
)
41+
}
42+
}
43+
44+
/// Reads a modal presentation out of a captured tree (#3565), before interactive filtering, compaction,
45+
/// depth, or scope, so every view of one screen answers one value.
46+
///
47+
/// This is the Swift half of a rule the TypeScript engine also runs (`collectPresentedSurface` in
48+
/// `packages/capture-kit/src/ios-snapshot-engine/engine.ts`). The golden table
49+
/// `contracts/fixtures/ios-presented-surface.json` pins the two against each other over the same tree
50+
/// shapes, and each side mirrors the vocabulary below by hand the way the system-surface host registry
51+
/// mirrors its own fixture.
52+
///
53+
/// UIKit appends each modal presentation as a later sibling of the container it presents over, directly
54+
/// under the window, and dims what is behind it with a dimming view that is a direct child of that
55+
/// container — the same shape the occlusion cut rests on. Because this rule decides only whether
56+
/// something is presented and not what the user can still reach, it deliberately does NOT read a
57+
/// dimming view's enablement: a sheet resting at an undimmed detent keeps a dimming view that takes no
58+
/// touches and is still presented.
59+
///
60+
/// `absent` is earned and never defaulted, because a consumer arms a dismissal confirmation on it and a
61+
/// wrong `absent` disarms that confirmation: the producer has to have stated the capture is complete,
62+
/// every node has to carry the class name this rule reads, each node has to be numbered once so the
63+
/// parent links this walk follows name the node they mean, and the one content window under the app root
64+
/// has to hold exactly one presentation container.
65+
public enum SnapshotPresentedSurface {
66+
/// The container UIKit appends per presentation, and the one it presents app content inside.
67+
public static let presentationContainerClass = "UITransitionView"
68+
69+
/// The dimming view a presentation dims the content behind it with. Only a presentation installs one,
70+
/// so one anywhere inside the app's own windows answers presented. Its position is not a fact this rule
71+
/// may lean on: a live capture of a sheet holds one as a direct child of the presenting container AND
72+
/// one two levels deep under the app's own container's drop shadow, so the rule never asks which
73+
/// container a dimming view sits in.
74+
public static let dimmingViewClass = "UIDimmingView"
75+
76+
/// The private root classes of a UIAlertController. An iPhone alert arrives in its own window with no
77+
/// dimming view anywhere in the capture, so these classes are the only evidence it holds.
78+
public static let alertRootClassPrefix = "_UIAlertController"
79+
80+
/// Windows UIKit hosts outside the app's own scene. They can carry none of the app's presentations, so
81+
/// neither the window nor anything under its subtree counts as evidence in either direction. This is a
82+
/// denylist of the hosts iOS is known to name: a window class it does not name is read as app content,
83+
/// which is what keeps an app hosting its scene window in a subclass readable at all. The cost is that
84+
/// an unnamed host carrying a dimming view of its own would read as presented; no such host has been
85+
/// observed, while custom scene window classes are routine.
86+
public static let systemWindowClasses: Set<String> = [
87+
"UITextEffectsWindow",
88+
"UIRemoteKeyboardWindow",
89+
]
90+
91+
/// Element types that name a presentation wherever they appear, keyed by the shared element-type
92+
/// numbering every iOS producer reads. A producer whose own name table stops short of the name
93+
/// publishes `Element(<number>)` for the same node, so both spellings count and the raw-value spelling
94+
/// is derived from these numbers rather than listed beside them.
95+
public static let presentingElementTypes: [String: Int] = [
96+
"Sheet": 5,
97+
"Drawer": 6,
98+
"Alert": 7,
99+
"Dialog": 8,
100+
]
101+
102+
/// The fact for one captured tree, where `completeness` is the producer's own claim and never a
103+
/// reading of the tree.
104+
public static func classify(
105+
nodes: [PresentedSurfaceNode],
106+
completeness: PresentedSurfaceCompleteness
107+
) -> PresentedSurface {
108+
let windowPositions = appWindowPositions(nodes)
109+
let examinable = positionsOutsideSystemWindows(nodes, windowPositions)
110+
// Positive evidence is evidence of what the producer did read, so a cap does not defeat it; absence
111+
// is what a cap defeats, along with any capture that cannot state its own window structure.
112+
if nodes.enumerated().contains(where: { examinable.contains($0.offset) && isPresentingEvidence($0.element) }) {
113+
return .presented
114+
}
115+
guard completeness == .complete else { return .unknown }
116+
guard !windowPositions.isEmpty, nodes.allSatisfy({ $0.className != nil }) else { return .unknown }
117+
// `absent` is a claim about every window the producer read, and window structure is read through
118+
// parent links, which cannot tell two nodes sharing a number apart. Positive evidence is already
119+
// reported above, keyed on the node it was read off rather than on its number, so the most a
120+
// collision can cost this rule is the answer it declines to make — never a false `absent`.
121+
guard Set(nodes.map(\.index)).count == nodes.count else { return .unknown }
122+
123+
let contentWindows = windowPositions.filter { !isSystemWindow(nodes[$0]) }
124+
guard contentWindows.count == 1 else { return .unknown }
125+
return presentationContainers(of: nodes[contentWindows[0]], in: nodes).count == 1 ? .absent : .unknown
126+
}
127+
128+
/// The fact for one acquisition as its producer holds it.
129+
public static func classify(_ acquisition: SnapshotAcquisition) -> PresentedSurface {
130+
classify(
131+
nodes: acquisition.nodes.map(PresentedSurfaceNode.init),
132+
completeness: acquisition.truncated ? .capped : .complete
133+
)
134+
}
135+
136+
private static func isPresentingEvidence(_ node: PresentedSurfaceNode) -> Bool {
137+
if node.className == dimmingViewClass { return true }
138+
if node.className?.hasPrefix(alertRootClassPrefix) == true { return true }
139+
return isPresentingElementType(node.elementType)
140+
}
141+
142+
private static func isPresentingElementType(_ elementType: String) -> Bool {
143+
if presentingElementTypes[elementType] != nil { return true }
144+
guard let rawValue = unnamedElementTypeRawValue(elementType) else { return false }
145+
return presentingElementTypes.values.contains(rawValue)
146+
}
147+
148+
/// The raw element type behind the `Element(<number>)` spelling a producer uses for a type its own
149+
/// name table does not cover. The number has to be bare digits: `Int` accepts a leading `+`, so a
150+
/// numeric parse would read `Element(+5)` as Sheet, and the rule's answers would then depend on how
151+
/// lenient one language's integer parser happens to be.
152+
private static func unnamedElementTypeRawValue(_ elementType: String) -> Int? {
153+
guard elementType.hasPrefix(unnamedElementTypePrefix), elementType.hasSuffix(")") else { return nil }
154+
let digits = elementType.dropFirst(unnamedElementTypePrefix.count).dropLast()
155+
guard !digits.isEmpty, digits.allSatisfy({ $0.isASCII && $0.isNumber }) else { return nil }
156+
return Int(digits)
157+
}
158+
159+
private static let unnamedElementTypePrefix = "Element("
160+
161+
private static func isSystemWindow(_ node: PresentedSurfaceNode) -> Bool {
162+
systemWindowClasses.contains(node.className ?? "")
163+
}
164+
165+
/// The app's windows: the nodes directly under its root, as the places in `nodes` they occupy. Both iOS
166+
/// acquisition tiers report the app root and then one node per window, so finding them needs no class
167+
/// facts — which is what lets a class-blind capture still be examined for the evidence it does report.
168+
private static func appWindowPositions(_ nodes: [PresentedSurfaceNode]) -> [Int] {
169+
let roots = nodes.enumerated().filter { $0.element.parentIndex == nil }.map(\.offset)
170+
guard roots.count == 1 else { return [] }
171+
let rootIndex = nodes[roots[0]].index
172+
return nodes.enumerated().filter { $0.element.parentIndex == rootIndex }.map(\.offset)
173+
}
174+
175+
/// The nodes to read evidence from: every node except the ones the walk reached from a window UIKit
176+
/// hosts outside the app's scene, keyed by the place in `nodes` each occupies rather than by its number.
177+
/// A walk has to reach the node it is standing on, and two nodes sharing a number cannot be told apart
178+
/// by the links that reach them; keying on the number would let a keyboard host's subtree swallow an
179+
/// app window's own evidence, which is a direct read the rule would then throw away. Keying on the
180+
/// occurrence keeps that read, and the numbering guard in `classify` is what stops such a tree from
181+
/// earning absence. A capture with no class facts cannot single any window out, which is the honest
182+
/// reading: nothing in it is excludable.
183+
private static func positionsOutsideSystemWindows(
184+
_ nodes: [PresentedSurfaceNode],
185+
_ windowPositions: [Int]
186+
) -> Set<Int> {
187+
var positionsByParentIndex: [Int: [Int]] = [:]
188+
for (position, node) in nodes.enumerated() {
189+
guard let parentIndex = node.parentIndex else { continue }
190+
positionsByParentIndex[parentIndex, default: []].append(position)
191+
}
192+
var examinable = Set(nodes.indices)
193+
for windowPosition in windowPositions where isSystemWindow(nodes[windowPosition]) {
194+
for occurrence in subtreePositions(windowPosition, in: nodes, byParentIndex: positionsByParentIndex) {
195+
examinable.remove(occurrence)
196+
}
197+
}
198+
return examinable
199+
}
200+
201+
/// A node and everything reachable from it through parent links, as the places in `nodes` they occupy,
202+
/// so reported depths are not needed. Visited is keyed on the occurrence and seeded with the root: a
203+
/// producer that repeats an index deeper in the tree closes a loop this walk would follow forever, and
204+
/// the classification runs outside the runner's bounded main-thread work, so no timeout would catch the
205+
/// spin. The golden table's cycle case is pinned on exactly that shape, in both languages.
206+
private static func subtreePositions(
207+
_ rootPosition: Int,
208+
in nodes: [PresentedSurfaceNode],
209+
byParentIndex positionsByParentIndex: [Int: [Int]]
210+
) -> [Int] {
211+
var collected: [Int] = [rootPosition]
212+
var visited: Set<Int> = [rootPosition]
213+
var walked = 0
214+
while walked < collected.count {
215+
for position in positionsByParentIndex[nodes[collected[walked]].index] ?? [] where visited.insert(position).inserted {
216+
collected.append(position)
217+
}
218+
walked += 1
219+
}
220+
return collected
221+
}
222+
223+
private static func presentationContainers(
224+
of window: PresentedSurfaceNode,
225+
in nodes: [PresentedSurfaceNode]
226+
) -> [PresentedSurfaceNode] {
227+
children(of: window, in: nodes).filter { $0.className == presentationContainerClass }
228+
}
229+
230+
private static func children(
231+
of parent: PresentedSurfaceNode,
232+
in nodes: [PresentedSurfaceNode]
233+
) -> [PresentedSurfaceNode] {
234+
nodes.filter { $0.parentIndex == parent.index }
235+
}
236+
}

0 commit comments

Comments
 (0)