Skip to content

Commit dbae81f

Browse files
committed
feat(ios): publish the presented surface on every iOS capture (#3565)
Have both iOS capture routes answer the fact beside the tree they return. The engine classifies the acquired tree once and publishes the answer on the acquisition's publication, so it travels through the runtime projection and reaches a raw capture, an interactive capture, a depth-cut capture, and a scoped capture of one screen as the same value. Completeness comes from the producer's own truncation statement and is never inferred from the tree: a producer that reports nothing about truncation has stated nothing, and the classifier may not read a silent field as "complete". The XCTest runner classifies its pre-presentation acquisition and stamps the answer on the payload it returns, which is the only reading a runner-declared capture can carry — that tier publishes no UIKit class names, so its honest answer is `presented` or `unknown` and never `absent`.
1 parent c8e86bd commit dbae81f

9 files changed

Lines changed: 322 additions & 14 deletions

File tree

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -86,19 +86,29 @@ struct SnapshotBackendCapture {
8686
/// tree tier reads the keyboard, so only that tier has one; the daemon reads a missing band as "this
8787
/// producer could not measure", which is exactly what the query-sweep and private-AX tiers did.
8888
var keyboardBand: KeyboardBandFactPayload? = nil
89+
/// Whether this capture read a modal presentation over the app (#3565), classified on the
90+
/// acquisition this capture presents. A tier that classified nothing carries nothing, and the host
91+
/// reads that as no evidence rather than as absence.
92+
var presentedSurface: PresentedSurface? = nil
8993
}
9094

9195
extension RunnerTests {
96+
/// Assembles what one tier answered, from the tree it presented and the acquisition it read. The
97+
/// presented-surface fact is classified on the acquisition rather than the presentation, because
98+
/// interactive filtering, depth and scope are all cuts into what the tier actually saw (#3565): a
99+
/// capture of a presented screen that kept four nodes still read the presentation.
92100
static func makeSnapshotBackendCapture(
93-
from result: AgentDeviceSnapshotPresentation.SnapshotPresentationResult
101+
from result: AgentDeviceSnapshotPresentation.SnapshotPresentationResult,
102+
acquired acquisition: SnapshotAcquisition
94103
) -> SnapshotBackendCapture {
95104
SnapshotBackendCapture(
96105
payload: DataPayload(nodes: result.nodes, truncated: result.truncated),
97106
effectiveDepth: result.effectiveDepth,
98107
customActions: result.customActions,
99108
qualityPayload: result.qualityNodes.map {
100109
DataPayload(nodes: $0, truncated: result.truncated)
101-
}
110+
},
111+
presentedSurface: SnapshotPresentedSurface.classify(acquisition)
102112
)
103113
}
104114

@@ -576,7 +586,7 @@ extension RunnerTests {
576586
acquired: normalizedAcquisition.hint.projection
577587
)
578588
}
579-
return Self.makeSnapshotBackendCapture(from: result)
589+
return Self.makeSnapshotBackendCapture(from: result, acquired: normalizedAcquisition)
580590
}
581591
} catch let failure as SnapshotPresentationFailure {
582592
return SnapshotBackendAttempt(
@@ -754,6 +764,7 @@ extension RunnerTests {
754764
return SnapshotQualityPayload(nodes: nodes, truncated: quality.truncated == true)
755765
},
756766
snapshotQuality: quality,
767+
presentedSurface: capture.presentedSurface,
757768
keyboard: capture.keyboardBand,
758769
runnerFatal: payload.runnerFatal,
759770
runnerFatalReason: payload.runnerFatalReason

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift‎

Lines changed: 77 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -269,24 +269,91 @@ extension RunnerTests {
269269
XCTAssertNil(payload.qualityPayload?.scope)
270270
}
271271

272+
/// The classification has to happen on the acquisition a tier read, not on the presentation the tier
273+
/// returns, and survive onto the response envelope. This drives the production assembly function with a
274+
/// tree that keeps only what the runner's own AX tier publishes — no UIKit class names, an element type
275+
/// the runner's name table cannot spell — and asks for a capture that dropped every node the acquisition
276+
/// held. Deleting the classification, or reading the presented tree instead of the acquired one, fails
277+
/// it (#3565).
278+
func testSnapshotCaptureAssemblyClassifiesTheAcquisitionItPresented() throws {
279+
let options = PresentationOptions(interactiveOnly: true, depth: nil, scope: nil, raw: false)
280+
let acquisition = SnapshotAcquisition(
281+
hint: SnapshotPresentation.captureHint(for: options),
282+
nodes: [
283+
planTestRawNode(index: 0, type: "Application", parentIndex: nil),
284+
planTestRawNode(index: 1, type: "Window", parentIndex: 0),
285+
planTestRawNode(index: 2, type: "Element(7)", parentIndex: 1),
286+
],
287+
truncated: false,
288+
effectiveDepth: nil,
289+
viewport: .reported(box: CGRect(x: 0, y: 0, width: 402, height: 874))
290+
)
291+
let presented = try SnapshotPresentation.presentRegular(acquisition, options: options)
292+
293+
let capture = Self.makeSnapshotBackendCapture(from: presented, acquired: acquisition)
294+
XCTAssertEqual(capture.presentedSurface, .presented, "positive evidence below the presented cut")
295+
296+
// The same presentation assembled against an acquisition that holds no evidence must answer from
297+
// the acquisition. If the classifier ever read the presented tree instead, this pair collapses to
298+
// the same answer and the assertion above stops proving anything.
299+
let cleared = acquisition.replacingNodes([
300+
planTestRawNode(index: 0, type: "Application", parentIndex: nil)
301+
])
302+
XCTAssertEqual(
303+
Self.makeSnapshotBackendCapture(from: presented, acquired: cleared).presentedSurface,
304+
.unknown,
305+
"the fact follows the acquisition, not the presentation it produced"
306+
)
307+
308+
let payload = stampedSnapshotPayload(capture, backend: .recursiveTree, state: .healthy, reason: nil)
309+
XCTAssertEqual(payload.presentedSurface, .presented)
310+
// A tier that assembled no fact publishes nothing: silence is how the host reads "this producer
311+
// published no fact", and it must not be reachable by a default.
312+
XCTAssertNil(
313+
stampedSnapshotPayload(
314+
SnapshotBackendCapture(payload: capture.payload, effectiveDepth: nil),
315+
backend: .recursiveTree,
316+
state: .healthy,
317+
reason: nil
318+
).presentedSurface
319+
)
320+
}
321+
322+
private func planTestRawNode(index: Int, type: String, parentIndex: Int?) -> RawAXNode {
323+
RawAXNode(
324+
index: index,
325+
type: type,
326+
label: nil,
327+
identifier: nil,
328+
value: nil,
329+
rect: SnapshotRect(x: 0, y: 0, width: 10, height: 10),
330+
enabled: true,
331+
focused: nil,
332+
selected: nil,
333+
hittable: nil,
334+
depth: parentIndex == nil ? 0 : 1,
335+
parentIndex: parentIndex,
336+
hiddenContentAbove: nil,
337+
hiddenContentBelow: nil
338+
)
339+
}
340+
272341
func testDirectPresentationDoesNotClaimPlanTiming() {
273342
let options = PresentationOptions(
274343
interactiveOnly: false,
275344
depth: nil,
276345
scope: nil,
277346
raw: true
278347
)
279-
let result = SnapshotPresentation.presentRaw(
280-
SnapshotAcquisition(
281-
hint: SnapshotPresentation.captureHint(for: options),
282-
nodes: [],
283-
truncated: false,
284-
effectiveDepth: nil,
285-
viewport: .reported(box: CGRect(x: 0, y: 0, width: 402, height: 874))
286-
),
287-
options: options
348+
let acquisition = SnapshotAcquisition(
349+
hint: SnapshotPresentation.captureHint(for: options),
350+
nodes: [],
351+
truncated: false,
352+
effectiveDepth: nil,
353+
viewport: .reported(box: CGRect(x: 0, y: 0, width: 402, height: 874))
288354
)
289-
let capture = Self.makeSnapshotBackendCapture(from: result)
355+
let result = SnapshotPresentation.presentRaw(acquisition, options: options)
356+
let capture = Self.makeSnapshotBackendCapture(from: result, acquired: acquisition)
290357

291358
let payload = stampedSnapshotPayload(
292359
capture,

‎packages/capture-kit/src/ios-snapshot-engine/engine.test.ts‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -397,6 +397,63 @@ function acquisition(
397397
};
398398
}
399399

400+
test("the published presented-surface fact is the acquisition's, for every projection", () => {
401+
const requests = [
402+
createIosSnapshotRequest(),
403+
createIosSnapshotRequest({ interactiveOnly: true }),
404+
createIosSnapshotRequest({ raw: true, depth: 1 }),
405+
createIosSnapshotRequest({ scope: 'Automation lab' }),
406+
];
407+
const answers = requests.map(
408+
(request) =>
409+
publishIosSnapshot(acquiredInput(request, presentedTreeNodes()), request).presentedSurface,
410+
);
411+
412+
// One screen answers one value: the fact is read on the acquired tree, so a projection that dropped
413+
// the presentation from what it shows still reports the presentation the producer saw.
414+
assert.deepEqual(answers, ['presented', 'presented', 'presented', 'presented']);
415+
});
416+
417+
test('an acquisition that states no completeness cannot be read as un-presented', () => {
418+
const request = createIosSnapshotRequest();
419+
const unstated: IosSnapshotAcquisition = {
420+
...acquisition(request, ordinaryTreeNodes()),
421+
truncated: undefined,
422+
};
423+
424+
assert.equal(
425+
publishIosSnapshot({ stage: 'acquired', acquisition: unstated }, request).presentedSurface,
426+
'unknown',
427+
);
428+
assert.equal(
429+
publishIosSnapshot(acquiredInput(request, ordinaryTreeNodes()), request).presentedSurface,
430+
'absent',
431+
);
432+
});
433+
434+
test('a runner-declared publication states no presented-surface fact of its own', () => {
435+
const request = createIosSnapshotRequest();
436+
const input: IosSnapshotInput = {
437+
stage: 'presented',
438+
presentation: {
439+
producer: 'apple-runner',
440+
intent: 'full',
441+
payload: { nodes: presentedTreeNodes(), truncated: false },
442+
},
443+
validation: validationFacts(request),
444+
};
445+
446+
// The runner classifies its own pre-presentation acquisition and publishes the reading beside the
447+
// payload; re-deriving one from the presented tree here would be a second, weaker answer. The same tree
448+
// answers at the acquired stage, so this pins the stage gate and not a classifier that reads neither:
449+
// a publication path that stopped classifying at all would fail the second assertion, not this one.
450+
assert.equal(publishIosSnapshot(input, request).presentedSurface, undefined);
451+
assert.equal(
452+
publishIosSnapshot(acquiredInput(request, presentedTreeNodes()), request).presentedSurface,
453+
'presented',
454+
);
455+
});
456+
400457
function acquiredInput(request: IosSnapshotRequest, nodes: RawSnapshotNode[]): IosSnapshotInput {
401458
return { stage: 'acquired', acquisition: acquisition(request, nodes) };
402459
}
@@ -421,6 +478,49 @@ function nestedNodes(): RawSnapshotNode[] {
421478
];
422479
}
423480

481+
/** A UIKit-shaped tree over which a presentation is up: a second container carrying a dimming view. */
482+
function presentedTreeNodes(): RawSnapshotNode[] {
483+
return [
484+
appNode(0, 'UIApplication', 'Application', viewport, undefined),
485+
appNode(1, 'UIWindow', 'Window', viewport, 0),
486+
appNode(2, 'UITransitionView', 'Other', viewport, 1),
487+
node(3, 'StaticText', 'Automation lab', { x: 10, y: 20, width: 200, height: 20 }, 2, 3),
488+
appNode(4, 'UITransitionView', 'Other', viewport, 1),
489+
{ ...appNode(5, 'UIDimmingView', 'Other', viewport, 4), userInteractionEnabled: true },
490+
node(6, 'StaticText', 'Automation sheet', { x: 10, y: 120, width: 200, height: 20 }, 4, 4),
491+
];
492+
}
493+
494+
/** A UIKit-shaped tree with nothing presenting: one window, its own single container. */
495+
function ordinaryTreeNodes(): RawSnapshotNode[] {
496+
return [
497+
appNode(0, 'UIApplication', 'Application', viewport, undefined),
498+
appNode(1, 'UIWindow', 'Window', viewport, 0),
499+
appNode(2, 'UITransitionView', 'Other', viewport, 1),
500+
appNode(
501+
3,
502+
'RCTParagraphComponentView',
503+
'StaticText',
504+
{ x: 10, y: 20, width: 200, height: 20 },
505+
2,
506+
),
507+
];
508+
}
509+
510+
/** A node shaped like the simulator AX bridge reports one: UIKit class name in `role`. */
511+
function appNode(
512+
index: number,
513+
role: string,
514+
type: string,
515+
rect: Rect,
516+
parentIndex: number | undefined,
517+
): RawSnapshotNode {
518+
return {
519+
...node(index, type, undefined, rect, parentIndex, parentIndex === undefined ? 0 : 1),
520+
role,
521+
};
522+
}
523+
424524
function scopedNodes(): RawSnapshotNode[] {
425525
return [
426526
node(0, 'Application', 'App', viewport),

‎packages/capture-kit/src/ios-snapshot-engine/engine.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,29 @@ export function publishIosSnapshot(
5454
...(presentation.validatedViewport
5555
? { validatedViewport: presentation.validatedViewport }
5656
: {}),
57+
...(input.stage === 'acquired'
58+
? {
59+
presentedSurface: collectPresentedSurface(
60+
input.acquisition.nodes,
61+
acquisitionCompleteness(input.acquisition),
62+
),
63+
}
64+
: {}),
5765
};
5866
}
5967

68+
/**
69+
* What an acquisition claims about its own completeness, from the producer's truncation statement and
70+
* never from the tree: a producer that reports nothing about truncation has stated nothing, and a
71+
* presented-surface classification may not read a silent field as "complete" (#2188 invariant 5).
72+
*/
73+
function acquisitionCompleteness(
74+
acquisition: IosSnapshotAcquisition,
75+
): PresentedSurfaceCompleteness {
76+
if (acquisition.truncated === undefined) return 'unstated';
77+
return acquisition.truncated ? 'capped' : 'complete';
78+
}
79+
6080
export function presentIosSnapshot(
6181
input: IosSnapshotInput,
6282
request: IosSnapshotRequest,
Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
import assert from 'node:assert/strict';
2+
import { test } from 'vitest';
3+
import { presentIosSnapshotAcquisition } from '../ios-snapshot-runtime.ts';
4+
import { createIosSnapshotRequest, deriveIosCaptureHint } from '../../ios-snapshot-planning.ts';
5+
import type { IosSnapshotAcquisition } from '@agent-device/contracts/ios-snapshot';
6+
import type { SnapshotRuntimeAcquiredResult } from '@agent-device/contracts/interactor-types';
7+
import type { RawSnapshotNode } from '@agent-device/kernel/snapshot';
8+
9+
// #3565: this seam is where an acquisition becomes the snapshot result the daemon, the Linux provider
10+
// and the macOS desktop consume, so it is where the presented-surface fact enters `SnapshotResult`.
11+
// Nothing else reaches it: the engine's own tests stop at the publication it builds, and the carriage
12+
// tests enter above it through a backend, so a projection that dropped the field here would leave a
13+
// settle confirmation reading silence as "no producer spoke".
14+
15+
const SCREEN = { x: 0, y: 0, width: 390, height: 844 };
16+
17+
/** One window, one presentation container, and the dimming view that container owns. */
18+
const PRESENTED_NODES: readonly RawSnapshotNode[] = [
19+
{ index: 0, type: 'Application', role: 'UIApplication', rect: SCREEN },
20+
{ index: 1, parentIndex: 0, type: 'Window', role: 'UIWindow', rect: SCREEN },
21+
{ index: 2, parentIndex: 1, type: 'Other', role: 'UITransitionView', rect: SCREEN },
22+
{
23+
index: 3,
24+
parentIndex: 2,
25+
type: 'Other',
26+
role: 'UIDimmingView',
27+
rect: SCREEN,
28+
userInteractionEnabled: true,
29+
},
30+
];
31+
32+
/** The same tree as a producer that reports no UIKit class names publishes it. */
33+
const CLASS_BLIND_NODES: readonly RawSnapshotNode[] = PRESENTED_NODES.map(
34+
({ role: _role, ...node }) => node,
35+
);
36+
37+
/**
38+
* Both cases state that nothing was capped, so the answers below differ by class evidence alone and
39+
* not by a completeness statement the producer never made.
40+
*/
41+
function acquiring(nodes: readonly RawSnapshotNode[]): SnapshotRuntimeAcquiredResult {
42+
const hint = deriveIosCaptureHint(createIosSnapshotRequest({}));
43+
const acquisition: IosSnapshotAcquisition = {
44+
producer: 'simulator-ax-bridge',
45+
intent: 'full',
46+
hint: { ...hint, acquisitionIntent: 'full' },
47+
nodes,
48+
truncated: false,
49+
viewport: { kind: 'reported', rect: SCREEN },
50+
lineage: { targetId: 'simulator-1', generation: 'generation-1' },
51+
residue: [],
52+
};
53+
return { stage: 'acquired', acquisition };
54+
}
55+
56+
test('the snapshot result carries the presented-surface fact of the tree it projects (#3565)', () => {
57+
const presented = presentIosSnapshotAcquisition(acquiring(PRESENTED_NODES));
58+
59+
assert.equal(presented.presentedSurface, 'presented');
60+
});
61+
62+
// A capture whose producer reports no class names can show no window structure, so it cannot earn
63+
// `absent`. It answers `unknown` rather than nothing, which is what lets a consumer tell "this
64+
// producer could not read it" apart from "no producer classified this capture at all".
65+
test('a class-blind acquisition answers unknown rather than silence (#3565)', () => {
66+
const presented = presentIosSnapshotAcquisition(acquiring(CLASS_BLIND_NODES));
67+
68+
assert.equal(presented.presentedSurface, 'unknown');
69+
});

‎packages/capture-kit/src/snapshot/ios-snapshot-runtime.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ export function presentIosSnapshotAcquisition(
5454
? {}
5555
: { truncated: acquired.acquisition.truncated }),
5656
...(validatedViewport ? { viewport: validatedViewport } : {}),
57+
...(presentation.presentedSurface ? { presentedSurface: presentation.presentedSurface } : {}),
5758
...snapshotWarnings(acquired.acquisition.residue),
5859
};
5960
} catch (error) {

‎packages/contracts/src/ios-snapshot.ts‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
1-
import type { RawSnapshotNode, Rect, SnapshotNode } from '@agent-device/kernel/snapshot';
1+
import type {
2+
PresentedSurface,
3+
RawSnapshotNode,
4+
Rect,
5+
SnapshotNode,
6+
} from '@agent-device/kernel/snapshot';
27

38
export type IosSnapshotProducer =
49
| 'apple-runner'
@@ -224,6 +229,14 @@ export type IosSnapshotPublication = Readonly<{
224229
* that measured against it. A raw projection validates no box and carries none.
225230
*/
226231
validatedViewport?: Rect;
232+
/**
233+
* Whether this capture read a modal presentation over the app (#3565), classified on the acquired
234+
* tree so every projection of one screen — interactive, full, depth-cut, scoped — answers the same
235+
* value. Only an acquisition the engine itself classified carries it: a runner-declared capture
236+
* arrives already presented, and the runner's own reading of its pre-presentation acquisition
237+
* travels on the payload beside it.
238+
*/
239+
presentedSurface?: PresentedSurface;
227240
}>;
228241

229242
export type IosSnapshotEngine = Readonly<{

0 commit comments

Comments
 (0)