Skip to content

Commit c2677fd

Browse files
authored
fix(ios): refuse snapshot-derived input without a screen origin (#4439)
* fix(ios): refuse unanchored fallback interaction geometry * test: colocate screen-origin regressions with owning modules * refactor(ios): keep viewport evidence selection in its owner * refactor(ios): keep origin refusal with capture evidence * fix(ios): retain provider snapshot origin authority * test(ios): prove reported viewport evidence survives presentation
1 parent 5781ba4 commit c2677fd

26 files changed

Lines changed: 839 additions & 29 deletions

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -274,7 +274,8 @@ extension RunnerTests {
274274
}
275275

276276
/// The app's reported viewport when XCTest can read it, else the bridge's own root frame declared
277-
/// `.derived`, which cannot anchor a rotation: rotated system surfaces then stay as reported (#2612).
277+
/// `.derived`, which cannot establish a screen origin or anchor a rotation. The wire retains this
278+
/// uncertainty, so local coordinates cannot authorize input (#4399).
278279
private func privateAXSnapshotViewport(
279280
app: XCUIApplication,
280281
bundleId: String?,

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -501,6 +501,7 @@ struct DataPayload: Codable {
501501
var qualityPayload: SnapshotQualityPayload? = nil
502502
var snapshotQuality: SnapshotQuality?
503503
var snapshotQualityNotes: [SnapshotQualityNote]? = nil
504+
var snapshotViewport: SnapshotViewportPayload?
504505
/// Set when the capture describes an in-place system surface, not the app itself (#2438).
505506
var systemSurface: SystemSurfaceProvenancePayload?
506507
/// Whether this capture read a modal presentation over the app's own content (#3565). The runner's
@@ -645,6 +646,30 @@ struct SnapshotQualityPayload: Codable {
645646
}
646647
}
647648

649+
/// Preserves the acquisition's viewport authority across the runner boundary.
650+
struct SnapshotViewportPayload: Codable {
651+
let kind: String
652+
let rect: SnapshotRect?
653+
let reason: String?
654+
655+
init(_ viewport: SnapshotViewport) {
656+
switch viewport {
657+
case .reported(let box, _):
658+
kind = "reported"
659+
rect = SnapshotRect(box.rect)
660+
reason = nil
661+
case .derived(let box):
662+
kind = "derived"
663+
rect = SnapshotRect(box.rect)
664+
reason = nil
665+
case .missing(let missingReason):
666+
kind = "missing"
667+
rect = nil
668+
reason = missingReason == .invalid ? "invalid" : "not-provided"
669+
}
670+
}
671+
}
672+
648673
/// A runner failure the session-loss retry may recover from by re-resolving the target.
649674
enum RetryableResponseFailure: Equatable {
650675
case targetAppUnavailable

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,11 @@ extension RunnerTests {
9898
acquired acquisition: SnapshotAcquisition
9999
) -> SnapshotBackendCapture {
100100
SnapshotBackendCapture(
101-
payload: DataPayload(nodes: result.nodes, truncated: result.truncated),
101+
payload: DataPayload(
102+
nodes: result.nodes,
103+
truncated: result.truncated,
104+
snapshotViewport: SnapshotViewportPayload(acquisition.viewport)
105+
),
102106
effectiveDepth: result.effectiveDepth,
103107
customActions: result.customActions,
104108
qualityPayload: result.qualityNodes.map {
@@ -625,6 +629,7 @@ extension RunnerTests {
625629
},
626630
snapshotQuality: quality,
627631
snapshotQualityNotes: capture.qualityNotes.isEmpty ? nil : capture.qualityNotes,
632+
snapshotViewport: payload.snapshotViewport,
628633
presentedSurface: capture.presentedSurface,
629634
keyboard: capture.keyboardBand,
630635
runnerFatal: payload.runnerFatal,

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -373,6 +373,42 @@ extension RunnerTests {
373373
XCTAssertNil(payload.snapshotQuality?.timing)
374374
}
375375

376+
func testSnapshotViewportAuthoritySurvivesOutcomeStamping() throws {
377+
let box = CGRect(x: 241, y: 224, width: 551, height: 813)
378+
let options = PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: true)
379+
let cases: [(SnapshotViewport, String, String?)] = [
380+
(.reported(box: box), "reported", nil),
381+
(.derived(box: box), "derived", nil),
382+
(.missing(reason: .notProvided), "missing", "not-provided"),
383+
]
384+
for (viewport, kind, reason) in cases {
385+
let acquisition = SnapshotAcquisition(
386+
hint: SnapshotPresentation.captureHint(for: options),
387+
nodes: [], truncated: false, effectiveDepth: nil, viewport: viewport
388+
)
389+
let presented = SnapshotPresentation.presentRaw(acquisition, options: options)
390+
let payload = stampedSnapshotPayload(
391+
Self.makeSnapshotBackendCapture(from: presented, acquired: acquisition),
392+
backend: .privateAX, state: .recovered, reason: nil
393+
)
394+
let encoded = try XCTUnwrap(
395+
JSONSerialization.jsonObject(with: JSONEncoder().encode(payload)) as? [String: Any]
396+
)
397+
let evidence = try XCTUnwrap(encoded["snapshotViewport"] as? [String: Any])
398+
XCTAssertEqual(evidence["kind"] as? String, kind)
399+
XCTAssertEqual(evidence["reason"] as? String, reason)
400+
if kind == "missing" {
401+
XCTAssertNil(evidence["rect"])
402+
} else {
403+
let rect = try XCTUnwrap(evidence["rect"] as? [String: Any])
404+
XCTAssertEqual(rect["x"] as? Double, 241)
405+
XCTAssertEqual(rect["y"] as? Double, 224)
406+
XCTAssertEqual(rect["width"] as? Double, 551)
407+
XCTAssertEqual(rect["height"] as? Double, 813)
408+
}
409+
}
410+
}
411+
376412
/// The raw plan is derived from what each backend can actually serve, not from a second
377413
/// hand-maintained list. Non-vacuity: flipping `querySweep.supportsRawProjection` to true adds it
378414
/// to the plan and fails the first two assertions — which is exactly the shape of #1797 D4, a

‎apple/runner/README.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,10 @@ Probe work that outlives its slice belongs to the capture that started it, so th
106106
Every bounded main-thread dispatch that outlives its slice counts as occupying the main thread.
107107
Thus, if a viewport read takes too long, the plan skips the sweep instead of queueing it.
108108

109+
Private-AX fallback keeps the viewport authority it acquired.
110+
When a penalty suppresses the XCTest viewport read, its root-derived box does not establish a screen origin.
111+
The payload carries that uncertainty without a new probe or a changed channel budget.
112+
109113
A failure that XCTest records ends `testCommand` when the main-thread block that recorded it returns, whatever `continueAfterFailure` says.
110114
Mute XCTest's AX-server rejection in both fetch wordings, so XCTest does not record it.
111115
Then the runner session continues and keeps its per-bundle penalty and depth memory.
@@ -114,6 +118,9 @@ Then the runner session continues and keeps its per-bundle penalty and depth mem
114118

115119
`XCUIElement.frame` and the app-anchored `XCUICoordinate` use screen space on a placed iPad window.
116120
`XCUIElementSnapshot.frame` uses window content space.
121+
Capture-plan responses retain `reported`, `derived`, or `missing` viewport evidence through outcome stamping.
122+
Only `reported` establishes the screen origin for capture-derived input.
123+
An unanchored capture remains readable, but its element mutations refuse with `window-origin-unknown` and `dispatched: no`.
117124
The text-entry probe owns one screen box, `ResolvedTextEntryInput.frameOnScreen`.
118125
A command reads the vector from the resolved window's box once, when it builds its target.
119126
The readiness retap reads the vector again on its own branch.

‎apple/runner/RUNNER_PROTOCOL.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,12 @@ Successful and failed responses use the same top-level envelope:
166166

167167
`data` is command-specific. Common fields include snapshot nodes, text lookup results, gesture timing, visibility metadata, and screenshot or recording output details.
168168

169+
Capture-plan snapshots carry `snapshotViewport`, the acquisition's declared viewport evidence.
170+
Its shape is `{kind: "reported" | "derived", rect: {x, y, width, height}}` or
171+
`{kind: "missing", reason: "not-provided" | "invalid"}`.
172+
`derived` and `missing` establish no screen origin.
173+
A root box used to validate presented nodes does not upgrade this evidence.
174+
169175
## Maintenance Rules
170176

171177
- Treat the TypeScript and Swift wire models as a single contract.

‎apple/snapshot-presentation/README.md‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,13 @@ In the runner, the viewport is a declared fact, not a rectangle.
113113
- `missing(reason)`.
114114

115115
Only `reported` carries an orientation, so only `reported` can anchor a rotation.
116+
Only `reported` establishes a screen origin for coordinates derived from this capture.
117+
The runner carries the acquisition's viewport evidence across the wire.
118+
The host can validate a locally presented tree against its root when acquisition evidence is missing.
119+
That validation box does not establish a screen origin or a public screen viewport.
120+
Capture-private viewport evidence survives compaction, session storage, and session cloning.
121+
Coordinate-derived mutations refuse `derived` or `missing` authority with `window-origin-unknown` and `dispatched: no`.
122+
Read-only observations remain available.
116123
If the viewport has no box, presentation skips the clip.
117124
If the viewport has no box, a node whose actionability depends on containment has no wire `hittable`.
118125
Disabled or degenerate nodes stay `false`.
@@ -158,7 +165,8 @@ The capture publishes reported geometry when no space can be named:
158165
- The platform did not report an interface orientation.
159166
- The app frame is too close to square to tell it from its own quarter turn.
160167

161-
Consumers fail open on that geometry.
168+
Consumers fail open on rotation they cannot name.
169+
An explicitly unknown screen origin does not authorize coordinate-derived input.
162170

163171
`SnapshotCoordinateSpace.swift` holds one rotation table that capture shares with synthesized dispatch.
164172
Both languages replay it against `contracts/fixtures/window-coordinate-space.json`.

‎docs/adr/0004-ios-snapshot-backend-strategy.md‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,13 @@ The fold cuts screens under a modal only where the producer states containment.
5151
**Geometry.** One published capture publishes one space: the app's orientation space.
5252
The producer settles it in one pass after acquisition, because a merged tree carries no axis that a downstream reader could use.
5353
A window declares its subtree's space from its own box. Other nodes inherit it from the window above them.
54-
If no space can be named, the capture publishes reported geometry, and consumers fail open.
54+
If no rotation can be named, the capture publishes reported geometry, and consumers fail open on rotation.
5555
A placed iPad window publishes screen space in points, because its viewport, its screenshot, and the digitizer use that space.
56+
The acquisition's `reported`, `derived`, or `missing` viewport evidence survives the runner boundary and capture transformations.
57+
Only `reported` establishes the screen origin for capture-derived input.
58+
A root-derived validation box preserves readable fallback trees but cannot upgrade unknown origin evidence.
59+
Element mutations, mutating find, Maestro selector geometry, and snapshot-derived gesture bounds refuse an unknown origin with `window-origin-unknown` and `dispatched: no` before input.
60+
Independent native queries and authored screen coordinates retain their own authorities.
5661

5762
**System surfaces.** `contracts/fixtures/ios-system-surface-hosts.json` is the closed registry of system hosts that present UI out of the app's process.
5863
Observe and operate a registered host in place. Never activate it. `open` refuses to launch it.

‎docs/adr/0011-interaction-guarantee-contract.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ Completed process history is in git history.
1414
- Prove every non-waived cell through a tagged contract scenario.
1515
- Delegation-on-error does not prove success-path parity.
1616
- On the native-ref path, before `tapTarget`/`fillTarget`, check offscreen, occlusion, and non-hittable against the node from the daemon snapshot.
17+
- Explicitly unanchored capture geometry cannot authorize input. Check its viewport authority before point resolution, visibility rescue, or native-ref dispatch. Mutating find, Maestro selector coordinates, and snapshot-derived gesture bounds use the same check.
1718
- The Apple runner does not resend a mutating command whose first send may have run.
1819
- Multiple mutating selector matches collapse only if they form one ancestor–descendant chain and every member resolves to the same actionable node. Otherwise, the mutation fails with `AMBIGUOUS_MATCH`. Maestro's explicit expected-point/non-hittable compatibility path stays separate.
1920
- Keep synthesized iOS gesture policy outside the element dispatch-path matrix.

‎packages/capture-kit/src/snapshot-state.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import type { CommandFlags } from '@agent-device/contracts/command';
22
import {
3+
copySnapshotViewportEvidence,
34
readSnapshotOcclusionContextEvidence,
45
readSnapshotTextInputGeometry,
56
attachSnapshotTextInputGeometry,
@@ -108,6 +109,7 @@ export function buildSnapshotState(
108109
);
109110
const textInputGeometry = readSnapshotTextInputGeometry(data.nodes);
110111
if (textInputGeometry) attachSnapshotTextInputGeometry(nodes, textInputGeometry);
112+
copySnapshotViewportEvidence(data.nodes, nodes);
111113
return {
112114
nodes,
113115
truncated: data?.truncated,

0 commit comments

Comments
 (0)