Skip to content

Commit b4f3e2f

Browse files
committed
feat(ios): publish the captured keyboard band as a producer-declared fact
A capture carried only a tree, so every acting path rebuilt the keyboard band from tree nodes and had to guess whether the geometry arrived normalized. The runner can measure the band directly and the portrait bridge tree already contains it, so the producer now states it and the guard measures a point against a rect. `SnapshotKeyboardBandFact` is optional on the snapshot result: a producer that publishes nothing is unmeasurable by construction, and the tree-derived rule stays for the query sweep, private-AX, Android, and provider paths. The tap guard prefers the fact when one arrived, and `resolveKeyboardTapOcclusion` keeps its public shape so the interaction-guarantee contract is unchanged. Closes #2660
1 parent a556e11 commit b4f3e2f

36 files changed

Lines changed: 1226 additions & 37 deletions

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift‎

Lines changed: 241 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import XCTest
2+
import AgentDeviceSnapshotPresentation
23

34
private enum KeyboardDismissObservationTiming {
45
static let timeout: TimeInterval = 2
@@ -30,6 +31,164 @@ enum RunnerKeyboardDismissMechanism: String {
3031
case dismissKey
3132
}
3233

34+
/// The band the visible keyboard owns, measured by the one producer that can ask the keyboard
35+
/// directly instead of rebuilding it from a captured tree.
36+
///
37+
/// `app.keyboards.firstMatch` answers in the app's own orientation space, which no tree tier can say
38+
/// about its own rects while the app is rotated (#2612, #2653). Publishing the measurement is what
39+
/// lets the daemon's tap guard answer with a point-in-rect test instead of guessing where a
40+
/// plausible-looking band would be (#2660).
41+
enum RunnerKeyboardBandFact: Equatable {
42+
case visible(CGRect)
43+
case absent
44+
case unmeasurable(String)
45+
46+
var payload: KeyboardBandFactPayload {
47+
switch self {
48+
case .visible(let frame):
49+
return KeyboardBandFactPayload(
50+
kind: "visible",
51+
frame: SnapshotRect(
52+
x: Double(frame.origin.x),
53+
y: Double(frame.origin.y),
54+
width: Double(frame.size.width),
55+
height: Double(frame.size.height)
56+
),
57+
reason: nil
58+
)
59+
case .absent:
60+
return KeyboardBandFactPayload(kind: "absent", frame: nil, reason: nil)
61+
case .unmeasurable(let reason):
62+
return KeyboardBandFactPayload(kind: "unmeasurable", frame: nil, reason: reason)
63+
}
64+
}
65+
66+
var logKind: String {
67+
switch self {
68+
case .visible: return "visible"
69+
case .absent: return "absent"
70+
case .unmeasurable: return "unmeasurable"
71+
}
72+
}
73+
74+
var logReason: String {
75+
switch self {
76+
case .visible, .absent: return "-"
77+
case .unmeasurable(let reason): return reason
78+
}
79+
}
80+
}
81+
82+
/// The reason codes a capture publishes beside `unmeasurable`. Key behavior travels on these, never
83+
/// on prose: a daemon log line and a `snapshot --json` reader have to agree on what `unmeasurable`
84+
/// meant without parsing a sentence.
85+
enum RunnerKeyboardBandReason {
86+
/// The `app.keyboards` read threw or otherwise produced no answer at all.
87+
static let queryFailed = "keyboard-frame-query-failed"
88+
/// The keyboard exists and reports no usable box. It is on screen — only its band is unknown, which
89+
/// is not the same claim as `absent`.
90+
static let unusableFrame = "keyboard-frame-unusable"
91+
/// The read did not return inside the slice this capture had left for it, so the band was abandoned
92+
/// rather than waited on: a keyboard this capture cannot afford to measure is not worth a stalled
93+
/// capture, and the daemon's tap guard falls back to the tree rule for this capture alone.
94+
static let queryTimeout = "keyboard-frame-query-timeout"
95+
/// No slice was left at all, so the read was never dispatched.
96+
static let budgetExhausted = "capture-plan-budget-exhausted"
97+
}
98+
99+
/// Whether a keyboard's reported box can be a band at all. A zero-area or non-finite box is what an
100+
/// element XCTest cannot measure looks like.
101+
///
102+
/// Reads `origin` and `size` rather than the rect's own accessors: `CGRect.width` and `CGRect.height`
103+
/// standardize, so a box reported with a negative size answers them with a positive one and an
104+
/// unmeasurable element would ship as a measured band. The daemon's reader applies the same rule to
105+
/// the wire frame, so neither language can publish a band the other would refuse.
106+
func runnerKeyboardFrameIsUsable(_ frame: CGRect) -> Bool {
107+
let size = frame.size
108+
let origin = frame.origin
109+
return [origin.x, origin.y, size.width, size.height].allSatisfy(\.isFinite)
110+
&& size.width > 0 && size.height > 0
111+
}
112+
113+
/// The one decision the keyboard-band probe makes, kept away from the live query so the unit lane can
114+
/// pin it without a keyboard on screen. Three outcomes and no fourth: a read that failed says it
115+
/// failed, a keyboard that is not there is proven absent, and a keyboard whose box cannot be measured
116+
/// is unmeasurable rather than absent — the difference is whether anything may claim this screen is
117+
/// clear of a keyboard.
118+
func runnerKeyboardBandFact(
119+
readSucceeded: Bool,
120+
exists: Bool,
121+
frame: CGRect
122+
) -> RunnerKeyboardBandFact {
123+
guard readSucceeded else { return .unmeasurable(RunnerKeyboardBandReason.queryFailed) }
124+
guard exists else { return .absent }
125+
guard runnerKeyboardFrameIsUsable(frame) else {
126+
return .unmeasurable(RunnerKeyboardBandReason.unusableFrame)
127+
}
128+
return .visible(frame)
129+
}
130+
131+
private struct KeyboardBandProbeTimeout: Error {}
132+
133+
extension RunnerTests {
134+
/// The band the visible keyboard owns, read once per capture.
135+
///
136+
/// Returns nil only where there is no iOS keyboard to look for. Every other outcome is stated,
137+
/// including the ones where looking failed: a missing field would leave the daemon guessing which
138+
/// tier answered, while a published `unmeasurable` names the reason where the band would have been.
139+
///
140+
/// Its own main-thread hop with its own slice, taken immediately after the viewport/orientation hop
141+
/// and before the tree is captured, so band and tree describe one moment while a keyboard this
142+
/// capture cannot afford to measure costs the capture nothing: a timeout abandons the fact and leaves
143+
/// the tree tier standing, the same trade `boundedBlockingSystemAlertSnapshot` makes for the probe it
144+
/// bounds (#2660).
145+
func captureKeyboardBandFact(app: XCUIApplication, deadline: Date) -> RunnerKeyboardBandFact? {
146+
#if os(iOS)
147+
let slice = min(Self.keyboardBandProbeBudget, max(0, deadline.timeIntervalSinceNow))
148+
guard slice > 0 else {
149+
NSLog(
150+
"AGENT_DEVICE_RUNNER_KEYBOARD_BAND_FACT kind=unmeasurable reason=%@ elapsedMs=0",
151+
RunnerKeyboardBandReason.budgetExhausted
152+
)
153+
return .unmeasurable(RunnerKeyboardBandReason.budgetExhausted)
154+
}
155+
let startedAt = Date()
156+
let fact: RunnerKeyboardBandFact
157+
do {
158+
fact = try runMainThreadWork(
159+
"keyboard_band",
160+
timeout: slice,
161+
timeoutError: { KeyboardBandProbeTimeout() }
162+
) {
163+
self.keyboardBandFact(app: app)
164+
}
165+
} catch {
166+
fact = .unmeasurable(RunnerKeyboardBandReason.queryTimeout)
167+
}
168+
NSLog(
169+
"AGENT_DEVICE_RUNNER_KEYBOARD_BAND_FACT kind=%@ reason=%@ elapsedMs=%d",
170+
fact.logKind,
171+
fact.logReason,
172+
Int(Date().timeIntervalSince(startedAt) * 1000)
173+
)
174+
return fact
175+
#else
176+
return nil
177+
#endif
178+
}
179+
180+
private func keyboardBandFact(app: XCUIApplication) -> RunnerKeyboardBandFact {
181+
let read = safely("KEYBOARD_BAND_FACT") { () -> RunnerKeyboardBandFact? in
182+
let keyboard = app.keyboards.firstMatch
183+
guard keyboard.exists else {
184+
return runnerKeyboardBandFact(readSucceeded: true, exists: false, frame: .zero)
185+
}
186+
return runnerKeyboardBandFact(readSucceeded: true, exists: true, frame: keyboard.frame)
187+
}
188+
return read ?? .unmeasurable(RunnerKeyboardBandReason.queryFailed)
189+
}
190+
}
191+
33192
extension RunnerTests {
34193
func isKeyboardVisible(app: XCUIApplication) -> Bool {
35194
return visibleKeyboardFrame(app: app) != nil
@@ -275,6 +434,88 @@ func runnerScreenshotStabilitySettled(
275434

276435
#if AGENT_DEVICE_RUNNER_UNIT_TESTS
277436
extension RunnerTests {
437+
func testKeyboardBandFactReadFailureIsStatedAsUnmeasurableNotAbsence() {
438+
// Absence would let a consumer claim the screen is clear of a keyboard on the strength of a read
439+
// that never answered.
440+
let fact = runnerKeyboardBandFact(
441+
readSucceeded: false,
442+
exists: false,
443+
frame: CGRect(x: 0, y: 198, width: 402, height: 204)
444+
)
445+
XCTAssertEqual(fact, .unmeasurable(RunnerKeyboardBandReason.queryFailed))
446+
XCTAssertEqual(fact.payload.kind, "unmeasurable")
447+
XCTAssertEqual(fact.payload.reason, RunnerKeyboardBandReason.queryFailed)
448+
XCTAssertNil(fact.payload.frame)
449+
}
450+
451+
func testKeyboardBandFactPublishesAbsenceWhenTheQueryFindsNoKeyboard() {
452+
let fact = runnerKeyboardBandFact(readSucceeded: true, exists: false, frame: .zero)
453+
XCTAssertEqual(fact, .absent)
454+
XCTAssertEqual(fact.payload.kind, "absent")
455+
XCTAssertNil(fact.payload.frame)
456+
XCTAssertNil(fact.payload.reason)
457+
}
458+
459+
func testKeyboardBandFactPublishesTheMeasuredBandInAppOrientationSpace() {
460+
// The landscape band measured on iPhone 17 Pro (iOS 26.2) after #2653: full width across the
461+
// bottom of a 402 pt-tall app, which is what the tree reported as a strip down the left edge.
462+
let frame = CGRect(x: 0, y: 198, width: 874, height: 204)
463+
let fact = runnerKeyboardBandFact(readSucceeded: true, exists: true, frame: frame)
464+
XCTAssertEqual(fact, .visible(frame))
465+
let payload = fact.payload
466+
XCTAssertEqual(payload.kind, "visible")
467+
XCTAssertEqual(payload.frame, SnapshotRect(x: 0, y: 198, width: 874, height: 204))
468+
XCTAssertNil(payload.reason)
469+
}
470+
471+
func testKeyboardBandFactRefusesUnusableGeometryInsteadOfClaimingAbsence() {
472+
let nan = CGFloat(Double.nan)
473+
let infinite = CGFloat.infinity
474+
let unusable: [CGRect] = [
475+
.zero,
476+
CGRect(x: 0, y: 198, width: 0, height: 204),
477+
CGRect(x: 0, y: 198, width: 874, height: -1),
478+
CGRect(x: 0, y: 198, width: -874, height: 204),
479+
CGRect(x: 0, y: nan, width: 874, height: 204),
480+
CGRect(x: 0, y: 198, width: nan, height: 204),
481+
CGRect(x: infinite, y: 198, width: 874, height: 204)
482+
]
483+
for frame in unusable {
484+
let fact = runnerKeyboardBandFact(readSucceeded: true, exists: true, frame: frame)
485+
XCTAssertEqual(
486+
fact,
487+
.unmeasurable(RunnerKeyboardBandReason.unusableFrame),
488+
"expected \(frame) to be refused as a band"
489+
)
490+
}
491+
}
492+
493+
func testKeyboardBandFactPayloadRoundTripsThroughTheWireShape() throws {
494+
let cases: [RunnerKeyboardBandFact] = [
495+
.visible(CGRect(x: 0, y: 583, width: 402, height: 291)),
496+
.absent,
497+
.unmeasurable(RunnerKeyboardBandReason.queryTimeout)
498+
]
499+
for fact in cases {
500+
let data = try JSONEncoder().encode(fact.payload)
501+
// `encodeIfPresent` for the two optional fields: a fact carries its own evidence and nothing
502+
// else, so the daemon never has to distinguish a null from an absent key.
503+
let object = try XCTUnwrap(JSONSerialization.jsonObject(with: data) as? [String: Any])
504+
switch fact {
505+
case .visible:
506+
XCTAssertNil(object["reason"])
507+
XCTAssertNotNil(object["frame"])
508+
case .absent:
509+
XCTAssertNil(object["frame"])
510+
XCTAssertNil(object["reason"])
511+
case .unmeasurable:
512+
XCTAssertNil(object["frame"])
513+
XCTAssertNotNil(object["reason"])
514+
}
515+
XCTAssertEqual(try JSONDecoder().decode(KeyboardBandFactPayload.self, from: data), fact.payload)
516+
}
517+
}
518+
278519
func testRunnerScreenshotStabilitySettledNeedsEnoughSamples() {
279520
XCTAssertFalse(runnerScreenshotStabilitySettled([], requiredConsecutiveMatches: 3))
280521
XCTAssertFalse(runnerScreenshotStabilitySettled([Data([1])], requiredConsecutiveMatches: 3))

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -253,6 +253,10 @@ struct DataPayload: Codable {
253253
var snapshotQuality: SnapshotQuality?
254254
/// Set when the capture describes an in-place system surface, not the app itself (#2438).
255255
var systemSurface: SystemSurfaceProvenancePayload?
256+
/// The keyboard band this capture measured, when the tier that answered reads keyboards at all
257+
/// (#2660). Absent means the query-sweep or private-AX tier answered, and the daemon's tap guard
258+
/// keeps deriving the band from the tree.
259+
var keyboard: KeyboardBandFactPayload?
256260
var gestureStartUptimeMs: Double?
257261
var gestureEndUptimeMs: Double?
258262
var x: Double?
@@ -296,6 +300,16 @@ struct DataPayload: Codable {
296300
var sequenceResults: [SequenceStepResult]?
297301
}
298302

303+
/// `kind` mirrors the TS `SnapshotKeyboardBandFact`: "visible" carries `frame`, "unmeasurable"
304+
/// carries `reason`, and "absent" carries nothing because there is nothing to say. `frame` is in the
305+
/// app's own orientation space — the same space `SnapshotGeometrySpace` publishes every node rect in
306+
/// — so the daemon compares it against node rects without transforming either side (#2660).
307+
struct KeyboardBandFactPayload: Codable, Equatable {
308+
let kind: String
309+
let frame: SnapshotRect?
310+
let reason: String?
311+
}
312+
299313
/// `kind` mirrors the TS `IosSystemSurfaceKind` (e.g. "web-auth").
300314
struct SystemSurfaceProvenancePayload: Codable {
301315
let bundleId: String

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,12 @@ extension RunnerTests {
1818
let viewport: CGRect
1919
/** Which way the app's interface is turned from the device's native space (#2612). */
2020
let interfaceOrientation: Int
21+
/**
22+
* The keyboard band this capture measured, published beside the tree so the daemon's tap guard
23+
* measures against the producer's own reading rather than a band it derives from these rects
24+
* (#2660). Nil only where the platform has no iOS keyboard to measure.
25+
*/
26+
let keyboardBand: RunnerKeyboardBandFact?
2127
}
2228

2329
private struct SnapshotEvaluation {
@@ -115,6 +121,13 @@ extension RunnerTests {
115121

116122
static let flatInteractiveFallbackBudget: TimeInterval = 1.0
117123

124+
/// What one capture may spend reading the keyboard band before it gives up on the fact and lets the
125+
/// tap guard fall back to the tree rule. The scroll path pays this query per gesture and stays well
126+
/// inside a second; the number here is a ceiling for a read that normally returns in milliseconds,
127+
/// sized so a hostile keyboard surface cannot extend a capture the way the unbounded read it
128+
/// replaced would have (#2660).
129+
static let keyboardBandProbeBudget: TimeInterval = 0.3
130+
118131
// The single production entry point -- always compiled, no unit-test overload. A unit test
119132
// exercises this exact function; the only injectable seam lives inside
120133
// `boundedBlockingSystemAlertSnapshot`'s probe closure (see `systemModalProbeOverrideForTesting`
@@ -895,6 +908,11 @@ extension RunnerTests {
895908
}
896909
let viewport = geometry.viewport
897910
let interfaceOrientation = geometry.interfaceOrientation
911+
// The band is read here, immediately after the pair above and before the tree is captured, so the
912+
// geometry the band is compared against is the geometry this capture publishes. It keeps its own
913+
// hop and its own slice rather than joining that one: a keyboard this capture cannot measure must
914+
// cost the fact and not the tree tier behind it (#2660).
915+
let keyboardBand = captureKeyboardBandFact(app: app, deadline: captureDeadline)
898916

899917
let treeSliceBudget = treeCaptureSliceBudgetOverride ?? treeCaptureSliceBudget
900918
let slice = min(treeSliceBudget, max(0.5, captureDeadline.timeIntervalSinceNow))
@@ -906,7 +924,8 @@ extension RunnerTests {
906924
queryRoot: app,
907925
rootSnapshot: rootSnapshot,
908926
viewport: viewport,
909-
interfaceOrientation: interfaceOrientation
927+
interfaceOrientation: interfaceOrientation,
928+
keyboardBand: keyboardBand
910929
)
911930
}
912931

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,10 @@ struct SnapshotBackendCapture {
6262
var customActions: SnapshotCustomActionCoverage? = nil
6363
var qualityPayload: DataPayload? = nil
6464
var timing: SnapshotCaptureTiming? = nil
65+
/// The keyboard band this capture measured, carried beside the tree it came with (#2660). Only the
66+
/// tree tier reads the keyboard, so only that tier has one; the daemon reads a missing band as "this
67+
/// producer could not measure", which is exactly what the query-sweep and private-AX tiers did.
68+
var keyboardBand: KeyboardBandFactPayload? = nil
6569
}
6670

6771
extension RunnerTests {
@@ -416,6 +420,9 @@ extension RunnerTests {
416420
let hint = SnapshotPresentation.captureHint(for: options)
417421
var timer = SnapshotPhaseTimer()
418422
let acquisition: SnapshotAcquisition?
423+
// The band is read inside the tree tier's own bounded work, so it has to be lifted out of the
424+
// acquisition phase and carried to the stamping step, where the payload is assembled (#2660).
425+
var keyboardBand: RunnerKeyboardBandFact?
419426
do {
420427
acquisition = try timer.measure(.acquisition) {
421428
switch kind {
@@ -430,6 +437,7 @@ extension RunnerTests {
430437
else {
431438
return nil
432439
}
440+
keyboardBand = context.keyboardBand
433441
return try self.runMainThreadWork(
434442
"tree_processing",
435443
timeout: min(self.treeCaptureSliceBudget, max(0.5, deadline.timeIntervalSinceNow)),
@@ -503,6 +511,7 @@ extension RunnerTests {
503511

504512
var capture = presented
505513
capture.timing = timer.timing
514+
capture.keyboardBand = keyboardBand?.payload
506515
return SnapshotBackendAttempt(
507516
outcome: .captured(capture),
508517
timing: timer.timing
@@ -641,6 +650,7 @@ extension RunnerTests {
641650
return SnapshotQualityPayload(nodes: nodes, truncated: quality.truncated == true)
642651
},
643652
snapshotQuality: quality,
653+
keyboard: capture.keyboardBand,
644654
runnerFatal: payload.runnerFatal,
645655
runnerFatalReason: payload.runnerFatalReason
646656
)

0 commit comments

Comments
 (0)