Skip to content

Commit 1e7092c

Browse files
thymikeeclaude
andauthored
fix(ios): stop the XCTest-channel penalty from re-arming itself (#3896)
* fix(ios): prevent XCTest channel penalty recovery loops * fix(ios): schedule bounded recovery without losing hostile-tree deferral * fix(ios): run recovery probes after returning accepted captures * fix(ios): run the due XCTest recovery probe inside the deferred capture Drop the post-response recovery probe: it started XCTest main-thread work after the snapshot had answered, so its abandoned tree work could refuse the next tap with RUNNER_BUSY while the snapshot's occupancy stamp said idle. The penalty is now one SnapshotXCTestChannelPenalty value (arm, clear, isProbeDue, claimProbe). When a probe is due, effectiveSnapshotCapturePlan puts the bounded tree ahead of private AX in the deferred plan, so any abandoned probe work is charged to that capture and its busy stamp. Restore the occupancy test's second-plan assertions that an armed penalty defers the next plan's XCTest tiers and starts no sweep query. Refs #3844 Co-Authored-By: Claude Opus 5.5 <[email protected]> * docs(adr): record the in-band XCTest recovery probe in ADR 0004 Replace the post-response probe text with the in-band policy and note that, after the iPad amendment, Simulator captures meet it only when the bridge sends them to XCTest. Refs #3844 Co-Authored-By: Claude Opus 5.5 <[email protected]> --------- Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 72c051d commit 1e7092c

8 files changed

Lines changed: 386 additions & 29 deletions

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ extension RunnerTests {
132132
privateAXAcceptedDepthBundleId = bundleId
133133
privateAXAcceptedDepthProcessIdentifier = processIdentifier
134134
privateAXAcceptedDepth = depth
135-
privateAXAcceptedDepthUntil = Date().addingTimeInterval(snapshotXCTestChannelPenaltyDuration)
135+
privateAXAcceptedDepthUntil = Date().addingTimeInterval(SnapshotXCTestChannelPenalty.duration)
136136
privateAXAcceptedDepthLock.unlock()
137137
NSLog("AGENT_DEVICE_RUNNER_PRIVATE_AX_DEPTH_REMEMBERED depth=%ld bundle=%@", depth, bundleId ?? "")
138138
}

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift‎

Lines changed: 79 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,45 @@ enum SnapshotXCTestChannelPlanState: Equatable {
4747
case boundedXCTestProbe
4848
}
4949

50+
/// The XCTest accessibility channel penalty (#1105/#1156). While it is active, regular plans for its
51+
/// bundle defer XCTest tiers. Each recovery interval, one deferred plan runs the bounded tree probe
52+
/// first, inside that capture, so a screen that is light again can clear the penalty before it
53+
/// expires (#3844).
54+
struct SnapshotXCTestChannelPenalty: Equatable {
55+
static let duration: TimeInterval = 120
56+
static let recoveryProbeInterval: TimeInterval = 30
57+
58+
private(set) var bundleId: String?
59+
private(set) var until = Date.distantPast
60+
private(set) var probeDueAt = Date.distantFuture
61+
62+
/// A penalty armed without a bundle id applies to whatever target is current.
63+
func isActive(for bundleId: String?, at now: Date) -> Bool {
64+
guard now < until else { return false }
65+
guard let penalized = self.bundleId else { return true }
66+
return penalized == bundleId
67+
}
68+
69+
func isProbeDue(for bundleId: String?, at now: Date) -> Bool {
70+
isActive(for: bundleId, at: now) && now >= probeDueAt
71+
}
72+
73+
mutating func arm(bundleId: String?, at now: Date) {
74+
self.bundleId = bundleId
75+
until = now.addingTimeInterval(Self.duration)
76+
probeDueAt = now.addingTimeInterval(Self.recoveryProbeInterval)
77+
}
78+
79+
/// A started probe moves the next one a full interval out, whatever the probe finds.
80+
mutating func claimProbe(at now: Date) {
81+
probeDueAt = now.addingTimeInterval(Self.recoveryProbeInterval)
82+
}
83+
84+
mutating func clear() {
85+
self = SnapshotXCTestChannelPenalty()
86+
}
87+
}
88+
5089
/// How one tier's bounded work ended. `deadlineExhausted` is a tier timeout: the tier stopped
5190
/// starting work it could not finish inside its own slice, so what it returns is a partial result
5291
/// it never completed collecting. The plan keeps that result only as the fallback and lets the next
@@ -64,6 +103,12 @@ struct EffectiveSnapshotCapturePlan {
64103
/// Non-nil only when the plan was narrowed by an explicit internal backend preference. This
65104
/// keeps the quality marker tied to the plan decision rather than to an untrusted request field.
66105
let preferredBackend: SnapshotBackendKind?
106+
107+
/// A deferred plan that starts with an XCTest tier is running the due recovery probe.
108+
var leadsWithXCTestRecoveryProbe: Bool {
109+
xCTestChannelState == .deferredToIndependentBackend
110+
&& plan.first?.usesXCTestAccessibilityChannel == true
111+
}
67112
}
68113

69114
/// What the plan runner does when every backend failed or stayed sparse.
@@ -144,8 +189,7 @@ extension RunnerTests {
144189

145190
func penalizeSnapshotXCTestChannel(bundleId: String?, reason: String) {
146191
snapshotXCTestChannelPenaltyLock.lock()
147-
snapshotXCTestChannelPenaltyBundleId = bundleId
148-
snapshotXCTestChannelPenaltyUntil = Date().addingTimeInterval(snapshotXCTestChannelPenaltyDuration)
192+
snapshotXCTestChannelPenalty.arm(bundleId: bundleId, at: Date())
149193
snapshotXCTestChannelPenaltyLock.unlock()
150194
NSLog(
151195
"AGENT_DEVICE_RUNNER_SNAPSHOT_XCTEST_CHANNEL_PENALIZED bundle=%@ reason=%@",
@@ -156,9 +200,8 @@ extension RunnerTests {
156200

157201
func clearSnapshotXCTestChannelPenalty(reason: String) {
158202
snapshotXCTestChannelPenaltyLock.lock()
159-
let hadActivePenalty = Date() < snapshotXCTestChannelPenaltyUntil
160-
snapshotXCTestChannelPenaltyBundleId = nil
161-
snapshotXCTestChannelPenaltyUntil = Date.distantPast
203+
let hadActivePenalty = Date() < snapshotXCTestChannelPenalty.until
204+
snapshotXCTestChannelPenalty.clear()
162205
snapshotXCTestChannelPenaltyLock.unlock()
163206
if hadActivePenalty {
164207
NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_XCTEST_CHANNEL_PENALTY_CLEARED reason=%@", reason)
@@ -168,10 +211,20 @@ extension RunnerTests {
168211
func isSnapshotXCTestChannelPenalized(bundleId: String?) -> Bool {
169212
snapshotXCTestChannelPenaltyLock.lock()
170213
defer { snapshotXCTestChannelPenaltyLock.unlock() }
171-
guard Date() < snapshotXCTestChannelPenaltyUntil else { return false }
172-
// A penalty recorded without a bundle id applies to whatever target is current.
173-
guard let penalized = snapshotXCTestChannelPenaltyBundleId else { return true }
174-
return penalized == bundleId
214+
return snapshotXCTestChannelPenalty.isActive(for: bundleId, at: Date())
215+
}
216+
217+
func isSnapshotXCTestRecoveryProbeDue(bundleId: String?) -> Bool {
218+
snapshotXCTestChannelPenaltyLock.lock()
219+
defer { snapshotXCTestChannelPenaltyLock.unlock() }
220+
return snapshotXCTestChannelPenalty.isProbeDue(for: bundleId, at: Date())
221+
}
222+
223+
func claimSnapshotXCTestRecoveryProbe(bundleId: String?) {
224+
snapshotXCTestChannelPenaltyLock.lock()
225+
snapshotXCTestChannelPenalty.claimProbe(at: Date())
226+
snapshotXCTestChannelPenaltyLock.unlock()
227+
NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_XCTEST_RECOVERY_PROBE bundle=%@", bundleId ?? "")
175228
}
176229

177230
/// The pre-seeded first-failure a penalized plan stamps into its verdict. The deferred case
@@ -244,11 +297,13 @@ extension RunnerTests {
244297
/// Pure plan-reorder rule: an internal preferred backend pins the regular plan to that backend;
245298
/// this is the only force seam used by same-backend evidence and conformance captures. A
246299
/// penalized XCTest accessibility channel uses independent backends when the platform has one,
247-
/// otherwise it keeps XCTest work on a short probe. The raw diagnostic plan keeps tree-first
248-
/// errors, and unknown plans are left untouched.
300+
/// otherwise it keeps XCTest work on a short probe. A due recovery probe puts that short tree
301+
/// ahead of the independent backend. The raw diagnostic plan keeps tree-first errors, and unknown
302+
/// plans are left untouched.
249303
static func effectiveSnapshotCapturePlan(
250304
_ plan: [SnapshotBackendKind],
251305
xCTestChannelPenalized: Bool,
306+
xCTestRecoveryProbeDue: Bool = false,
252307
availableBackends: Set<SnapshotBackendKind> = Set(SnapshotBackendKind.allCases),
253308
preferredBackend: String? = nil
254309
) -> EffectiveSnapshotCapturePlan {
@@ -278,10 +333,12 @@ extension RunnerTests {
278333
if !recoveryPlan.isEmpty {
279334
// The independent backend reads only the app it can match as the active AX application, so
280335
// an out-of-process surface over the app (the Save Password sheet) leaves it empty for the
281-
// whole penalty. The tree, on the bounded probe's short slice, stays behind it; the query
282-
// sweep does not, since its grind is what the penalty keeps off the main thread.
336+
// whole penalty. The tree, on the bounded probe's short slice, stays behind it, or leads when
337+
// the recovery probe is due; the query sweep does not, since its grind is what the penalty
338+
// keeps off the main thread.
339+
let boundedTree = availablePlan.filter { $0 == .recursiveTree }
283340
return EffectiveSnapshotCapturePlan(
284-
plan: recoveryPlan + availablePlan.filter { $0 == .recursiveTree },
341+
plan: xCTestRecoveryProbeDue ? boundedTree + recoveryPlan : recoveryPlan + boundedTree,
285342
xCTestChannelState: .deferredToIndependentBackend,
286343
treeCaptureSliceBudgetOverride: Self.penalizedXCTestProbeTreeSliceBudget,
287344
preferredBackend: nil
@@ -324,8 +381,10 @@ extension RunnerTests {
324381
// deliberately, so no degradation warning should render for it.
325382
var xCTestChannelPenalized = false
326383
var xCTestChannelPenalizedByBreaker = false
384+
var xCTestRecoveryProbeDue = false
327385
#if os(iOS)
328386
xCTestChannelPenalizedByBreaker = isSnapshotXCTestChannelPenalized(bundleId: target.bundleId)
387+
xCTestRecoveryProbeDue = isSnapshotXCTestRecoveryProbeDue(bundleId: target.bundleId)
329388
xCTestChannelPenalized = Self.snapshotXCTestChannelTreatedAsPenalized(
330389
penalized: xCTestChannelPenalizedByBreaker,
331390
preferredBackend: options.preferredBackend
@@ -334,6 +393,7 @@ extension RunnerTests {
334393
let effective = Self.effectiveSnapshotCapturePlan(
335394
plan,
336395
xCTestChannelPenalized: xCTestChannelPenalized,
396+
xCTestRecoveryProbeDue: xCTestRecoveryProbeDue,
337397
availableBackends: Set(SnapshotBackendKind.allCases.filter(\.isAvailableOnCurrentPlatform)),
338398
preferredBackend: options.preferredBackend
339399
)
@@ -374,6 +434,9 @@ extension RunnerTests {
374434
}
375435
continue
376436
}
437+
if kind == effectivePlan.first && effective.leadsWithXCTestRecoveryProbe {
438+
claimSnapshotXCTestRecoveryProbe(bundleId: target.bundleId)
439+
}
377440
if kind.usesXCTestAccessibilityChannel { xCTestTierRan = true }
378441
let attempt = try captureWithBackend(
379442
kind,
@@ -386,7 +449,8 @@ extension RunnerTests {
386449
kind,
387450
attempt: attempt,
388451
bundleId: target.bundleId,
389-
penaltySuppressed: suppressXCTestPenalty
452+
penaltySuppressed: suppressXCTestPenalty,
453+
channelState: effective.xCTestChannelState
390454
)
391455
if case let .failed(failure, phase: _) = attempt.outcome {
392456
if Self.isAxSnapshotFailure(failure) { axFailure = failure }

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift‎

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -119,9 +119,10 @@ extension RunnerTests {
119119
static func snapshotXCTestPenaltyReason(
120120
kind: SnapshotBackendKind,
121121
attempt: SnapshotBackendAttempt,
122-
slowThresholdMs: Double
122+
slowThresholdMs: Double,
123+
channelState: SnapshotXCTestChannelPlanState = .normal
123124
) -> String? {
124-
guard kind.usesXCTestAccessibilityChannel else { return nil }
125+
guard channelState == .normal, kind.usesXCTestAccessibilityChannel else { return nil }
125126
if case let .failed(failure, phase: .acquisition) = attempt.outcome,
126127
failure.code == Self.xCTestSnapshotTimeoutCode
127128
{
@@ -134,18 +135,46 @@ extension RunnerTests {
134135
return "slow_\(kind.rawValue)_capture_\(Int(attempt.timing.acquisitionMs))ms"
135136
}
136137

138+
/// Only an accepted, fast tree proves that a bounded recovery probe restored the channel.
139+
static func snapshotXCTestRecoveryProbeSucceeded(
140+
kind: SnapshotBackendKind,
141+
attempt: SnapshotBackendAttempt,
142+
channelState: SnapshotXCTestChannelPlanState
143+
) -> Bool {
144+
guard channelState != .normal,
145+
kind == .recursiveTree,
146+
attempt.timing.acquisitionMs <= Self.penalizedXCTestProbeTreeSliceBudget * 1_000,
147+
case let .captured(capture) = attempt.outcome
148+
else { return false }
149+
return Self.snapshotTierRejectionReason(
150+
outcome: attempt.tierOutcome,
151+
kind: kind,
152+
payload: capture.qualityPayload ?? capture.payload
153+
) == nil
154+
}
155+
137156
func recordXCTestSnapshotBackendAttemptIfNeeded(
138157
_ kind: SnapshotBackendKind,
139158
attempt: SnapshotBackendAttempt,
140159
bundleId: String?,
141-
penaltySuppressed: Bool
160+
penaltySuppressed: Bool,
161+
channelState: SnapshotXCTestChannelPlanState
142162
) {
143163
guard !penaltySuppressed else { return }
164+
if Self.snapshotXCTestRecoveryProbeSucceeded(
165+
kind: kind,
166+
attempt: attempt,
167+
channelState: channelState
168+
) {
169+
clearSnapshotXCTestChannelPenalty(reason: "healthy_tree_recovery_probe")
170+
return
171+
}
144172
guard
145173
let reason = Self.snapshotXCTestPenaltyReason(
146174
kind: kind,
147175
attempt: attempt,
148-
slowThresholdMs: snapshotXCTestSlowCaptureThreshold * 1_000
176+
slowThresholdMs: snapshotXCTestSlowCaptureThreshold * 1_000,
177+
channelState: channelState
149178
)
150179
else { return }
151180
penalizeSnapshotXCTestChannel(

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -125,9 +125,7 @@ final class RunnerTests: XCTestCase {
125125
// channel when an independent recovery backend exists, or use a bounded XCTest probe when it
126126
// does not, for the same screen class (#1105/#1156).
127127
let snapshotXCTestChannelPenaltyLock = NSLock()
128-
var snapshotXCTestChannelPenaltyBundleId: String?
129-
var snapshotXCTestChannelPenaltyUntil = Date.distantPast
130-
let snapshotXCTestChannelPenaltyDuration: TimeInterval = 120
128+
var snapshotXCTestChannelPenalty = SnapshotXCTestChannelPenalty()
131129
let snapshotXCTestPenaltyWarmupExemption = SnapshotXCTestPenaltyWarmupExemption()
132130
// Sticky per-bundle hint for the private AX depth ladder: deep RN screens reject the default
133131
// depth with kAXErrorIllegalArgument on EVERY capture, so once a shallower rung is accepted

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -430,6 +430,14 @@ extension RunnerTests {
430430
XCTAssertEqual(quality.state, .recovered)
431431
XCTAssertEqual(quality.backend, SnapshotBackendKind.recursiveTree.rawValue)
432432
XCTAssertGreaterThan(payload.nodes?.count ?? 0, 1, "the bounded tree answers with a real tree")
433+
let timing = try XCTUnwrap(quality.timing)
434+
guard timing.acquisitionMs <= Self.penalizedXCTestProbeTreeSliceBudget * 1_000 else {
435+
throw XCTSkip("the accepted tree's total acquisition exceeded the fast-probe threshold on this host")
436+
}
437+
XCTAssertFalse(
438+
isSnapshotXCTestChannelPenalized(bundleId: captureTarget.bundleId),
439+
"an accepted fast tree probe must clear the existing penalty early"
440+
)
433441
}
434442

435443
/// When the bounded tree also grinds past its slice, the plan ends sparse, and the verdict still
@@ -465,9 +473,15 @@ extension RunnerTests {
465473
tearDownPenalizedPrivateAXBlindTarget()
466474
}
467475

476+
let originalPenaltyUntil = snapshotXCTestChannelPenalty.until
468477
let payload = try runDeferredPlanOffMain(target: captureTarget) {
469478
RunnerBlockingSnapshotGate.release.signal()
470479
}
480+
XCTAssertEqual(
481+
snapshotXCTestChannelPenalty.until,
482+
originalPenaltyUntil,
483+
"the bounded recovery probe must not renew its own penalty when its tree work is abandoned"
484+
)
471485

472486
let quality = try XCTUnwrap(payload.snapshotQuality)
473487
XCTAssertEqual(quality.state, .sparse)
@@ -537,6 +551,64 @@ extension RunnerTests {
537551
XCTAssertNotEqual(quality.state, .healthy, "a tree without its collapsed tab items is not healthy")
538552
}
539553

554+
/// A due recovery probe runs inside the capture, ahead of private AX. Its abandoned tree work is
555+
/// charged to that capture: the plan answers while the work is outstanding, so that response's
556+
/// occupancy stamp reports it. The probe cannot renew the penalty, and the next plan inside the
557+
/// interval defers without entering the tree.
558+
func testDueRecoveryProbeRunsInsideTheCaptureAndChargesItsAbandonedWorkToIt() throws {
559+
let captureTarget = try launchPenalizedPrivateAXBlindTarget(
560+
bundleId: "com.callstack.agentdevice.runner.due-probe-test"
561+
)
562+
defer { tearDownPenalizedPrivateAXBlindTarget() }
563+
let options = PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false)
564+
let acquisition = try XCTUnwrap(privateAXSnapshotAcquisition(
565+
target: captureTarget, hint: SnapshotPresentation.captureHint(for: options),
566+
deadline: Date().addingTimeInterval(10)
567+
))
568+
privateAXAcquisitionOverrideForTesting = { acquisition }
569+
let snapshotMethod = try XCTUnwrap(class_getInstanceMethod(XCUIApplication.self, #selector(XCUIElement.snapshot)))
570+
let stubMethod = try XCTUnwrap(class_getInstanceMethod(RunnerBlockingSnapshotStub.self, #selector(RunnerBlockingSnapshotStub.snapshot)))
571+
let original = method_getImplementation(snapshotMethod)
572+
RunnerBlockingSnapshotGate.release = DispatchSemaphore(value: 0)
573+
RunnerBlockingSnapshotGate.entered = DispatchSemaphore(value: 0)
574+
method_setImplementation(snapshotMethod, method_getImplementation(stubMethod))
575+
defer {
576+
RunnerBlockingSnapshotGate.release.signal()
577+
method_setImplementation(snapshotMethod, original)
578+
waitForAbandonedMainThreadWorkToDrain()
579+
}
580+
snapshotXCTestChannelPenalty.claimProbe(
581+
at: Date(timeIntervalSinceNow: -SnapshotXCTestChannelPenalty.recoveryProbeInterval)
582+
)
583+
let expiry = snapshotXCTestChannelPenalty.until
584+
final class Occupancy { var atAnswer = false }
585+
let occupancy = Occupancy()
586+
587+
let probed = try runDeferredPlanOffMain(target: captureTarget) {
588+
occupancy.atAnswer = self.hasAbandonedMainThreadWork()
589+
RunnerBlockingSnapshotGate.release.signal()
590+
}
591+
592+
XCTAssertEqual(RunnerBlockingSnapshotGate.entered.wait(timeout: .now()), .success, "the probe ran before the answer")
593+
XCTAssertTrue(occupancy.atAnswer, "the capture that ran the probe answers as busy")
594+
XCTAssertEqual(probed.snapshotQuality?.backend, SnapshotBackendKind.privateAX.rawValue)
595+
XCTAssertEqual(probed.snapshotQuality?.reasonCode, "deferred")
596+
XCTAssertEqual(snapshotXCTestChannelPenalty.until, expiry, "a failed probe must not renew the penalty")
597+
XCTAssertFalse(isSnapshotXCTestRecoveryProbeDue(bundleId: captureTarget.bundleId))
598+
waitForAbandonedMainThreadWorkToDrain()
599+
600+
let deferred = try runDeferredPlanOffMain(target: captureTarget)
601+
XCTAssertEqual(deferred.snapshotQuality?.backend, SnapshotBackendKind.privateAX.rawValue)
602+
XCTAssertEqual(RunnerBlockingSnapshotGate.entered.wait(timeout: .now()), .timedOut, "no probe inside the interval")
603+
}
604+
605+
private func waitForAbandonedMainThreadWorkToDrain() {
606+
let deadline = Date().addingTimeInterval(3)
607+
while hasAbandonedMainThreadWork(), Date() < deadline {
608+
RunLoop.current.run(until: Date().addingTimeInterval(0.01))
609+
}
610+
}
611+
540612
private func launchPenalizedPrivateAXBlindTarget(bundleId: String) throws -> SnapshotCaptureTarget {
541613
app.launchArguments = ["--agent-device-selector-read-regression"]
542614
app.launch()

0 commit comments

Comments
 (0)