Skip to content

Commit a430292

Browse files
committed
Merge origin/main into t3/ios-fill-performance
* origin/main: feat(test-app): add a Bluesky-depth heavy feed screen (#3884) fix(android): read a child that vanished mid-read as gone (#3901) feat(ios): route iPad Simulator snapshots through the host AX bridge (#3895) Prepare shared resource ownership and selection for #3325 (#3887) refactor(kernel): track held execution locks in keyed-lock (#3879) perf(ios): ask a ready runner about surface hosts before spawning probes (#3877) fix(ios): keep the bridge while a surface host only lingers (#3873) perf(ios): synthesize taps on resolved in-app back controls (#3880) fix(macos): open a running app by its own path instead of its bundle id (#3889) fix(macos): accept mixed-case bundle ids in open (#3867) fix(macos-helper): scroll SwiftUI forms whose rows are scroll-area children (#3888)
2 parents 2d34505 + 5f6dfb3 commit a430292

103 files changed

Lines changed: 3523 additions & 692 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityNodeReads.java‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,19 @@ static AccessibilityNodeInfo root(AccessibilityWindowInfo window) {
3030
return window.getRoot();
3131
}
3232

33+
/**
34+
* The child, or null when it is gone. When the child's window closes during the read, the
35+
* framework can throw a NullPointerException from its result integrity check instead of answering
36+
* null (seen on API 35 while an alert dialog closed), so that throw means the same as null.
37+
*/
3338
static AccessibilityNodeInfo child(AccessibilityNodeInfo node, int index) {
34-
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
35-
return node.getChild(index, PREFETCH_STRATEGY);
39+
try {
40+
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
41+
return node.getChild(index, PREFETCH_STRATEGY);
42+
}
43+
return node.getChild(index);
44+
} catch (NullPointerException vanished) {
45+
return null;
3646
}
37-
return node.getChild(index);
3847
}
3948
}

‎apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -259,6 +259,18 @@ func scrollBarValue(current: Double, signedTravel: Double, overflow: Double) ->
259259
min(1, max(0, current + signedTravel / overflow))
260260
}
261261

262+
/// A SwiftUI `Form` lists each row as a child of the scroll area, with no single document view as
263+
/// tall as the content, so the content is the span all of the area's children cover.
264+
func scrollContentOverflow(area: RectResponse, contentFrames: [RectResponse], isVertical: Bool)
265+
-> Double
266+
{
267+
let spans = contentFrames
268+
.filter { $0.width > 0 && $0.height > 0 }
269+
.map { isVertical ? ($0.y, $0.y + $0.height) : ($0.x, $0.x + $0.width) }
270+
guard let start = spans.map(\.0).min(), let end = spans.map(\.1).max() else { return 0 }
271+
return end - start - (isVertical ? area.height : area.width)
272+
}
273+
262274
func isPressableControlRole(_ role: String) -> Bool {
263275
pressableControlRoles.contains(role)
264276
}
@@ -426,15 +438,17 @@ private func performScrollBarScroll(
426438
guard let bar = elementAttribute(element, attribute: barAttribute as String),
427439
isAttributeSettable(bar, attribute: kAXValueAttribute as String),
428440
let value = numberAttribute(bar, attribute: kAXValueAttribute as String),
429-
let area = rectAttribute(element),
430-
let content = children(of: element)
431-
.filter({ role(of: $0) != "AXScrollBar" })
432-
.compactMap(rectAttribute)
433-
.max(by: { $0.width * $0.height < $1.width * $1.height })
441+
let area = rectAttribute(element)
434442
else {
435443
continue
436444
}
437-
let overflow = isVertical ? content.height - area.height : content.width - area.width
445+
let overflow = scrollContentOverflow(
446+
area: area,
447+
contentFrames: children(of: element)
448+
.filter({ role(of: $0) != "AXScrollBar" })
449+
.compactMap(rectAttribute),
450+
isVertical: isVertical
451+
)
438452
guard overflow > 0 else { continue }
439453
let next = scrollBarValue(current: value, signedTravel: signedTravel, overflow: overflow)
440454
return AXUIElementSetAttributeValue(bar, kAXValueAttribute as CFString, NSNumber(value: next))

‎apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/BackgroundInteractionTests.swift‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,33 @@ final class BackgroundInteractionTests: XCTestCase {
7878
XCTAssertEqual(scrollBarValue(current: 0.1, signedTravel: -600, overflow: 1200), 0)
7979
}
8080

81+
func testScrollOverflowSpansEveryContentChild() {
82+
let area = RectResponse(x: 0, y: 100, width: 400, height: 600)
83+
let rows = (0..<10).map { row in
84+
RectResponse(x: 20, y: 100 + Double(row) * 120, width: 360, height: 100)
85+
}
86+
XCTAssertEqual(scrollContentOverflow(area: area, contentFrames: rows, isVertical: true), 580)
87+
XCTAssertEqual(scrollContentOverflow(area: area, contentFrames: rows, isVertical: false), -40)
88+
}
89+
90+
func testScrollOverflowCountsAChildFarOutsideTheContent() {
91+
let area = RectResponse(x: 0, y: 100, width: 400, height: 600)
92+
let rows = (0..<10).map { row in
93+
RectResponse(x: 20, y: 100 + Double(row) * 120, width: 360, height: 100)
94+
}
95+
let header = RectResponse(x: 0, y: -5000, width: 400, height: 40)
96+
XCTAssertEqual(
97+
scrollContentOverflow(area: area, contentFrames: rows + [header], isVertical: true), 5680)
98+
}
99+
100+
func testScrollOverflowIgnoresEmptyFrames() {
101+
let area = RectResponse(x: 0, y: 100, width: 400, height: 600)
102+
let document = RectResponse(x: 0, y: -400, width: 400, height: 2000)
103+
let empty = RectResponse(x: 0, y: -5000, width: 0, height: 0)
104+
XCTAssertEqual(
105+
scrollContentOverflow(area: area, contentFrames: [document, empty], isVertical: true), 1400)
106+
}
107+
81108
/// Chromium marks wrapper groups pressable; only a control role may win the press outright.
82109
func testWrapperGroupsAreNotPressableControls() {
83110
XCTAssertTrue(isPressableControlRole("AXButton"))

‎apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ - (NSString *)accessibilityValue {
8383
#endif
8484

8585
@interface AgentDeviceRunnerViewController : UIViewController
86+
@property(nonatomic, strong) UILabel *navigationBackStatus;
8687
@property(nonatomic, strong) UILabel *maskedToggleStatus;
8788
@property(nonatomic, strong) UILabel *maskedDeliveryStatus;
8889
@property(nonatomic, assign) NSUInteger maskedTogglePresses;
@@ -633,6 +634,10 @@ - (void)updateMaskedToggleStatus {
633634
}
634635
#endif
635636

637+
- (void)navigationBackPressed {
638+
self.navigationBackStatus.text = @"Back performed";
639+
}
640+
636641
- (void)viewDidLoad {
637642
[super viewDidLoad];
638643

@@ -652,6 +657,24 @@ - (void)viewDidLoad {
652657

653658
// Keep the fixture behind a launch argument so normal runner snapshots remain unchanged.
654659
#if TARGET_OS_IOS
660+
if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-back-regression"]) {
661+
self.navigationBackStatus = label;
662+
label.accessibilityIdentifier = @"agent-device-back-status";
663+
if ([NSProcessInfo.processInfo.arguments containsObject:@"--navigation-bar"]) {
664+
UINavigationBar *bar = [[UINavigationBar alloc] initWithFrame:CGRectMake(0, 62, self.view.bounds.size.width, 44)];
665+
UINavigationItem *item = [[UINavigationItem alloc] initWithTitle:@"Thread"];
666+
item.leftBarButtonItem = [[UIBarButtonItem alloc] initWithTitle:@"Go back" style:UIBarButtonItemStylePlain target:self action:@selector(navigationBackPressed)];
667+
bar.items = @[item];
668+
[self.view addSubview:bar];
669+
} else {
670+
UIButton *back = [UIButton buttonWithType:UIButtonTypeSystem];
671+
back.frame = CGRectMake(12, 62, 88, 44);
672+
[back setTitle:@"Go back" forState:UIControlStateNormal];
673+
[back addTarget:self action:@selector(navigationBackPressed) forControlEvents:UIControlEventTouchUpInside];
674+
[self.view addSubview:back];
675+
}
676+
}
677+
655678
if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-replacement-regression"]) {
656679
self.alertActionStatus = label;
657680
label.accessibilityIdentifier = @"agent-device-alert-actions";

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift‎

Lines changed: 41 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -60,14 +60,15 @@ extension RunnerTests {
6060
#else
6161
let buttons = app.navigationBars.buttons.allElementsBoundByIndex
6262
if let back = buttons.first(where: { $0.isHittable }) {
63+
// `isHittable` promises a hittable point, not a hittable center: a bar button partly under an
64+
// overlay keeps the element tap, which aims at the point XCTest computed.
6365
back.tap()
6466
return .performed
6567
}
6668
if isSnapshotXCTestChannelPenalized(bundleId: mainOwned.bundleId) {
6769
NSLog("AGENT_DEVICE_RUNNER_IN_APP_BACK_SKIPPED_XCTEST_ENUMERATION bundle=%@", mainOwned.bundleId ?? "")
68-
} else if let back = topNavigationBackElement(app: app) {
69-
tapElementCenter(app: app, element: back)
70-
return .performed
70+
} else if let frame = topNavigationBackFrame(app: app) {
71+
return tapResolvedNavigationBackControl(app: app, frame: frame)
7172
}
7273
return tapTopLeadingNavigationFallback(app: app)
7374
#endif
@@ -84,32 +85,48 @@ extension RunnerTests {
8485
#endif
8586
}
8687

87-
private func topNavigationBackElement(app: XCUIApplication) -> XCUIElement? {
88+
@MainActor
89+
private func tapResolvedNavigationBackControl(app: XCUIApplication, frame: CGRect) -> InAppBackOutcome {
90+
guard SnapshotGeometry.isPositiveFinite(frame) else { return .unavailable }
91+
let context = synthesizedCoordinateContext(
92+
app: app,
93+
policy: synthesizedGesturePolicy(.coordinateTap)
94+
)
95+
switch performSynthesizedGesture(app, kind: .coordinateTap, context: context, synthesize: {
96+
synthesizedTapAt(app: app, x: frame.midX, y: frame.midY, context: context)
97+
}) {
98+
case .performed:
99+
return .performed
100+
case .refused:
101+
return .unavailable
102+
case .xctestFallback:
103+
let fallback = performGesture(app) {
104+
tapAt(app: app, x: frame.midX, y: frame.midY)
105+
}
106+
if case .performed = fallback.outcome { return .performed }
107+
return .unavailable
108+
}
109+
}
110+
111+
private func topNavigationBackFrame(app: XCUIApplication) -> CGRect? {
88112
#if os(iOS)
89-
let frame = onScreenWindowFrame(app: app)
113+
let window = onScreenWindowFrame(app: app)
90114
let candidates = app.buttons.matching(Self.navigationBackPredicate()).allElementsBoundByIndex.compactMap {
91-
element -> (XCUIElement, Int)? in
115+
element -> (frame: CGRect, rank: Int)? in
92116
guard element.exists, element.isHittable else { return nil }
93-
guard Self.isTopNavigationBackCandidateFrame(
94-
element.frame,
95-
in: frame,
96-
identifier: element.identifier
97-
) else { return nil }
98-
guard let rank = Self.navigationBackControlRank(
99-
label: element.label,
100-
identifier: element.identifier
101-
) else {
102-
return nil
103-
}
104-
return (element, rank)
117+
let frame = element.frame
118+
let identifier = element.identifier
119+
guard Self.isTopNavigationBackCandidateFrame(frame, in: window, identifier: identifier)
120+
else { return nil }
121+
guard let rank = Self.navigationBackControlRank(label: element.label, identifier: identifier)
122+
else { return nil }
123+
return (frame, rank)
105124
}
106125
return candidates.sorted { lhs, rhs in
107-
if lhs.1 != rhs.1 { return lhs.1 < rhs.1 }
108-
let leftFrame = lhs.0.frame
109-
let rightFrame = rhs.0.frame
110-
if leftFrame.minY != rightFrame.minY { return leftFrame.minY < rightFrame.minY }
111-
return leftFrame.minX < rightFrame.minX
112-
}.first?.0
126+
if lhs.rank != rhs.rank { return lhs.rank < rhs.rank }
127+
if lhs.frame.minY != rhs.frame.minY { return lhs.frame.minY < rhs.frame.minY }
128+
return lhs.frame.minX < rhs.frame.minX
129+
}.first?.frame
113130
#else
114131
return nil
115132
#endif

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,67 @@
11
import XCTest
2+
#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS)
3+
import ObjectiveC.runtime
4+
#endif
25

36
#if AGENT_DEVICE_RUNNER_UNIT_TESTS
47
extension RunnerTests {
8+
#if os(iOS)
9+
@MainActor
10+
func testRankedInAppBackUsesSynthesisWithoutXCTestTap() throws {
11+
try assertInAppBackDispatch(navigationBar: false)
12+
}
13+
14+
/// A navigation bar button keeps XCTest's element tap, which aims at its computed hittable point.
15+
@MainActor
16+
func testNavigationBarBackKeepsTheElementTap() throws {
17+
try assertInAppBackDispatch(navigationBar: true)
18+
}
19+
20+
@MainActor
21+
private func assertInAppBackDispatch(navigationBar: Bool) throws {
22+
app.launchArguments = ["--agent-device-back-regression"] + (navigationBar ? ["--navigation-bar"] : [])
23+
app.launch()
24+
mainOwned.app = app
25+
mainOwned.bundleId = "com.callstack.agentdevice.runner"
26+
clearSnapshotXCTestChannelPenalty(reason: "navigation-test")
27+
defer {
28+
invalidateCachedTarget(reason: "unit_test_cleanup")
29+
app.terminate()
30+
}
31+
let status = app.staticTexts["agent-device-back-status"]
32+
XCTAssertTrue(status.waitForFixtureExistence(timeout: appExistenceTimeout))
33+
var xctestTaps = 0
34+
var restorations: [() -> Void] = []
35+
defer { restorations.reversed().forEach { $0() } }
36+
for type in [XCUIElement.self, XCUICoordinate.self] {
37+
let selector = NSSelectorFromString("tap")
38+
let method = try XCTUnwrap(class_getInstanceMethod(type, selector))
39+
let original = method_getImplementation(method)
40+
let replacement = imp_implementationWithBlock({ (target: AnyObject) in
41+
xctestTaps += 1
42+
typealias Tap = @convention(c) (AnyObject, Selector) -> Void
43+
unsafeBitCast(original, to: Tap.self)(target, selector)
44+
} as @convention(block) (AnyObject) -> Void)
45+
method_setImplementation(method, replacement)
46+
restorations.append {
47+
method_setImplementation(method, original)
48+
imp_removeBlock(replacement)
49+
}
50+
}
51+
52+
guard case .performed = tapInAppBackControl(app: app) else {
53+
return XCTFail("the selected back control must dispatch")
54+
}
55+
let changed = expectation(for: NSPredicate(format: "label == %@", "Back performed"), evaluatedWith: status)
56+
wait(for: [changed], timeout: appExistenceTimeout)
57+
if navigationBar {
58+
XCTAssertEqual(xctestTaps, 1, "a navigation bar button keeps the element tap")
59+
} else {
60+
XCTAssertEqual(xctestTaps, 0, "a ranked back control uses synthesized dispatch without idle waits")
61+
}
62+
}
63+
#endif
64+
565
func testTopLeadingNavigationFallbackPointTargetsHeaderControlBand() throws {
666
let point = try XCTUnwrap(
767
Self.topLeadingNavigationFallbackPoint(

‎apple/snapshot-bridge/README.md‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@ This directory contains the small private accessibility reader used by the
44
Apple platform acquisition facet. The framed server and request validation
55
live in `SnapshotBridge.m`; private runtime binding lives in
66
`SnapshotBridgeRuntime.m`. It is compiled for the iOS Simulator on first use
7-
and is never downloaded, pre-signed, or built by npm installation.
7+
and is never downloaded, pre-signed, or built by npm installation. It serves
8+
iPhone and iPad Simulators, which both run the iOS Simulator runtime (#3844).
89

910
The guest process uses the `XCTAccessibilityFramework` remote-access client
1011
from the simulator runtime and the `userTestingSnapshotForElement:options:error:`
@@ -49,6 +50,17 @@ not replace the native primary owner. An ownership refusal is evidence about
4950
the screen, so the route sends only that capture to XCTest and keeps the app
5051
generation on the bridge.
5152

53+
## Window placement
54+
55+
The snapshot API reports the frames of an app that runs in an iPadOS window
56+
(Windowed Apps or Stage Manager) relative to that window, not to the screen.
57+
Before acquisition, the reader compares the AXRuntime frame of the primary app
58+
with the frame of the system app. Both frames use the native space of the
59+
screen, also in landscape. If they differ, the reader returns `app-window-not-full-screen` without
60+
the app tree. Like an ownership refusal, this is evidence about the screen, so
61+
the route sends only that capture to XCTest. If the runtime cannot report either
62+
frame, the reader keeps the capture.
63+
5264
## Remote content
5365

5466
The reader snapshots one process. Two hosts put another process's UI into that

‎apple/snapshot-bridge/SnapshotBridgeRuntime.m‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
NSString *const kProtocolVersionKey = @"protocolVersion";
1919
NSString *const kSourceVersionKey = @"sourceVersion";
2020
NSString *const kRequestIdKey = @"requestId";
21-
NSString *const kSourceVersion = @"agent-device-simulator-ax-v1.8.0";
21+
NSString *const kSourceVersion = @"agent-device-simulator-ax-v1.9.0";
2222
const NSUInteger kProtocolVersion = 1;
2323
const uint32_t kMaximumFrameBytes = 16 * 1024 * 1024;
2424
const NSUInteger kMaximumDepth = 128;
@@ -69,7 +69,9 @@ + (nullable XCAccessibilityElement *)elementWithProcessIdentifier:(pid_t)pid;
6969

7070
@protocol AXForegroundElement <NSObject>
7171
+ (nullable id<AXForegroundElement>)primaryApp;
72+
+ (nullable id<AXForegroundElement>)systemApplication;
7273
- (pid_t)pid;
74+
- (CGRect)frame;
7375
@end
7476

7577
static NSNumber *finiteNumber(double value)
@@ -187,6 +189,31 @@ - (BOOL)isPrimaryForegroundProcess:(pid_t)pid
187189
return [application respondsToSelector:@selector(pid)] && [application pid] == pid;
188190
}
189191

192+
// The snapshot API reports the frames of an app that runs in an iPadOS window relative to that
193+
// window, so its tree cannot be placed on the screen. AXRuntime reports the primary app's window and
194+
// the system app, which spans the screen, in one native space. A window that differs from the screen
195+
// is refused; a runtime that cannot report either frame keeps the capture.
196+
- (BOOL)primaryAppWindowDiffersFromScreen
197+
{
198+
Class<AXForegroundElement> elementClass = (Class<AXForegroundElement>)objc_lookUpClass("AXElement");
199+
if (![elementClass respondsToSelector:@selector(primaryApp)] ||
200+
![elementClass respondsToSelector:@selector(systemApplication)]) {
201+
return NO;
202+
}
203+
id<AXForegroundElement> application = [elementClass primaryApp];
204+
id<AXForegroundElement> system = [elementClass systemApplication];
205+
if (![application respondsToSelector:@selector(frame)] || ![system respondsToSelector:@selector(frame)]) {
206+
return NO;
207+
}
208+
CGRect window = [application frame];
209+
CGRect screen = [system frame];
210+
if (CGRectIsEmpty(window) || CGRectIsEmpty(screen)) return NO;
211+
return fabs(CGRectGetMinX(window) - CGRectGetMinX(screen)) > 0.5 ||
212+
fabs(CGRectGetMinY(window) - CGRectGetMinY(screen)) > 0.5 ||
213+
fabs(CGRectGetWidth(window) - CGRectGetWidth(screen)) > 0.5 ||
214+
fabs(CGRectGetHeight(window) - CGRectGetHeight(screen)) > 0.5;
215+
}
216+
190217
- (nullable id)jsonValue:(id)value name:(NSString *)name
191218
{
192219
if (!value || value == [NSNull null]) return nil;
@@ -383,6 +410,11 @@ - (nullable NSDictionary *)snapshotForProcess:(pid_t)pid
383410
finishRequestWatchdog(watchdog, watchdogState);
384411
return nil;
385412
}
413+
if ([self primaryAppWindowDiffersFromScreen]) {
414+
if (error) *error = failureResponse(requestId, @"unsupported", @"app-window-not-full-screen", @"target app window does not fill the screen");
415+
finishRequestWatchdog(watchdog, watchdogState);
416+
return nil;
417+
}
386418
snapshot = captureSnapshotTree((__bridge id)raw, maxDepth, maxNodes, nativeLevelsHint,
387419
^id(id element, NSUInteger depth, NSUInteger nodes, NSError **captureError) {
388420
if (![self isPrimaryForegroundProcess:pid]) {

0 commit comments

Comments
 (0)