Skip to content

Commit f3ab9cc

Browse files
thymikeeclaude
andcommitted
refactor(interaction): read the tap's point observation from preAction membership
The resolved target carried the same tree twice: `preAction` and `pointObservation`. tapCommand now forwards `preAction` as the point observation only when the dispatched node is one of its nodes. The #1542 rescue returns a patched copy, so a rescued point has no observation, and any future node-replacing stage fails closed the same way. The diagnostic `source` label leaves the wire, so the daemon no longer needs the `tapTarget` runtime input or its `selector-capture` default, and `target` returns to `iosTapCorroboration`. The runner log keeps `reason=`. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent fc9e85b commit f3ab9cc

22 files changed

Lines changed: 62 additions & 173 deletions

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -161,10 +161,9 @@ extension RunnerTests {
161161
// Clear before dispatch so a throwing gesture cannot retain the preceding field's witness.
162162
clearRememberedTextEntryTap()
163163
NSLog(
164-
"AGENT_DEVICE_RUNNER_COORDINATE_TAP_TEXT_INPUT_PROBE_SKIPPED bundle=%@ reason=%@ source=%@",
164+
"AGENT_DEVICE_RUNNER_COORDINATE_TAP_TEXT_INPUT_PROBE_SKIPPED bundle=%@ reason=%@",
165165
mainOwned.bundleId ?? "",
166-
xCTestChannelPenalized ? "penalized-xctest-channel" : "evidence",
167-
command.coordinateTapTextInputEvidence?.source ?? "none"
166+
xCTestChannelPenalized ? "penalized-xctest-channel" : "evidence"
168167
)
169168
}
170169
var fallback: GestureFallback?

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -214,7 +214,6 @@ extension Command {
214214
}
215215

216216
struct CoordinateTapTextInputEvidence: Codable {
217-
let source: String
218217
let targetBundleId: String
219218
let point: RunnerGesturePoint
220219
let textInputRects: [RunnerGestureViewport]

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift‎

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -106,21 +106,6 @@ extension RunnerTests {
106106
XCTAssertFalse(shouldProbeCoordinateTapTextInput(xCTestChannelPenalized: true, evidence: nil, point: nil, targetBundleId: nil))
107107
}
108108

109-
func testCoordinateTapTextInputEvidenceSourceIsDiagnosticOnly() {
110-
let evidence = CoordinateTapTextInputEvidence(
111-
source: "unverified-daemon-tag",
112-
targetBundleId: "com.example.app",
113-
point: RunnerGesturePoint(x: 10, y: 20),
114-
textInputRects: []
115-
)
116-
XCTAssertFalse(shouldProbeCoordinateTapTextInput(
117-
xCTestChannelPenalized: false,
118-
evidence: evidence,
119-
point: CGPoint(x: 10, y: 20),
120-
targetBundleId: "com.example.app"
121-
))
122-
}
123-
124109
func testOnlySynthesizedSequenceTapStepsTakeTheCoordinateTapPolicy() {
125110
XCTAssertEqual(synthesizedPolicyKind(forSequenceStep: sequenceStep("tap", synthesized: true)), .coordinateTap)
126111
XCTAssertNil(synthesizedPolicyKind(forSequenceStep: sequenceStep("tap", synthesized: nil)))

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ extension RunnerTests {
110110
let frame = button.frame
111111
textInputProbeIssueForTesting = XCTIssue(type: .assertionFailure, compactDescription: "excluded point must not query text inputs")
112112
let tap = try runnerCommandFixture(
113-
#"{"appBundleId":"com.callstack.agentdevice.runner","command":"tap","commandId":"tap-excluded-point","x":\#(frame.midX),"y":\#(frame.midY),"synthesized":true,"coordinateTapTextInputEvidence":{"source":"ref-frame","targetBundleId":"com.callstack.agentdevice.runner","point":{"x":\#(frame.midX),"y":\#(frame.midY)},"textInputRects":[{"x":\#(fieldFrame.minX),"y":\#(fieldFrame.minY),"width":\#(fieldFrame.width),"height":\#(fieldFrame.height)}]}}"#
113+
#"{"appBundleId":"com.callstack.agentdevice.runner","command":"tap","commandId":"tap-excluded-point","x":\#(frame.midX),"y":\#(frame.midY),"synthesized":true,"coordinateTapTextInputEvidence":{"targetBundleId":"com.callstack.agentdevice.runner","point":{"x":\#(frame.midX),"y":\#(frame.midY)},"textInputRects":[{"x":\#(fieldFrame.minX),"y":\#(fieldFrame.minY),"width":\#(fieldFrame.width),"height":\#(fieldFrame.height)}]}}"#
114114
)
115115
let response = try execute(command: tap)
116116
XCTAssertTrue(response.ok, String(describing: response.error))
@@ -142,7 +142,7 @@ extension RunnerTests {
142142
mainOwned.processIdentifier = try XCTUnwrap(Self.processIdentifier(of: app))
143143
clearSnapshotXCTestChannelPenalty(reason: "fresh-runner")
144144
let tap = try runnerCommandFixture(
145-
#"{"appBundleId":"com.callstack.agentdevice.runner","command":"tap","commandId":"tap-input-evidence","x":\#(frame.midX),"y":\#(frame.midY),"synthesized":true,"coordinateTapTextInputEvidence":{"source":"selector-capture","targetBundleId":"com.callstack.agentdevice.runner","point":{"x":\#(frame.midX),"y":\#(frame.midY)},"textInputRects":[{"x":\#(frame.minX),"y":\#(frame.minY),"width":\#(frame.width),"height":\#(frame.height)}]}}"#
145+
#"{"appBundleId":"com.callstack.agentdevice.runner","command":"tap","commandId":"tap-input-evidence","x":\#(frame.midX),"y":\#(frame.midY),"synthesized":true,"coordinateTapTextInputEvidence":{"targetBundleId":"com.callstack.agentdevice.runner","point":{"x":\#(frame.midX),"y":\#(frame.midY)},"textInputRects":[{"x":\#(frame.minX),"y":\#(frame.minY),"width":\#(frame.width),"height":\#(frame.height)}]}}"#
146146
)
147147
XCTAssertTrue(try execute(command: tap).ok)
148148
XCTAssertNotNil(textEntryTapWitness)

‎contracts/fixtures/coordinate-tap-text-input.json‎

Lines changed: 0 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@
2828
"appBundleId": "com.example.app",
2929
"activeBundleId": "com.example.app",
3030
"expectedEvidence": {
31-
"source": "selector-capture",
3231
"targetBundleId": "com.example.app",
3332
"point": {
3433
"x": 60,
@@ -76,7 +75,6 @@
7675
"appBundleId": "com.example.app",
7776
"activeBundleId": "com.example.app",
7877
"expectedEvidence": {
79-
"source": "selector-capture",
8078
"targetBundleId": "com.example.app",
8179
"point": {
8280
"x": 60,
@@ -124,7 +122,6 @@
124122
"appBundleId": "com.example.app",
125123
"activeBundleId": "com.example.app",
126124
"expectedEvidence": {
127-
"source": "selector-capture",
128125
"targetBundleId": "com.example.app",
129126
"point": {
130127
"x": 60,
@@ -172,7 +169,6 @@
172169
"appBundleId": "com.example.app",
173170
"activeBundleId": "com.example.app",
174171
"expectedEvidence": {
175-
"source": "selector-capture",
176172
"targetBundleId": "com.example.app",
177173
"point": {
178174
"x": 60,
@@ -220,7 +216,6 @@
220216
"appBundleId": "com.example.app",
221217
"activeBundleId": "com.example.app",
222218
"expectedEvidence": {
223-
"source": "selector-capture",
224219
"targetBundleId": "com.example.app",
225220
"point": {
226221
"x": 60,
@@ -269,7 +264,6 @@
269264
"appBundleId": "com.example.app",
270265
"activeBundleId": "com.example.app",
271266
"expectedEvidence": {
272-
"source": "selector-capture",
273267
"targetBundleId": "com.example.app",
274268
"point": {
275269
"x": 60,
@@ -306,7 +300,6 @@
306300
"appBundleId": "com.example.app",
307301
"activeBundleId": "com.example.app",
308302
"expectedEvidence": {
309-
"source": "selector-capture",
310303
"targetBundleId": "com.example.app",
311304
"point": {
312305
"x": 60,
@@ -343,7 +336,6 @@
343336
"appBundleId": "com.example.app",
344337
"activeBundleId": "com.example.app",
345338
"expectedEvidence": {
346-
"source": "selector-capture",
347339
"targetBundleId": "com.example.app",
348340
"point": {
349341
"x": 60,
@@ -373,7 +365,6 @@
373365
"appBundleId": "com.example.app",
374366
"activeBundleId": "com.example.app",
375367
"expectedEvidence": {
376-
"source": "selector-capture",
377368
"targetBundleId": "com.example.app",
378369
"point": {
379370
"x": 60,
@@ -449,7 +440,6 @@
449440
"appBundleId": "com.example.app",
450441
"activeBundleId": "com.apple.SafariViewService",
451442
"expectedEvidence": {
452-
"source": "selector-capture",
453443
"targetBundleId": "com.example.app",
454444
"point": {
455445
"x": 60,
@@ -479,7 +469,6 @@
479469
"appBundleId": "com.example.app",
480470
"activeBundleId": "com.example.app",
481471
"expectedEvidence": {
482-
"source": "selector-capture",
483472
"targetBundleId": "com.example.app",
484473
"point": {
485474
"x": 60,
@@ -493,44 +482,6 @@
493482
"y": 40
494483
}
495484
},
496-
{
497-
"name": "ref-frame-away-from-input",
498-
"nodes": [
499-
{
500-
"type": "TextField",
501-
"rect": {
502-
"x": 10,
503-
"y": 120,
504-
"width": 100,
505-
"height": 40
506-
}
507-
}
508-
],
509-
"point": {
510-
"x": 60,
511-
"y": 40
512-
},
513-
"appBundleId": "com.example.app",
514-
"activeBundleId": "com.example.app",
515-
"expectedEvidence": {
516-
"source": "ref-frame",
517-
"targetBundleId": "com.example.app",
518-
"point": {
519-
"x": 60,
520-
"y": 40
521-
},
522-
"textInputRects": [
523-
{
524-
"x": 10,
525-
"y": 120,
526-
"width": 100,
527-
"height": 40
528-
}
529-
]
530-
},
531-
"probe": false,
532-
"source": "ref-frame"
533-
},
534485
{
535486
"name": "UITextField-without-automation-type",
536487
"nodes": [
@@ -551,7 +502,6 @@
551502
"appBundleId": "com.example.app",
552503
"activeBundleId": "com.example.app",
553504
"expectedEvidence": {
554-
"source": "selector-capture",
555505
"targetBundleId": "com.example.app",
556506
"point": {
557507
"x": 60,
@@ -588,7 +538,6 @@
588538
"appBundleId": "com.example.app",
589539
"activeBundleId": "com.example.app",
590540
"expectedEvidence": {
591-
"source": "selector-capture",
592541
"targetBundleId": "com.example.app",
593542
"point": {
594543
"x": 60,
@@ -625,7 +574,6 @@
625574
"appBundleId": "com.example.app",
626575
"activeBundleId": "com.example.app",
627576
"expectedEvidence": {
628-
"source": "selector-capture",
629577
"targetBundleId": "com.example.app",
630578
"point": {
631579
"x": 60,
@@ -662,7 +610,6 @@
662610
"appBundleId": "com.example.app",
663611
"activeBundleId": "com.example.app",
664612
"expectedEvidence": {
665-
"source": "selector-capture",
666613
"targetBundleId": "com.example.app",
667614
"point": {
668615
"x": 60,
@@ -699,7 +646,6 @@
699646
"appBundleId": "com.example.app",
700647
"activeBundleId": "com.example.app",
701648
"expectedEvidence": {
702-
"source": "selector-capture",
703649
"targetBundleId": "com.example.app",
704650
"point": {
705651
"x": 59,
@@ -736,7 +682,6 @@
736682
"appBundleId": "com.example.app",
737683
"activeBundleId": "com.example.app",
738684
"expectedEvidence": {
739-
"source": "selector-capture",
740685
"targetBundleId": "com.example.app",
741686
"point": {
742687
"x": 57,

‎contracts/fixtures/runner-requests.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
{"name": "ios-simulator.interactions-single-press-double-tap.single", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "sequence", "steps": [{"kind": "doubleTap", "x": 10, "y": 20}], "appBundleId": "com.example.app", "commandId": "<commandId>"}},
2929
{"name": "ios-simulator.interactions-single-press-long-press.hold", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "longPress", "x": 10, "y": 20, "durationMs": 600, "appBundleId": "com.example.app", "commandId": "<commandId>"}},
3030
{"name": "ios-simulator.interactions-single-press-tap.synthesized", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "tap", "x": 10, "y": 20, "synthesized": true, "appBundleId": "com.example.app", "commandId": "<commandId>"}},
31-
{"name": "ios-simulator.interactions-single-press-tap.text-input-evidence", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "tap", "x": 10, "y": 20, "coordinateTapTextInputEvidence": {"source": "selector-capture", "targetBundleId": "com.example.app", "point": {"x": 10, "y": 20}, "textInputRects": [{"x": 0, "y": 100, "width": 200, "height": 44}]}, "synthesized": true, "appBundleId": "com.example.app", "commandId": "<commandId>"}},
31+
{"name": "ios-simulator.interactions-single-press-tap.text-input-evidence", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "tap", "x": 10, "y": 20, "coordinateTapTextInputEvidence": {"targetBundleId": "com.example.app", "point": {"x": 10, "y": 20}, "textInputRects": [{"x": 0, "y": 100, "width": 200, "height": 44}]}, "synthesized": true, "appBundleId": "com.example.app", "commandId": "<commandId>"}},
3232
{"name": "ios-simulator.interactions-tap-element-selector.expected-point", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "tap", "selectorKey": "label", "selectorValue": "Go", "allowNonHittableCoordinateFallback": true, "x": 10, "y": 20, "synthesized": true, "appBundleId": "com.example.app", "commandId": "<commandId>"}},
3333
{"name": "ios-simulator.interactions-tap.synthesized", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "tap", "x": 10, "y": 20, "synthesized": true, "appBundleId": "com.example.app", "commandId": "<commandId>"}},
3434
{"name": "ios-simulator.interactions-type.append", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "type", "text": "hello", "delayMs": 10, "textEntryMode": "append", "appBundleId": "com.example.app", "commandId": "<commandId>"}},

‎packages/contracts/src/interaction.ts‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -181,11 +181,6 @@ export type ResolvedInteractionTarget =
181181
resolution?: ResolutionDisclosure;
182182
recordingTarget?: RecordingTargetOverride;
183183
preAction?: SurfaceScopedNodes;
184-
/**
185-
* The observation whose geometry produced `point`. Absent when a live read replaced that
186-
* observation's off-screen rect (#1542); the dispatch then has no observation of its point.
187-
*/
188-
pointObservation?: SurfaceScopedNodes;
189184
/**
190185
* ADR 0014, "Mutation admission and read behavior": the source tree of the dispatched node, reported by the
191186
* reconciliation that chose it. `'observation'` means the session's newest capture, so the
@@ -209,11 +204,6 @@ export type ResolvedInteractionTarget =
209204
resolution?: ResolutionDisclosure;
210205
recordingTarget?: RecordingTargetOverride;
211206
preAction?: SurfaceScopedNodes;
212-
/**
213-
* The observation whose geometry produced `point`. Absent when a live read replaced that
214-
* observation's off-screen rect (#1542); the dispatch then has no observation of its point.
215-
*/
216-
pointObservation?: SurfaceScopedNodes;
217207
};
218208

219209
/**

‎packages/contracts/src/interactor-types.ts‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -89,10 +89,8 @@ export type ElementSelectorTapOptions = {
8989
expectedPoint?: Point;
9090
};
9191

92-
/** Facts from a request's resolution capture or active ref frame; the native owner decides admission. */
92+
/** Text-input geometry from the capture that measured the tap point; the native owner decides admission. */
9393
export type CoordinateTapTextInputEvidence = Readonly<{
94-
/** Diagnostics only; admission depends on the target, dispatch point, and acquired geometry. */
95-
source: 'selector-capture' | 'ref-frame';
9694
targetBundleId: string;
9795
point: Point;
9896
textInputRects: readonly Rect[];

‎packages/daemon/src/__tests__/support/touch-command-surface-fake.ts‎

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,9 @@ import {
5454
* alternatives, each a conjunction of `key=value` terms). A target that does not resolve refuses
5555
* with the typed `ref_not_found` / `selector_not_found` reason and `dispatched: 'no'`. A selector
5656
* node outside the capture's root rect dispatches only at a live rect the backend's #1542 read
57-
* confirms, without a point observation; otherwise it refuses with `offscreen_selector`. Promotion,
58-
* occlusion, the real visibility rule, bounds guards, label recovery, and `--verify` evidence are
59-
* command-family behavior and are asserted beside that code.
57+
* confirms, as a patched copy that is not one of the capture's nodes; otherwise it refuses with
58+
* `offscreen_selector`. Promotion, occlusion, the real visibility rule, bounds guards, label
59+
* recovery, and `--verify` evidence are command-family behavior and are asserted beside that code.
6060
*
6161
* Values the fake supplies itself rather than reading from a backend: an `is` read always passes
6262
* (`pass: true`, `matches: 1`); selector reads (`get`, `is`) capture once with
@@ -380,14 +380,13 @@ async function resolveTarget(
380380
}
381381
const liveRect = await readRescuedRect(config, tree.nodes, node, target.selector, options);
382382
const dispatched = liveRect ? { ...node, rect: liveRect } : node;
383-
const observed = observedAt(tree);
384383
const resolved = {
385384
kind: 'selector' as const,
386385
target: { kind: 'selector' as const, selector: target.selector },
387386
point: centerOf(dispatched) as Point,
388387
node: dispatched,
389388
selectorChain: selectorChainFor(config, node, tree.nodes, chainAction(nativeRefRoute)),
390-
...(liveRect ? { preAction: observed.preAction } : observed),
389+
preAction: { nodes: tree.nodes },
391390
};
392391
resolvedTreeQuality.set(resolved, tree.snapshotQuality);
393392
return resolved;
@@ -447,18 +446,12 @@ function resolvedFromNode(
447446
point: centerOf(node),
448447
node,
449448
selectorChain: selectorChainFor(config, node, tree.nodes, action),
450-
...observedAt(tree),
449+
preAction: { nodes: tree.nodes },
451450
};
452451
resolvedTreeQuality.set(resolved, tree.snapshotQuality);
453452
return resolved;
454453
}
455454

456-
/** The tree a point was measured on: both its pre-action baseline and its point observation. */
457-
function observedAt(tree: Pick<SnapshotState, 'nodes'>) {
458-
const observation = { nodes: tree.nodes };
459-
return { preAction: observation, pointObservation: observation };
460-
}
461-
462455
function refBody(ref: string): string {
463456
return ref.replace(/^@/, '').replace(/~s\d+$/, '');
464457
}
@@ -532,7 +525,11 @@ async function dispatchTap(
532525
if (resolved.kind === 'ref' && !resolved.point && config.backend.tapTarget) {
533526
return await config.backend.tapTarget(context(config, options), resolved.target, tapOptions);
534527
}
535-
const pointObservation = resolved.kind === 'point' ? undefined : resolved.pointObservation;
528+
// tapCommand forwards the pre-action tree only when the dispatched node is one of its nodes.
529+
const pointObservation =
530+
resolved.kind !== 'point' && resolved.node && resolved.preAction?.nodes.includes(resolved.node)
531+
? resolved.preAction
532+
: undefined;
536533
return await backendOperation(config.backend.tap?.bind(config.backend))(
537534
context(config, options),
538535
requirePointOf(resolved),

‎packages/daemon/src/execution/interaction/internal/coordinate-tap-text-input-evidence.test.ts‎

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@ type Fixture = {
1313
nodes: Omit<RawSnapshotNode, 'index'>[];
1414
point: { x: number; y: number };
1515
appBundleId: string;
16-
source?: CoordinateTapTextInputEvidence['source'];
1716
truncated?: boolean;
1817
expectedEvidence: CoordinateTapTextInputEvidence | null;
1918
};
@@ -54,23 +53,17 @@ test.each(cases)('$name preserves acquired input geometry through presentation',
5453
surfaceScopedNodes(snapshot),
5554
fixture.point,
5655
fixture.appBundleId,
57-
fixture.source ?? 'selector-capture',
5856
);
5957
expect(result ?? null).toEqual(fixture.expectedEvidence);
6058
});
6159

6260
test('an observation without acquisition geometry or a target bundle cannot establish exclusion', () => {
6361
const point = { x: 10, y: 20 };
64-
expect(
65-
buildCoordinateTapTextInputEvidence({ nodes: [] }, point, 'app', 'selector-capture'),
66-
).toBeUndefined();
62+
expect(buildCoordinateTapTextInputEvidence({ nodes: [] }, point, 'app')).toBeUndefined();
6763
const measured = { nodes: [{ index: 0, type: 'Button' }] };
6864
attachSnapshotTextInputGeometry(measured.nodes, { rects: [] });
69-
expect(
70-
buildCoordinateTapTextInputEvidence(measured, point, undefined, 'selector-capture'),
71-
).toBeUndefined();
72-
expect(buildCoordinateTapTextInputEvidence(measured, point, 'app', 'selector-capture')).toEqual({
73-
source: 'selector-capture',
65+
expect(buildCoordinateTapTextInputEvidence(measured, point, undefined)).toBeUndefined();
66+
expect(buildCoordinateTapTextInputEvidence(measured, point, 'app')).toEqual({
7467
targetBundleId: 'app',
7568
point,
7669
textInputRects: [],

0 commit comments

Comments
 (0)