Skip to content

Commit e93c795

Browse files
committed
fix: cover selector routes in device replays
1 parent d95c85f commit e93c795

11 files changed

Lines changed: 349 additions & 15 deletions

File tree

‎examples/test-app/app.config.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ module.exports = {
3737
android: {
3838
package: 'com.callstack.agentdevicelab',
3939
predictiveBackGestureEnabled: false,
40+
softwareKeyboardLayoutMode: 'pan',
4041
},
4142
},
4243
};

‎packages/ad-script/src/internal/__tests__/script.test.ts‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,33 @@ test('formatPortableActionLine preserves inline open runtime hints', () => {
5050
);
5151
});
5252

53+
test('open replay script round-trips explicit Android test IME selection', () => {
54+
const actions: SessionAction[] = [
55+
{
56+
ts: Date.now(),
57+
command: 'open',
58+
positionals: ['Demo'],
59+
flags: { testIme: false },
60+
},
61+
{
62+
ts: Date.now(),
63+
command: 'open',
64+
positionals: ['Demo'],
65+
flags: { testIme: true },
66+
},
67+
];
68+
69+
const script = formatReplayScriptForTest(actions);
70+
assert.match(script, /open "Demo" --no-test-ime/);
71+
assert.match(script, /open "Demo" --test-ime/);
72+
73+
const parsed = parseReplayScriptDetailed(script).actions;
74+
assert.equal(parsed[0]?.flags.testIme, false);
75+
assert.equal(parsed[1]?.flags.testIme, true);
76+
assert.deepEqual(parsed[0]?.positionals, ['Demo']);
77+
assert.deepEqual(parsed[1]?.positionals, ['Demo']);
78+
});
79+
5380
test('record replay script parses fps, quality, and hide-touches flags', () => {
5481
const script = 'record start "./capture.mp4" --fps 24 --quality high --hide-touches\n';
5582
const parsed = parseReplayScriptDetailed(script).actions;

‎packages/ad-script/src/internal/open-script.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,11 @@ export function appendOpenActionScriptArgs(
4646
if (action.flags?.relaunch) {
4747
parts.push('--relaunch');
4848
}
49+
if (action.flags?.testIme === true) {
50+
parts.push('--test-ime');
51+
} else if (action.flags?.testIme === false) {
52+
parts.push('--no-test-ime');
53+
}
4954
appendRuntimeHintFlags(parts, action.runtime);
5055
}
5156

@@ -61,6 +66,14 @@ export function parseReplayOpenFlags(args: string[]): {
6166
flags.relaunch = true;
6267
continue;
6368
}
69+
if (token === '--test-ime') {
70+
flags.testIme = true;
71+
continue;
72+
}
73+
if (token === '--no-test-ime') {
74+
flags.testIme = false;
75+
continue;
76+
}
6477
argsWithoutRelaunch.push(token);
6578
}
6679
const parsedRuntime = parseReplayRuntimeFlags(argsWithoutRelaunch);
Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
import assert from 'node:assert/strict';
2+
import { test } from 'vitest';
3+
4+
import type { RawSnapshotNode } from '@agent-device/kernel/snapshot';
5+
import { isAndroidInputMethodSnapshotNode } from '../android-input-method-overlays.ts';
6+
import { annotateCoveredSnapshotNodes } from '../snapshot-occlusion.ts';
7+
8+
test('an Android input-method window covers app targets emitted after it', () => {
9+
const nodes: RawSnapshotNode[] = [
10+
{
11+
index: 0,
12+
type: 'android.widget.FrameLayout',
13+
label: 'Q',
14+
identifier: 'com.google.android.inputmethod.latin:id/key_pos_0_0',
15+
bundleId: 'com.google.android.inputmethod.latin',
16+
rect: { x: 50, y: 500, width: 100, height: 100 },
17+
hittable: true,
18+
},
19+
{
20+
index: 1,
21+
type: 'android.widget.Button',
22+
label: 'Covered app target',
23+
identifier: 'covered-app-target',
24+
bundleId: 'com.example.app',
25+
rect: { x: 0, y: 450, width: 200, height: 200 },
26+
hittable: true,
27+
},
28+
];
29+
30+
const annotated = annotateCoveredSnapshotNodes(nodes, {
31+
isAdditionalOverlayNode: isAndroidInputMethodSnapshotNode,
32+
});
33+
34+
assert.equal(annotated[0]?.interactionBlocked, undefined);
35+
assert.equal(annotated[1]?.interactionBlocked, 'covered');
36+
assert.equal(annotated[1]?.hittable, false);
37+
});

‎src/snapshot/snapshot-occlusion.ts‎

Lines changed: 26 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -131,35 +131,46 @@ function findCoveringNode(
131131
): RawSnapshotNode | null {
132132
const cached = scan.coverCache.get(targetPosition);
133133
if (cached !== undefined) return cached;
134-
// Reentrancy guard: `visibleCoverRect` recurses into `findCoveringNode` for
135-
// the SAME targetPosition only through a cycle in `overlayPositions`
136-
// ordering, which cannot happen (positions strictly increase along any
137-
// recursive path — see the `position <= targetPosition` filter below) —
138-
// but seed `null` before recursing regardless, so a future edit that
139-
// breaks that invariant fails closed (no cover) instead of re-entering.
134+
// Reentrancy guard: an additional Android overlay may precede an app target
135+
// in the cross-window traversal, while that target may itself be overlay-like.
136+
// Seed `null` before following either direction so such a cycle fails closed
137+
// instead of re-entering.
140138
scan.coverCache.set(targetPosition, null);
141139

142140
const targetRect = positiveRect(target.rect);
143141
if (!targetRect) return finishFindCoveringNode(scan, targetPosition, null);
144142
const center = centerOfRect(targetRect);
143+
const targetIsAdditionalOverlay = options.isAdditionalOverlayNode?.(target) === true;
145144

146-
// Mutation-lane note: relaxing `<=` to `<` here would only change behavior
147-
// if `position === targetPosition` were reachable — a node covering
148-
// itself. That case is already excluded one line below regardless: a
149-
// self-candidate has `candidateRect === targetRect` (same node, same
150-
// object), so `areRectsApproximatelyEqual` in `visibleCoverRect` always
151-
// excludes it.
152145
for (const position of scan.overlayPositions) {
153-
if (position <= targetPosition) continue;
154-
const candidate = scan.nodes[position];
155-
if (candidate && canCoverPoint(scan, position, target, targetRect, center, options)) {
146+
if (
147+
!canPositionCoverTarget(scan, position, targetPosition, targetIsAdditionalOverlay, options)
148+
) {
149+
continue;
150+
}
151+
const candidate = scan.nodes[position]!;
152+
if (canCoverPoint(scan, position, target, targetRect, center, options)) {
156153
return finishFindCoveringNode(scan, targetPosition, candidate);
157154
}
158155
}
159156

160157
return finishFindCoveringNode(scan, targetPosition, null);
161158
}
162159

160+
function canPositionCoverTarget(
161+
scan: OcclusionScan,
162+
candidatePosition: number,
163+
targetPosition: number,
164+
targetIsAdditionalOverlay: boolean,
165+
options: SnapshotOcclusionOptions,
166+
): boolean {
167+
const candidate = scan.nodes[candidatePosition];
168+
if (!candidate) return false;
169+
if (candidatePosition > targetPosition) return true;
170+
if (candidatePosition === targetPosition || targetIsAdditionalOverlay) return false;
171+
return options.isAdditionalOverlayNode?.(candidate) === true;
172+
}
173+
163174
function finishFindCoveringNode(
164175
scan: OcclusionScan,
165176
targetPosition: number,

‎test/integration/android-emulator-e2e/live-replay-scenarios.ts‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import {
1515
replayAttemptTimeoutMs,
1616
} from '../live-device-e2e/replay-evidence.ts';
1717
import { runLiveReplayTestSuite } from '../live-device-e2e/replay-suite.ts';
18+
import { assertSelectorRouteReplay } from '../live-device-e2e/selector-route-replay.ts';
1819

1920
const C = PUBLIC_COMMANDS;
2021

@@ -97,4 +98,39 @@ export async function assertFixtureReplays(context: LiveContext): Promise<void>
9798
'helper-backed-gesture-recovery',
9899
'gesture fixture observed two-pointer pan, fling, pinch, rotate, and transform outcomes',
99100
);
101+
102+
await assertSelectorRouteReplay(
103+
context,
104+
path.resolve(
105+
'test/integration/replays/android/fixture/02-selector-routes-covered-diagnosis.ad',
106+
),
107+
runStep,
108+
{
109+
postReplayCleanup: [
110+
{
111+
step: 'open Android cleanup session after focused selector-route replay',
112+
args: [
113+
'open',
114+
'settings',
115+
'--platform',
116+
'android',
117+
'--serial',
118+
context.serial,
119+
'--session',
120+
`${context.session}-cleanup`,
121+
'--no-test-ime',
122+
'--json',
123+
],
124+
},
125+
{
126+
step: 'restore portrait after focused selector-route replay',
127+
args: ['orientation', 'portrait', '--session', `${context.session}-cleanup`, '--json'],
128+
},
129+
{
130+
step: 'close Android cleanup session after focused selector-route replay',
131+
args: ['close', '--session', `${context.session}-cleanup`, '--json'],
132+
},
133+
],
134+
},
135+
);
100136
}

‎test/integration/ios-simulator-e2e/live-replay-scenarios.ts‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import {
1515
replayAttemptTimeoutMs,
1616
} from '../live-device-e2e/replay-evidence.ts';
1717
import { runLiveReplayTestSuite } from '../live-device-e2e/replay-suite.ts';
18+
import { assertSelectorRouteReplay } from '../live-device-e2e/selector-route-replay.ts';
1819

1920
const C = PUBLIC_COMMANDS;
2021

@@ -75,4 +76,41 @@ export async function assertFixtureReplays(context: LiveContext): Promise<void>
7576
'gesture fixture counters prove pan/fling/pinch/rotate/transform',
7677
);
7778
verifyCommand(context, C.test, 'fixture scripts pass as a suite and emit non-empty JUnit');
79+
80+
await assertSelectorRouteReplay(
81+
context,
82+
path.resolve('test/integration/replays/ios/fixture/03-selector-routes-covered-diagnosis.ad'),
83+
runStep,
84+
{
85+
postReplayCleanup: [
86+
{
87+
step: 'open iOS cleanup session after focused selector-route replay',
88+
args: [
89+
'open',
90+
context.appId,
91+
'--platform',
92+
'ios',
93+
'--udid',
94+
context.udid,
95+
'--session',
96+
`${context.session}-cleanup`,
97+
'--state-dir',
98+
context.stateDir,
99+
'--json',
100+
],
101+
},
102+
{
103+
step: 'close iOS cleanup session after focused selector-route replay',
104+
args: [
105+
'close',
106+
'--session',
107+
`${context.session}-cleanup`,
108+
'--state-dir',
109+
context.stateDir,
110+
'--json',
111+
],
112+
},
113+
],
114+
},
115+
);
78116
}
Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
import assert from 'node:assert/strict';
2+
3+
import type { CliJsonResult } from '../cli-json.ts';
4+
import {
5+
assertReplayCommands,
6+
readReplayCommands,
7+
replayAttemptTimeoutMs,
8+
} from './replay-evidence.ts';
9+
10+
type SelectorRouteReplayStep<Context> = (
11+
context: Context,
12+
step: string,
13+
args: string[],
14+
options?: {
15+
allowFailure?: boolean;
16+
commonFlags?: boolean;
17+
expectFailure?: boolean;
18+
timeoutMs?: number;
19+
},
20+
) => Promise<CliJsonResult>;
21+
22+
export async function assertSelectorRouteReplay<
23+
Context extends { artifactDir: string; session: string; sessionOpen: boolean },
24+
>(
25+
context: Context,
26+
replayPath: string,
27+
runStep: SelectorRouteReplayStep<Context>,
28+
options: { postReplayCleanup?: { args: string[]; step: string }[] } = {},
29+
): Promise<void> {
30+
const commands = readReplayCommands(replayPath);
31+
assertReplayCommands(replayPath, commands, ['get', 'is', 'find', 'click']);
32+
const daemonTimeoutMs = replayAttemptTimeoutMs(replayPath) + 30_000;
33+
34+
try {
35+
const replay = await runStep(
36+
context,
37+
'run focused selector-route replay through covered-target diagnosis',
38+
['replay', replayPath, '--timeout', String(daemonTimeoutMs)],
39+
{ expectFailure: true, timeoutMs: daemonTimeoutMs + 30_000 },
40+
);
41+
assertCoveredTargetDivergence(replay, commands.length);
42+
} finally {
43+
const close = await runStep(context, 'close focused selector-route replay session', ['close'], {
44+
allowFailure: true,
45+
});
46+
assert.ok(
47+
close.status === 0 || close.json?.error?.code === 'SESSION_NOT_FOUND',
48+
JSON.stringify(close.json),
49+
);
50+
context.sessionOpen = false;
51+
for (const cleanup of options.postReplayCleanup ?? []) {
52+
await runStep(context, cleanup.step, cleanup.args, { commonFlags: false });
53+
}
54+
}
55+
}
56+
57+
function assertCoveredTargetDivergence(replay: CliJsonResult, expectedStep: number): void {
58+
const error = replay.json?.error;
59+
const divergence = error?.details?.divergence;
60+
assert.equal(error?.code, 'REPLAY_DIVERGENCE', JSON.stringify(replay.json));
61+
assert.equal(divergence?.kind, 'action-failure', JSON.stringify(divergence));
62+
assert.equal(divergence?.step?.index, expectedStep, JSON.stringify(divergence));
63+
assert.match(String(divergence?.action), /click.*(?:gearshape\.fill|field-name)/);
64+
assert.equal(divergence?.cause?.code, 'COMMAND_FAILED', JSON.stringify(divergence));
65+
assert.match(String(divergence?.cause?.message), /covered by another visible element/);
66+
}
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# Focused selector reads followed by a deliberate covered-target refusal.
2+
context platform=android kind=emulator timeout=120000
3+
4+
env APP_TARGET="com.callstack.agentdevicelab"
5+
6+
open "${APP_TARGET}" --no-test-ime --launch-url "agent-device-test-app:///form"
7+
wait "id=\"field-name\"" 30000
8+
get text "id=\"field-name\""
9+
get attrs "id=\"field-name\""
10+
is visible "id=\"field-name\""
11+
find id "field-name" get attrs
12+
orientation landscape-left
13+
click "id=\"field-name\""
14+
click "id=\"field-name\""
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# Focused selector reads followed by a deliberate covered-target refusal.
2+
context platform=ios kind=simulator timeout=120000
3+
4+
env APP_TARGET="Agent Device Tester"
5+
6+
open "${APP_TARGET}" --launch-url "agent-device-test-app:///automation?event=selector.routes&payload=%7B%22source%22%3A%22replay%22%7D"
7+
wait "id=\"automation-title\"" 30000
8+
get text "id=\"automation-title\""
9+
get attrs "id=\"automation-title\""
10+
is visible "id=\"automation-title\""
11+
find id "automation-title" get attrs
12+
app-switcher
13+
snapshot -i
14+
click "id=\"gearshape.fill\""

0 commit comments

Comments
 (0)