Skip to content

Commit b829e59

Browse files
committed
fix: exclude keyboard/IME chrome from replay divergence screen.refs
On a keyboard-open screen, target-binding divergence refs were dominated by keyboard KEY/window chrome, pushing the real actionable target past the 20-ref cap. Reuse settle's existing structural keyboard/IME chrome classifier (collectSettleChromeRefs, #1198/#1200) to filter divergence refs before the cap, instead of duplicating the classification.
1 parent f474f07 commit b829e59

3 files changed

Lines changed: 134 additions & 4 deletions

File tree

‎src/commands/interaction/runtime/settle.ts‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -494,7 +494,14 @@ function withoutSettleChrome(
494494
return nodes.filter((node) => !strippedIndexes.has(node.index));
495495
}
496496

497-
function collectSettleChromeRefs(
497+
/**
498+
* Refs of iOS keyboard-window chrome unioned with Android IME/system chrome
499+
* (see `collectSettleChrome`). Exported so any ref-selection budget — not
500+
* just the settle tail — can exclude keyboard/IME chrome without duplicating
501+
* the structural classification (e.g. the replay divergence `screen.refs`
502+
* cap; ADR 0012 decision 6).
503+
*/
504+
export function collectSettleChromeRefs(
498505
nodes: SnapshotNode[],
499506
appBundleId: string | undefined,
500507
): ReadonlySet<string> {

‎src/daemon/handlers/__tests__/session-replay-divergence.test.ts‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,3 +69,107 @@ test('buildReplayFailureDivergence dedupes suggestions using the strongest basis
6969
expect(divergence.suggestions[0]?.ref).toBe('e1');
7070
expect(divergence.suggestions[0]?.basis).toBe('id');
7171
});
72+
73+
// Live-shape fixture (iPhone 17 Pro sim, iOS 26): the software keyboard
74+
// renders in its own window ([Window] > [Keyboard] > 26 [Key] children).
75+
// Onscreen, that subtree alone blows past SCREEN_REF_CAPTURE_LIMIT (20),
76+
// which is exactly the bug: the real actionable target (a sibling of the
77+
// keyboard window, not inside it) sorts after the keys in document order and
78+
// gets truncated out unless keyboard/IME chrome is excluded from the budget
79+
// first.
80+
function keyboardSwampedNodes(actionableLabel: string) {
81+
const keyboardWindowIndex = 1;
82+
const keyboardContainerIndex = 2;
83+
const keys = Array.from({ length: 26 }, (_, key) => ({
84+
index: 3 + key,
85+
depth: 3,
86+
parentIndex: keyboardContainerIndex,
87+
type: 'Key',
88+
label: String.fromCharCode(97 + key),
89+
rect: { x: (key % 10) * 40, y: 600 + Math.floor(key / 10) * 54, width: 39, height: 54 },
90+
hittable: true,
91+
}));
92+
return [
93+
{
94+
index: 0,
95+
depth: 0,
96+
type: 'Application',
97+
label: 'Example',
98+
rect: { x: 0, y: 0, width: 402, height: 874 },
99+
hittable: true,
100+
},
101+
{
102+
index: keyboardWindowIndex,
103+
depth: 1,
104+
parentIndex: 0,
105+
type: 'Window',
106+
label: 'Keyboard Window',
107+
rect: { x: 0, y: 583, width: 402, height: 291 },
108+
hittable: true,
109+
},
110+
{
111+
index: keyboardContainerIndex,
112+
depth: 2,
113+
parentIndex: keyboardWindowIndex,
114+
type: 'Keyboard',
115+
label: 'Padding-Left',
116+
rect: { x: 0, y: 583, width: 402, height: 291 },
117+
},
118+
...keys,
119+
{
120+
index: 3 + keys.length,
121+
depth: 1,
122+
parentIndex: 0,
123+
type: 'Button',
124+
label: actionableLabel,
125+
identifier: 'push-article',
126+
rect: { x: 20, y: 100, width: 160, height: 44 },
127+
hittable: true,
128+
},
129+
];
130+
}
131+
132+
test('buildReplayFailureDivergence excludes keyboard chrome from screen.refs and surfaces the actionable target within the cap', async () => {
133+
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-divergence-keyboard-'));
134+
const sessionStore = new SessionStore(path.join(root, 'sessions'));
135+
const sessionName = 'default';
136+
sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' }));
137+
138+
mockDispatchCommand.mockResolvedValue({
139+
nodes: keyboardSwampedNodes('Push Article'),
140+
truncated: false,
141+
backend: 'xctest',
142+
});
143+
144+
const action = {
145+
ts: 0,
146+
command: 'press',
147+
positionals: ['label="Push Article"'],
148+
flags: {},
149+
result: { selectorChain: ['label="Push Article"'] },
150+
};
151+
const divergence = await buildReplayFailureDivergence({
152+
error: { code: 'COMMAND_FAILED', message: 'target not found' },
153+
action,
154+
index: 0,
155+
sourcePath: path.join(root, 'flow.ad'),
156+
sourceLine: 1,
157+
session: sessionStore.get(sessionName),
158+
sessionName,
159+
sessionStore,
160+
logPath: path.join(root, 'daemon.log'),
161+
responseLevel: 'default',
162+
planActions: [action],
163+
planDigest: 'test-plan-digest',
164+
});
165+
166+
expect(divergence.screen.state).toBe('available');
167+
const screen = divergence.screen as Extract<typeof divergence.screen, { state: 'available' }>;
168+
// No keyboard/IME chrome (container or individual keys) reached the ref list.
169+
expect(screen.refs.some((ref) => ref.role.toLowerCase() === 'key')).toBe(false);
170+
expect(screen.refs.some((ref) => ref.role.toLowerCase() === 'keyboard')).toBe(false);
171+
// The real actionable target — pushed past the cap by 28 keyboard nodes
172+
// before this fix — is now surfaced.
173+
expect(screen.refs.some((ref) => ref.label === 'Push Article')).toBe(true);
174+
expect(screen.truncated).toBeUndefined();
175+
});

‎src/daemon/handlers/session-replay-divergence.ts‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import {
1616
type Selector,
1717
} from '../../selectors/index.ts';
1818
import { collectReplaySelectorCandidates } from './session-replay-heal.ts';
19+
import { collectSettleChromeRefs } from '../../commands/interaction/runtime/settle.ts';
1920
import { buildReplayDivergenceResume } from './session-replay-resume.ts';
2021
import { formatDivergenceActionLabel, isTouchTargetCommand } from '../../replay/script-utils.ts';
2122
import {
@@ -157,7 +158,13 @@ export function boundReplayDivergenceForSession(params: {
157158
}
158159

159160
export type DivergenceObservation =
160-
| { state: 'available'; nodes: SnapshotNode[]; refsGeneration: number }
161+
| {
162+
state: 'available';
163+
nodes: SnapshotNode[];
164+
refsGeneration: number;
165+
/** Session's app bundle id at capture time; threaded to `buildDivergenceScreen`'s chrome filter (Android IME-scope guard — inert on iOS). */
166+
appBundleId: string | undefined;
167+
}
161168
| { state: 'unavailable'; reason: string; hint: string };
162169

163170
/** Adapts a capture observation to the `repairHint` container-presence test's input shape. */
@@ -216,6 +223,7 @@ export async function captureDivergenceObservation(params: {
216223
state: 'available',
217224
nodes: snapshot.nodes,
218225
refsGeneration: session.snapshotGeneration ?? 0,
226+
appBundleId: session.appBundleId,
219227
};
220228
} catch (error) {
221229
return {
@@ -249,7 +257,11 @@ export function buildDivergenceScreen(
249257
hint: sanitize(observation.hint),
250258
};
251259
}
252-
const { refs, truncated } = buildReplayDivergenceScreenRefs(observation.nodes, sanitize);
260+
const { refs, truncated } = buildReplayDivergenceScreenRefs(
261+
observation.nodes,
262+
sanitize,
263+
observation.appBundleId,
264+
);
253265
return {
254266
state: 'available',
255267
refsGeneration: observation.refsGeneration,
@@ -265,11 +277,18 @@ const SCREEN_REF_CAPTURE_LIMIT = 20;
265277
function buildReplayDivergenceScreenRefs(
266278
nodes: SnapshotNode[],
267279
sanitize: DivergenceFieldSanitizer,
280+
appBundleId: string | undefined,
268281
): {
269282
refs: ReplayDivergenceScreenRef[];
270283
truncated: boolean;
271284
} {
272-
const candidates = nodes.filter((node) => node.ref && node.interactionBlocked !== 'covered');
285+
// Keyboard/IME chrome must not consume the ref budget: it reuses the exact
286+
// structural classifier `--settle`'s tail already relies on (#1198/#1200)
287+
// rather than a second keyboard/IME node-type list.
288+
const chromeRefs = collectSettleChromeRefs(nodes, appBundleId);
289+
const candidates = nodes.filter(
290+
(node) => node.ref && node.interactionBlocked !== 'covered' && !chromeRefs.has(node.ref),
291+
);
273292
const refs = candidates.slice(0, SCREEN_REF_CAPTURE_LIMIT).map((node) => {
274293
const role = formatRole(node.type ?? 'Element');
275294
const label = displayLabel(node, role);

0 commit comments

Comments
 (0)