Skip to content

Commit f848b6f

Browse files
committed
fix(web): preserve the backend ref so snapshot refs match actionable refs
The web/agent-browser backend mints refs in tree order and skips non-interactive nodes, so its `@eN` refs are not dense. agent-device was dropping that ref in `normalizeAgentBrowserSnapshot` and then re-minting a dense positional `e${index+1}` in `attachRefs`. The ref an agent reads off the snapshot (dense, positional) therefore did not equal the ref the backend resolves on the next action (tree-ordered). On the ShopDemo login screen the username textbox displayed as one ref while the backend's ref for the same position pointed at the passcode field, so `fill @e3` landed in the wrong input. Preserve the backend ref on each web node and make `attachRefs` keep a node's existing `ref` when present, falling back to dense numbering for backends that do not mint refs (iOS/Android/maestro are unaffected).
1 parent 5bb3ea3 commit f848b6f

5 files changed

Lines changed: 82 additions & 2 deletions

File tree

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
import { describe, expect, test } from 'vitest';
2+
import { attachRefs, type RawSnapshotNode } from './snapshot.ts';
3+
4+
const node = (index: number, extra: Partial<RawSnapshotNode> = {}): RawSnapshotNode => ({
5+
index,
6+
...extra,
7+
});
8+
9+
describe('attachRefs', () => {
10+
test('preserves a backend-minted ref instead of re-minting a dense positional one', () => {
11+
// The web/agent-browser backend mints refs in tree order (non-dense — non-interactive
12+
// nodes are skipped). The first node's backend ref is e2 because e1 went to a node
13+
// that did not appear in the projected tree. Preserving it keeps the ref the agent
14+
// reads off the snapshot identical to the ref the backend resolves on the next action.
15+
const attached = attachRefs([node(0, { ref: 'e2' }), node(1, { ref: 'e3' })]);
16+
expect(attached.map((n) => n.ref)).toEqual(['e2', 'e3']);
17+
// index is preserved untouched — the positional identity is orthogonal to the ref.
18+
expect(attached.map((n) => n.index)).toEqual([0, 1]);
19+
});
20+
21+
test('re-mints dense positional refs for nodes without a backend ref', () => {
22+
const attached = attachRefs([node(0), node(1, { ref: 'e7' }), node(2)]);
23+
expect(attached.map((n) => n.ref)).toEqual(['e1', 'e7', 'e3']);
24+
});
25+
});

‎packages/kernel/src/snapshot.ts‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,15 @@ export type RawSnapshotNode = {
120120
hiddenContentBelow?: boolean;
121121
interactionBlocked?: 'covered';
122122
presentationHints?: string[];
123+
/**
124+
* Backend-minted ref for this node, when the capture backend already assigns a
125+
* stable, actionable ref (e.g. the web/agent-browser backend resolves actions
126+
* against its own `@eN` refs). `attachRefs` preserves this instead of re-minting
127+
* a dense positional ref, so the ref an agent sees in the snapshot is the same
128+
* ref the backend can resolve on the next action. Absent for backends that do
129+
* not mint refs — those fall back to dense `e${index}` numbering.
130+
*/
131+
ref?: string;
123132
/**
124133
* Accessibility custom actions the element exposes (iOS
125134
* `UIAccessibilityCustomAction`, React Native `accessibilityActions`). Merged
@@ -273,8 +282,15 @@ export type ScreenshotOverlayRef = {
273282
center: Point;
274283
};
275284

285+
/**
286+
* Assign a display ref to every node. A node that already carries a backend-minted
287+
* `ref` keeps it (see `RawSnapshotNode.ref`) — the web/agent-browser backend resolves
288+
* actions against its own refs, so re-minting a dense positional ref here would make
289+
* the snapshot show one ref while actions act on a different element. Backends that do
290+
* not mint refs get dense `e${index}` numbering, matching the historical behavior.
291+
*/
276292
export function attachRefs(nodes: RawSnapshotNode[]): SnapshotNode[] {
277-
return nodes.map((node, idx) => ({ ...node, ref: `e${idx + 1}` }));
293+
return nodes.map((node, idx) => ({ ...node, ref: node.ref ?? `e${idx + 1}` }));
278294
}
279295

280296
/**

‎packages/platform-web/src/__tests__/agent-browser-snapshot.test.ts‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,4 +59,34 @@ describe('normalizeAgentBrowserSnapshot', () => {
5959

6060
expect(result.nodes[0]?.enabled).toBe(true);
6161
});
62+
63+
test('preserves the backend ref so the displayed ref is the actionable one', async () => {
64+
// agent-browser mints refs in tree order and skips non-interactive nodes
65+
// (the leading `generic` container never gets a ref), so the refs are NOT dense:
66+
// e1 = generic (skipped here, no ref on it)
67+
// e2 = username textbox
68+
// e3 = passcode textbox <-- agent presses @e3 expecting username
69+
// e4 = sign-in button
70+
// The username textbox is the 2nd NODE in the snapshot but its backend ref is e2.
71+
// If agent-device re-mints dense positional refs, the snapshot would show the
72+
// username as @e2 while the passcode (node 3) shows @e3 — and an agent that
73+
// reads "@e3 = passcode" would land in the passcode field when it meant the
74+
// username, because the backend's own e3 resolves to a different element.
75+
// Preserving the backend ref keeps display ref == actionable ref.
76+
const result = await normalizeAgentBrowserSnapshot({
77+
snapshot: [
78+
'- textbox "Username" [ref=e2]',
79+
'- textbox "Passcode" [ref=e3]',
80+
'- button "Sign in" [ref=e4]',
81+
].join('\n'),
82+
refs: {
83+
e2: { role: 'textbox', name: 'Username' },
84+
e3: { role: 'textbox', name: 'Passcode' },
85+
e4: { role: 'button', name: 'Sign in' },
86+
},
87+
});
88+
89+
expect(result.nodes.map((node) => node.label)).toEqual(['Username', 'Passcode', 'Sign in']);
90+
expect(result.nodes.map((node) => node.ref)).toEqual(['e2', 'e3', 'e4']);
91+
});
6292
});

‎packages/platform-web/src/agent-browser-provider.test.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -666,6 +666,9 @@ function expectedNode(
666666
) {
667667
return {
668668
index,
669+
// The web normalizer now preserves the backend ref on each node. This fixture's
670+
// refs happen to be dense in tree order, so the ref matches the positional `e{index}`.
671+
ref: `e${index + 1}`,
669672
type,
670673
role: type,
671674
label,

‎packages/platform-web/src/agent-browser-snapshot.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,13 @@ export async function normalizeAgentBrowserSnapshot(
3131
if (fetchBox) await attachDraftRects(drafts, fetchBox);
3232

3333
return {
34-
nodes: drafts.map((draft, index) => ({ ...draft.node, index })),
34+
// Preserve each draft's backend ref as the node's `ref`. agent-browser resolves
35+
// actions (click/fill/hover) against its own `@eN` refs, which are minted in tree
36+
// order and are NOT dense — so the ref an agent reads off the snapshot must be the
37+
// backend ref, not a re-minted dense positional ref. `attachRefs` (downstream) keeps
38+
// it. Without this the snapshot could show `@e3` = username while the action on
39+
// `@e3` lands on the backend's `@e3` (a different element, e.g. the passcode field).
40+
nodes: drafts.map((draft, index) => ({ ...draft.node, index, ref: draft.ref })),
3541
truncated: readBooleanProperty(data, 'truncated'),
3642
};
3743
}

0 commit comments

Comments
 (0)