Skip to content

Commit bcaa106

Browse files
authored
refactor: extract snapshot and replay identity semantics (#1582)
* refactor: extract snapshot and replay identity semantics * refactor: move pure rect primitives from contracts to kernel/rect containsPoint, pickLargestRect, and isRectVisibleInViewport are raw rectangle arithmetic with no snapshot awareness, so they belong beside rectContains/rectArea in @agent-device/kernel/rect rather than in the snapshot-semantics vocabulary. The node-aware resolveViewportRect folds into contracts/snapshot-visibility.ts, retiring snapshot-geometry.ts; after this split, everything behavioral in @agent-device/contracts/snapshot is policy that interprets the snapshot model. * refactor: restore ADR-0012 rationale docs and dedupe replay identity shapes The #1478/#1581 extraction moved the identity/structural helpers but compressed their invariant documentation to one-liners; the deliberate no-ancestry-exclusion rule on idMatchCountInTree, the fail-closed guard comparison, and the who-throws/who-detects contracts on the two divergence reason markers now travel with their definitions again. LocalIdentity and NodeStructuralDenotation move to contracts/target-annotation.ts (beside TargetAncestryEntry, which WaitLandmarkMismatchEvidence now references directly), so the guard shapes in contracts/replay.ts are nominal instead of hand-rolled structural twins; ad-script re-exports the vocabulary beside the readers that produce it. Also inlines the demoteNonUniqueId pass-through wrapper in session-target-evidence.ts. * style: fix oxfmt formatting in snapshot-visibility * refactor: move findSnapshotAncestor into contracts/snapshot-tree The last root value import from src/selectors: predicates.ts reached src/snapshot/snapshot-processing.ts for the ancestor walker. The walker is index-based tree traversal with no presentation policy, so it joins buildSnapshotNodeMap in contracts/snapshot-tree.ts; both consumers repoint to the façade and the non-contiguous-index/cycle coverage moves to the package test. src/selectors now has zero value imports from root src in production files.
1 parent 56d9ee6 commit bcaa106

64 files changed

Lines changed: 851 additions & 659 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎packages/ad-script/src/index.ts‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,12 +80,22 @@ export {
8080

8181
export {
8282
annotationLocalIdentity,
83+
demoteNonUniqueLocalIdentity,
8384
firstAncestryMismatch,
85+
idMatchCountInTree,
8486
identityFieldMismatches,
87+
localIdentitiesEqual,
8588
matchesAncestryPrefix,
8689
matchesLocalIdentity,
90+
readNodeLocalIdentity,
91+
readNodeStructuralDenotation,
92+
siblingOrdinal,
93+
structuralDenotationsEqual,
94+
} from './internal/target-annotation-identity.ts';
95+
export type {
96+
LocalIdentity,
97+
NodeStructuralDenotation,
8798
} from './internal/target-annotation-identity.ts';
88-
export type { LocalIdentity } from './internal/target-annotation-identity.ts';
8999

90100
export { classifyTargetBindingMatch } from './internal/target-annotation-classification.ts';
91101
export type {

‎packages/ad-script/src/internal/__tests__/target-annotation-identity.test.ts‎

Lines changed: 66 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,13 @@
11
import { test } from 'vitest';
22
import assert from 'node:assert/strict';
3-
import { matchesAncestryPrefix, matchesLocalIdentity } from '../target-annotation-identity.ts';
3+
import type { RawSnapshotNode } from '@agent-device/kernel/snapshot';
4+
import {
5+
demoteNonUniqueLocalIdentity,
6+
idMatchCountInTree,
7+
matchesAncestryPrefix,
8+
matchesLocalIdentity,
9+
readNodeLocalIdentity,
10+
} from '../target-annotation-identity.ts';
411

512
// The `# agent-device:target-v1` SERDE (serialize/parse, normalization,
613
// bounds) lives alongside this in `target-annotation-serde.ts` — see
@@ -59,3 +66,61 @@ test('matchesLocalIdentity: with no recorded id, role+label must both match, abs
5966
assert.equal(matchesLocalIdentity({ role: 'button' }, { role: 'button' }), true);
6067
assert.equal(matchesLocalIdentity({ role: 'button', label: 'Save' }, { role: 'button' }), false);
6168
});
69+
70+
// ---------------------------------------------------------------------------
71+
// Shared id demotion. `idMatchCountInTree` / `demoteNonUniqueLocalIdentity`
72+
// are the ONE shared uniqueness predicate behind both id-demotion sites (the
73+
// `target-v1` identity tuple `session-target-evidence.ts` writes at record
74+
// time, and the selector chain `buildSelectorChainForNode` builds — see
75+
// `src/selectors/build.test.ts` for that consumer's own coverage). Neither
76+
// consumer test exercises the pair directly; these tests pin the shared
77+
// predicate itself, so a re-derivation of identity demotion that drops or
78+
// scopes the uniqueness rule fails here first.
79+
// ---------------------------------------------------------------------------
80+
81+
function sharedIdRows(): RawSnapshotNode[] {
82+
return [
83+
{
84+
index: 0,
85+
type: 'TextView',
86+
identifier: 'android:id/title',
87+
label: 'Network & internet',
88+
},
89+
{
90+
index: 1,
91+
type: 'TextView',
92+
identifier: 'android:id/title',
93+
label: 'Apps',
94+
},
95+
{ index: 2, type: 'Button', identifier: 'save', label: 'Save' },
96+
];
97+
}
98+
99+
test('idMatchCountInTree counts every node sharing the canonical id, independent of role or label', () => {
100+
const nodes = sharedIdRows();
101+
assert.equal(idMatchCountInTree(nodes, 'android:id/title'), 2);
102+
assert.equal(idMatchCountInTree(nodes, 'save'), 1);
103+
assert.equal(idMatchCountInTree(nodes, 'does-not-exist'), 0);
104+
});
105+
106+
test('demoteNonUniqueLocalIdentity drops only a shared id tier', () => {
107+
const nodes = sharedIdRows();
108+
109+
const sharedIdentity = readNodeLocalIdentity(nodes[0]!);
110+
assert.equal(sharedIdentity.id, 'android:id/title');
111+
assert.deepEqual(demoteNonUniqueLocalIdentity(sharedIdentity, nodes), {
112+
role: 'textview',
113+
label: 'Network & internet',
114+
});
115+
116+
const uniqueIdentity = readNodeLocalIdentity(nodes[2]!);
117+
assert.equal(uniqueIdentity.id, 'save');
118+
assert.deepEqual(demoteNonUniqueLocalIdentity(uniqueIdentity, nodes), uniqueIdentity);
119+
});
120+
121+
test('demoteNonUniqueLocalIdentity passes through identities without an id', () => {
122+
const nodes: RawSnapshotNode[] = [{ index: 0, type: 'Button', label: 'Unlabeled row' }];
123+
const identity = readNodeLocalIdentity(nodes[0]!);
124+
assert.equal(identity.id, undefined);
125+
assert.deepEqual(demoteNonUniqueLocalIdentity(identity, nodes), identity);
126+
});

‎packages/ad-script/src/internal/target-annotation-identity.ts‎

Lines changed: 130 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,16 +16,143 @@
1616
* here, not through `@agent-device/ad-replay`'s façade (#1555 review).
1717
*/
1818

19-
import type { TargetAncestryEntry, TargetAnnotationV1 } from '@agent-device/contracts/replay';
19+
import type { RawSnapshotNode } from '@agent-device/kernel/snapshot';
20+
import { normalizeType } from '@agent-device/contracts/snapshot';
21+
import type {
22+
LocalIdentity,
23+
NodeStructuralDenotation,
24+
TargetAncestryEntry,
25+
TargetAnnotationV1,
26+
} from '@agent-device/contracts/replay';
27+
import {
28+
normalizeIdentifierField,
29+
normalizeLabelField,
30+
normalizeRoleField,
31+
truncateToUtf8Bytes,
32+
TARGET_ANNOTATION_MAX_FIELD_BYTES,
33+
} from './target-annotation-serde.ts';
2034

2135
// ---------------------------------------------------------------------------
2236
// Local identity + ancestry-prefix matching (decision 3 "Local identity" /
2337
// "Ancestry"). Pure over the small structural shapes above — no tree
2438
// dependency, so both the writer (over `SnapshotNode`-derived values) and a
25-
// replay verifier can share it verbatim.
39+
// replay verifier can share it verbatim. The type vocabulary (`LocalIdentity`,
40+
// `NodeStructuralDenotation`) lives in `@agent-device/contracts/replay` so the
41+
// guard shapes there reference it nominally; this module re-exports it beside
42+
// the readers that produce it.
2643
// ---------------------------------------------------------------------------
2744

28-
export type LocalIdentity = { id?: string; role: string; label?: string };
45+
export type { LocalIdentity, NodeStructuralDenotation };
46+
47+
type IdentityTreeNode = Pick<RawSnapshotNode, 'type' | 'identifier' | 'label'>;
48+
49+
/**
50+
* ADR 0012 decision 3: the ONE snapshot-node local-identity reader —
51+
* normalized (NFC, label whitespace collapse, `normalizeType` role) AND
52+
* 256-byte field-capped, on every path. Shared by the record-time writer
53+
* (`src/daemon/session-target-evidence.ts`), replay-time verification
54+
* (`src/daemon/handlers/session-replay-target-verification.ts`), and the
55+
* dispatch-side post-resolution guard
56+
* (`src/commands/interaction/runtime/resolution.ts`), so all three compute
57+
* a node's identity with byte-identical semantics.
58+
*/
59+
export function readNodeLocalIdentity(
60+
node: Pick<RawSnapshotNode, 'type' | 'identifier' | 'label'>,
61+
): LocalIdentity {
62+
const role = normalizeRoleField(normalizeType(node.type ?? ''));
63+
const id = normalizeIdentifierField(node.identifier);
64+
const label = normalizeLabelField(node.label);
65+
return {
66+
...(id !== undefined ? { id: truncateToUtf8Bytes(id, TARGET_ANNOTATION_MAX_FIELD_BYTES) } : {}),
67+
role: truncateToUtf8Bytes(role, TARGET_ANNOTATION_MAX_FIELD_BYTES),
68+
...(label !== undefined
69+
? { label: truncateToUtf8Bytes(label, TARGET_ANNOTATION_MAX_FIELD_BYTES) }
70+
: {}),
71+
};
72+
}
73+
74+
/** Exact-equality comparison of two normalized local identities. */
75+
export function localIdentitiesEqual(a: LocalIdentity, b: LocalIdentity): boolean {
76+
return a.id === b.id && a.role === b.role && a.label === b.label;
77+
}
78+
79+
/**
80+
* ADR 0012 decision 3 amendment (#1269): how many nodes in `nodes` carry the
81+
* canonical identity `id`. THE single uniqueness predicate behind both
82+
* id-demotion sites — `computeTargetEvidence`'s `target-v1` identity tuple
83+
* and `buildSelectorChainForNode`'s selector chain. Sharing it is the
84+
* correctness guarantee: a non-unique id is dropped from BOTH or NEITHER,
85+
* never half-demoted (dropped from identity while still leading the chain, or
86+
* the reverse — the exact split #1269's fix exists to close).
87+
*
88+
* `id` is a canonical identity id (`readNodeLocalIdentity(node).id`: NFC +
89+
* 256-byte field cap, no trimming), and every candidate is measured through
90+
* the SAME reader, so the count is over exactly the identities the replay
91+
* verifier keys on. There is deliberately NO ancestry / parent-walk
92+
* exclusion: an id is non-selective the moment two nodes anywhere in the tree
93+
* carry it, independent of structural context or a broken parent linkage.
94+
*/
95+
export function idMatchCountInTree(nodes: readonly IdentityTreeNode[], id: string): number {
96+
let count = 0;
97+
for (const node of nodes) {
98+
if (readNodeLocalIdentity(node).id === id) count += 1;
99+
}
100+
return count;
101+
}
102+
103+
/**
104+
* ADR 0012 decision 3 amendment (#1269): `identity` with its `id` demoted
105+
* whenever the id is non-unique in `nodes`, built on the SAME
106+
* `idMatchCountInTree` predicate `buildSelectorChainForNode`'s
107+
* `selectableId` keys off directly. `computeTargetEvidence` uses this
108+
* whole-identity form; extracted so a third call site (#1280's
109+
* press-retarget identity-empty check, `src/core/press-retarget.ts`)
110+
* shares it rather than re-deriving the rule a third way. A demoted id
111+
* falls back to role+label, the same shape an unrecorded id already
112+
* computes.
113+
*/
114+
export function demoteNonUniqueLocalIdentity(
115+
identity: LocalIdentity,
116+
nodes: readonly IdentityTreeNode[],
117+
): LocalIdentity {
118+
if (identity.id === undefined) return identity;
119+
if (idMatchCountInTree(nodes, identity.id) <= 1) return identity;
120+
const { role, label } = identity;
121+
return { role, ...(label !== undefined ? { label } : {}) };
122+
}
123+
124+
type StructuralNode = Pick<RawSnapshotNode, 'index' | 'parentIndex'>;
125+
126+
/** Zero-based ordinal among the node's own parent's children, in document order. */
127+
export function siblingOrdinal(nodes: readonly StructuralNode[], node: StructuralNode): number {
128+
let ordinal = 0;
129+
for (const candidate of nodes) {
130+
if (candidate.parentIndex !== node.parentIndex) continue;
131+
if (candidate.index === node.index) return ordinal;
132+
ordinal += 1;
133+
}
134+
return 0;
135+
}
136+
137+
export function readNodeStructuralDenotation(
138+
node: StructuralNode,
139+
nodes: readonly StructuralNode[],
140+
): NodeStructuralDenotation {
141+
return { documentOrder: node.index, sibling: siblingOrdinal(nodes, node) };
142+
}
143+
144+
/**
145+
* The guard passes only when BOTH structural discriminators agree — a
146+
* fail-closed comparison: two same-local-identity duplicates differ in
147+
* `documentOrder` (always) and usually `sibling`, so the guard refuses
148+
* whenever dispatch resolved a different member than verification isolated.
149+
*/
150+
export function structuralDenotationsEqual(
151+
a: NodeStructuralDenotation,
152+
b: NodeStructuralDenotation,
153+
): boolean {
154+
return a.documentOrder === b.documentOrder && a.sibling === b.sibling;
155+
}
29156

30157
/** The recorded annotation's identity tier as a bare `LocalIdentity` (drop-empty-keys form). */
31158
export function annotationLocalIdentity(

‎packages/contracts/package.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,10 @@
5656
"types": "./src/facades/replay.ts",
5757
"default": "./src/facades/replay.ts"
5858
},
59+
"./snapshot": {
60+
"types": "./src/facades/snapshot.ts",
61+
"default": "./src/facades/snapshot.ts"
62+
},
5963
"./divergence": {
6064
"types": "./src/facades/divergence.ts",
6165
"default": "./src/facades/divergence.ts"
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
export { isScrollableNodeLike, isScrollableType } from '../snapshot-scroll.ts';
2+
export { buildSnapshotNodeMap, findSnapshotAncestor } from '../snapshot-tree.ts';
3+
export {
4+
findNearestScrollableAncestor,
5+
isNodeVisibleInEffectiveViewport,
6+
isNodeVisibleOnScreen,
7+
isTapPointInsideViewport,
8+
resolveEffectiveViewportRect,
9+
resolveViewportRect,
10+
} from '../snapshot-visibility.ts';
11+
export { extractNodeText, isFillableType, normalizeType } from '../snapshot-text.ts';

‎packages/contracts/src/replay.ts‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,57 @@
11
import type { DaemonError } from '@agent-device/kernel/errors';
22
import type { SnapshotDiagnosticsSummary } from './snapshot-diagnostics.ts';
3+
import type {
4+
LocalIdentity,
5+
NodeStructuralDenotation,
6+
TargetAncestryEntry,
7+
} from './target-annotation.ts';
8+
import { WAIT_REASONS } from './wait.ts';
9+
10+
/**
11+
* The verified-member denotation carried on the pre-action guard
12+
* (`DaemonRequest.internal.replayTargetGuard`): its normalized local identity
13+
* PLUS the structural discriminator path 6 used to isolate it among same
14+
* local-identity duplicates. Comparing local identity alone would let a
15+
* different duplicate (identical id/role/label) pass the guard — the exact
16+
* verified-but-taps-different-node mis-binding step 4 exists to prevent.
17+
* Lives in contracts so daemon and command layers can share the guard
18+
* without depending on the annotation codec package.
19+
*/
20+
export type ReplayTargetGuardDenotation = {
21+
identity: LocalIdentity;
22+
structural: NodeStructuralDenotation;
23+
};
24+
25+
/**
26+
* `details.reason` marker on the pre-action refusal thrown by dispatch's
27+
* post-resolution guard (`assertExpectedResolvedTarget`,
28+
* `src/commands/interaction/runtime/resolution.ts`), detected by the replay
29+
* step loop to convert the refusal into an identity-mismatch target-binding
30+
* divergence.
31+
*/
32+
export const REPLAY_TARGET_GUARD_MISMATCH_REASON = 'replay_target_guard_mismatch';
33+
34+
/**
35+
* #1349: `details.reason` marker on the timeout refusal `wait` throws when
36+
* candidates matching the recorded selector appeared during polling but none
37+
* ever carried the recorded landmark identity (local identity + leaf-anchored
38+
* ancestry prefix). The replay step loop converts it into an
39+
* identity-mismatch target-binding divergence; the wait never reports
40+
* success.
41+
*/
42+
export const WAIT_LANDMARK_MISMATCH_REASON = WAIT_REASONS.landmarkIdentityMismatch;
43+
44+
/**
45+
* The compact evidence `wait` retains from its LAST poll whose capture
46+
* matched the recorded selector: the domain size and the first match's
47+
* identity/ancestry, enough for the daemon to build the divergence's
48+
* `targetBinding` payload without shipping node trees through error details.
49+
*/
50+
export type WaitLandmarkMismatchEvidence = {
51+
matchCount: number;
52+
observed: LocalIdentity;
53+
observedAncestry: TargetAncestryEntry[];
54+
};
355

456
export type ReplayCommandResult = {
557
replayed: number;

src/snapshot/__tests__/tap-point-policy-parity.test.ts renamed to packages/contracts/src/snapshot-tap-point-policy.test.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import { test } from 'vitest';
22
import assert from 'node:assert/strict';
33
import fs from 'node:fs';
44
import path from 'node:path';
5-
import { isTapPointInsideViewport } from '../mobile-snapshot-semantics.ts';
5+
import { isTapPointInsideViewport } from './facades/snapshot.ts';
66
import type { Rect } from '@agent-device/kernel/snapshot';
77

88
// ADR 0011 Layer 2 golden parity table: the SAME JSON is asserted against the
@@ -33,7 +33,7 @@ const TABLE_PATH = path.resolve(
3333
'tap-point-policy.json',
3434
);
3535

36-
test('the TS tap-point rule agrees with every golden parity table case', () => {
36+
test('the contracts snapshot tap-point rule agrees with every golden parity table case', () => {
3737
const cases = JSON.parse(fs.readFileSync(TABLE_PATH, 'utf8')) as FixtureCase[];
3838
assert.ok(cases.length > 0, 'parity table must not be empty');
3939
const names = new Set(cases.map((fixture) => fixture.name));
Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
import type { Platform, PublicPlatform } from '@agent-device/kernel/device';
2+
import type { SnapshotState } from '@agent-device/kernel/snapshot';
3+
4+
export function normalizeType(type: string): string {
5+
let normalized = type
6+
.trim()
7+
.replace(/XCUIElementType/gi, '')
8+
.replace(/^AX/, '')
9+
.toLowerCase();
10+
const lastSeparator = Math.max(normalized.lastIndexOf('.'), normalized.lastIndexOf('/'));
11+
if (lastSeparator !== -1) {
12+
normalized = normalized.slice(lastSeparator + 1);
13+
}
14+
return normalized;
15+
}
16+
17+
export function isFillableType(type: string, platform: Platform | PublicPlatform): boolean {
18+
const normalized = normalizeType(type);
19+
if (!normalized) return true;
20+
if (platform === 'android') {
21+
return normalized.includes('edittext') || normalized.includes('autocompletetextview');
22+
}
23+
return (
24+
normalized.includes('textfield') ||
25+
normalized.includes('securetextfield') ||
26+
normalized.includes('searchfield') ||
27+
normalized.includes('textview') ||
28+
normalized.includes('textarea') ||
29+
normalized === 'search'
30+
);
31+
}
32+
33+
export function extractNodeText(node: SnapshotState['nodes'][number]): string {
34+
const candidates = [node.label, node.value, node.identifier]
35+
.map((value) => (typeof value === 'string' ? value.trim() : ''))
36+
.filter((value) => value.length > 0);
37+
return candidates[0] ?? '';
38+
}

0 commit comments

Comments
 (0)