Skip to content

Commit 24e2d8e

Browse files
committed
feat(replay): recorded landmark identity verification for wait, get-pattern coverage for is (#1349)
- New CommandDescriptor trait targetIdentityVerification pins the evidence-carrying command set and routes wait to a post-resolution phase so an annotated wait never enters the generic pre-dispatch verification (an absent landmark is its expected starting condition). - wait <selector> records landmark-mode target-v1 evidence (existence self-check; identity-empty matches record no annotation) and, on replay, keeps polling until a selector match carries the recorded identity; a deadline with only impostor matches fails closed as an identity-mismatch REPLAY_DIVERGENCE, a recorded-unverifiable annotation refuses before polling, and a plain timeout stays an action-failure divergence. - is (except exists) joins the get pattern: evidence at record time, generic pre-dispatch verification, and the post-resolution guard threaded through dispatch; direct-iOS fast paths for wait/is are gated during recording and guarded replays. - Read-only find stays intentionally unannotated (fuzzy-locator resolution has no selector-chain identity token), proven by test.
1 parent 16fdd01 commit 24e2d8e

20 files changed

Lines changed: 1371 additions & 79 deletions

‎src/commands/interaction/runtime/selector-read.test.ts‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -485,3 +485,151 @@ test('runtime selector convenience methods use explicit target helpers', async (
485485
assert.equal(visible.pass, true);
486486
assert.deepEqual(waited, { kind: 'text', text: 'Ready', waitedMs: 0 });
487487
});
488+
489+
// ---------------------------------------------------------------------------
490+
// #1349: wait's in-loop landmark identity verification (replay-only,
491+
// threaded as `target.recordedLandmark`). Polling semantics are preserved —
492+
// a same-selector impostor never aborts the wait; only the deadline turns
493+
// rejected candidates into the fail-closed landmark refusal.
494+
// ---------------------------------------------------------------------------
495+
496+
import { computeTargetEvidence } from '../../../daemon/session-target-evidence.ts';
497+
import { WAIT_LANDMARK_MISMATCH_REASON } from '../../../replay/target-identity-node.ts';
498+
import { createFakeClock } from './__tests__/test-utils/index.ts';
499+
import { AppError } from '../../../kernel/errors.ts';
500+
501+
function landmarkScreen(parentLabel: string) {
502+
return makeSnapshotState([
503+
{ index: 0, depth: 0, type: 'Other', label: parentLabel },
504+
{
505+
index: 1,
506+
depth: 1,
507+
parentIndex: 0,
508+
type: 'StaticText',
509+
label: 'Screen X',
510+
rect: { x: 0, y: 0, width: 100, height: 20 },
511+
},
512+
]);
513+
}
514+
515+
function recordedLandmarkFor(snapshot: ReturnType<typeof landmarkScreen>) {
516+
const node = snapshot.nodes[1]!;
517+
const evidence = computeTargetEvidence(
518+
{ node, preActionNodes: snapshot.nodes },
519+
{ mode: 'landmark' },
520+
);
521+
assert.ok(evidence);
522+
assert.equal(evidence.verification, 'verified');
523+
return evidence;
524+
}
525+
526+
function landmarkWaitDevice(captures: Array<ReturnType<typeof landmarkScreen>>) {
527+
let call = 0;
528+
const initial = captures[0]!;
529+
const device = createAgentDevice({
530+
backend: {
531+
platform: 'ios',
532+
captureSnapshot: async () => {
533+
const snapshot = captures[Math.min(call, captures.length - 1)]!;
534+
call += 1;
535+
return { snapshot };
536+
},
537+
} satisfies AgentDeviceBackend,
538+
artifacts: createLocalArtifactAdapter(),
539+
sessions: createMemorySessionStore([{ name: 'default', snapshot: initial }]),
540+
policy: localCommandPolicy(),
541+
clock: createFakeClock(),
542+
});
543+
return device;
544+
}
545+
546+
test('runtime wait keeps polling past a same-selector impostor and succeeds on the recorded landmark', async () => {
547+
const recordTime = landmarkScreen('Detail Screen');
548+
const recorded = recordedLandmarkFor(recordTime);
549+
const impostor = landmarkScreen('List Screen');
550+
const empty = makeSnapshotState([{ index: 0, depth: 0, type: 'Other', label: 'Loading' }]);
551+
const device = landmarkWaitDevice([empty, impostor, landmarkScreen('Detail Screen')]);
552+
553+
const result = await device.selectors.wait({
554+
session: 'default',
555+
target: {
556+
kind: 'selector',
557+
selector: 'label="Screen X"',
558+
timeoutMs: 10_000,
559+
recordedLandmark: recorded,
560+
},
561+
});
562+
563+
assert.equal(result.kind, 'selector');
564+
if (result.kind !== 'selector') throw new Error('unreachable');
565+
// Two rejected polls (absent, then impostor) before the landmark appeared.
566+
assert.equal(result.waitedMs >= 600, true);
567+
assert.equal(result.node?.label, 'Screen X');
568+
assert.equal(result.preActionNodes?.length, 2);
569+
});
570+
571+
test('runtime wait fails closed at the deadline when only impostors matched the selector', async () => {
572+
const recorded = recordedLandmarkFor(landmarkScreen('Detail Screen'));
573+
const device = landmarkWaitDevice([landmarkScreen('List Screen')]);
574+
575+
const error = await device.selectors
576+
.wait({
577+
session: 'default',
578+
target: {
579+
kind: 'selector',
580+
selector: 'label="Screen X"',
581+
timeoutMs: 1000,
582+
recordedLandmark: recorded,
583+
},
584+
})
585+
.then(
586+
() => undefined,
587+
(thrown: unknown) => thrown,
588+
);
589+
590+
assert.ok(error instanceof AppError);
591+
assert.equal(error.details?.reason, WAIT_LANDMARK_MISMATCH_REASON);
592+
assert.equal(error.details?.matchCount, 1);
593+
const observed = error.details?.observed as { role: string; label?: string };
594+
assert.equal(observed.label, 'Screen X');
595+
const ancestry = error.details?.observedAncestry as Array<{ role: string; label?: string }>;
596+
assert.equal(ancestry[0]?.label, 'List Screen');
597+
});
598+
599+
test('runtime wait with a recorded landmark keeps the plain timeout when the selector never matched', async () => {
600+
const recorded = recordedLandmarkFor(landmarkScreen('Detail Screen'));
601+
const empty = makeSnapshotState([{ index: 0, depth: 0, type: 'Other', label: 'Loading' }]);
602+
const device = landmarkWaitDevice([empty]);
603+
604+
await assert.rejects(
605+
device.selectors.wait({
606+
session: 'default',
607+
target: {
608+
kind: 'selector',
609+
selector: 'label="Screen X"',
610+
timeoutMs: 1000,
611+
recordedLandmark: recorded,
612+
},
613+
}),
614+
(thrown: unknown) => {
615+
assert.ok(thrown instanceof AppError);
616+
assert.match(thrown.message, /wait timed out for selector/);
617+
assert.equal(thrown.details?.reason, undefined);
618+
return true;
619+
},
620+
);
621+
});
622+
623+
test('runtime wait without a recorded landmark returns the satisfying match for record-time evidence', async () => {
624+
const device = landmarkWaitDevice([landmarkScreen('Detail Screen')]);
625+
626+
const result = await device.selectors.wait({
627+
session: 'default',
628+
target: { kind: 'selector', selector: 'label="Screen X"', timeoutMs: 1000 },
629+
});
630+
631+
assert.equal(result.kind, 'selector');
632+
if (result.kind !== 'selector') throw new Error('unreachable');
633+
assert.equal(result.node?.label, 'Screen X');
634+
assert.equal(result.preActionNodes?.length, 2);
635+
});

‎src/commands/interaction/runtime/selector-read.ts‎

Lines changed: 121 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,25 @@ import { parseSelectorChain, type SelectorChain } from '../../../selectors/parse
1616
import {
1717
findSelectorChainMatch,
1818
formatSelectorFailure,
19+
listSelectorChainMatches,
1920
resolveSelectorChain,
2021
selectorFailureHint,
2122
} from '../../../selectors/index.ts';
23+
import type { SelectorChainMatchList } from '../../../selectors/resolve.ts';
24+
import {
25+
readNodeLocalIdentity,
26+
WAIT_LANDMARK_MISMATCH_REASON,
27+
type WaitLandmarkMismatchEvidence,
28+
} from '../../../replay/target-identity-node.ts';
29+
import {
30+
buildAncestryChain,
31+
buildIndexMap,
32+
filterIdentitySet,
33+
} from '../../../replay/target-evidence-tree.ts';
34+
import {
35+
annotationLocalIdentity,
36+
type TargetAnnotationV1,
37+
} from '../../../replay/target-identity.ts';
2238
import { buildSelectorChainForNode } from '../../../selectors/build.ts';
2339
import {
2440
evaluateIsPredicate,
@@ -108,6 +124,8 @@ export type IsCommandOptions = CommandContext &
108124
predicate: 'visible' | 'hidden' | 'exists' | 'editable' | 'selected' | 'focused' | 'text';
109125
selector: string;
110126
expectedText?: string;
127+
/** ADR 0012 step 4: replay-only post-resolution guard; see resolution.ts. */
128+
expectedResolvedTarget?: ExpectedResolvedTarget;
111129
};
112130

113131
export type IsCommandResult = {
@@ -117,6 +135,9 @@ export type IsCommandResult = {
117135
matches?: number;
118136
text?: string;
119137
selectorChain?: string[];
138+
/** ADR 0012 decision 3 / #1349: the resolved node and its tree, for record-time evidence (absent for `exists`). */
139+
node?: SnapshotNode;
140+
preActionNodes?: SnapshotNode[];
120141
};
121142

122143
export type WaitCommandOptions = CommandContext &
@@ -125,14 +146,33 @@ export type WaitCommandOptions = CommandContext &
125146
| { kind: 'sleep'; durationMs: number }
126147
| { kind: 'text'; text: string; timeoutMs?: number | null }
127148
| { kind: 'ref'; ref: string; timeoutMs?: number | null }
128-
| { kind: 'selector'; selector: string; timeoutMs?: number | null }
149+
| {
150+
kind: 'selector';
151+
selector: string;
152+
timeoutMs?: number | null;
153+
/**
154+
* ADR 0012 / #1349, replay-only: the recorded landmark identity this
155+
* wait must observe before reporting success. Polling is unchanged —
156+
* the loop keeps waiting while no selector match carries this
157+
* identity, and a timeout with rejected candidates throws the
158+
* `WAIT_LANDMARK_MISMATCH_REASON` refusal instead of success.
159+
*/
160+
recordedLandmark?: TargetAnnotationV1;
161+
}
129162
| { kind: 'stable'; quietMs?: number | null; timeoutMs?: number | null };
130163
};
131164

132165
export type WaitCommandResult =
133166
| { kind: 'sleep'; waitedMs: number }
134167
| { kind: 'text'; waitedMs: number; text: string }
135-
| { kind: 'selector'; waitedMs: number; selector: string }
168+
| {
169+
kind: 'selector';
170+
waitedMs: number;
171+
selector: string;
172+
/** ADR 0012 decision 3: the satisfying match and the tree it came from, for record-time evidence. */
173+
node?: SnapshotNode;
174+
preActionNodes?: SnapshotNode[];
175+
}
136176
| {
137177
kind: 'stable';
138178
waitedMs: number;
@@ -343,6 +383,12 @@ export const isCommand: RuntimeCommand<IsCommandOptions, IsCommandResult> = asyn
343383
hint: selectorFailureHint([]),
344384
});
345385
}
386+
assertExpectedResolvedTarget(
387+
resolved.node,
388+
capture.snapshot.nodes,
389+
options.expectedResolvedTarget,
390+
'is',
391+
);
346392
const result = evaluateIsPredicate({
347393
predicate: options.predicate,
348394
node: resolved.node,
@@ -369,6 +415,8 @@ export const isCommand: RuntimeCommand<IsCommandOptions, IsCommandResult> = asyn
369415
selector: resolved.selector.raw,
370416
...(options.predicate === 'text' ? { text: result.actualText } : {}),
371417
selectorChain: chain.selectors.map((entry) => entry.raw),
418+
node: resolved.node,
419+
preActionNodes: capture.snapshot.nodes,
372420
};
373421
};
374422

@@ -417,6 +465,7 @@ export const waitCommand: RuntimeCommand<WaitCommandOptions, WaitCommandResult>
417465
options,
418466
options.target.selector,
419467
options.target.timeoutMs,
468+
options.target.recordedLandmark,
420469
);
421470
}
422471
if (options.target.kind === 'stable') {
@@ -513,28 +562,95 @@ async function waitForSelector(
513562
options: WaitCommandOptions,
514563
selectorExpression: string,
515564
timeoutMs: number | null | undefined,
565+
recordedLandmark: TargetAnnotationV1 | undefined,
516566
): Promise<WaitCommandResult> {
517567
const timeout = timeoutMs ?? DEFAULT_TIMEOUT_MS;
518568
const start = now(runtime);
519569
const chain = parseSelectorChain(selectorExpression);
520570
const capturePolicy = deriveSelectorCapturePolicy({ selectorChain: chain });
571+
// ADR 0012 / #1349: the LAST poll whose capture matched the recorded
572+
// selector without any match carrying the recorded landmark identity. A
573+
// transient same-selector impostor (the previous screen mid-transition)
574+
// must not abort a wait whose job is to wait through it, so the loop keeps
575+
// polling; only the deadline turns this into the fail-closed refusal.
576+
let landmarkMismatch: WaitLandmarkMismatchEvidence | undefined;
521577
while (now(runtime) - start < timeout) {
522578
// Presence-only poll: skip scroll-hint derivation (#1270), same as waitForFindMatch.
523579
const capture = await captureSelectorSnapshot(runtime, options, {
524580
updateSession: true,
525581
includeHiddenContentHints: false,
526582
...capturePolicy,
527583
});
528-
const match = findSelectorChainMatch(capture.snapshot.nodes, chain, {
584+
const nodes = capture.snapshot.nodes;
585+
const matchList = listSelectorChainMatches(nodes, chain, {
529586
platform: runtime.backend.platform,
530587
});
531-
if (match)
532-
return { kind: 'selector', selector: match.selector.raw, waitedMs: now(runtime) - start };
588+
if (matchList) {
589+
const landmark = resolveLandmarkMatch(nodes, matchList, recordedLandmark);
590+
if (landmark.kind === 'satisfied') {
591+
return {
592+
kind: 'selector',
593+
selector: matchList.selector.raw,
594+
waitedMs: now(runtime) - start,
595+
node: landmark.node,
596+
preActionNodes: nodes,
597+
};
598+
}
599+
landmarkMismatch = landmark.evidence;
600+
}
533601
await sleep(runtime, POLL_INTERVAL_MS);
534602
}
603+
if (landmarkMismatch) {
604+
throw new AppError(
605+
'COMMAND_FAILED',
606+
`wait matched selector ${selectorExpression} but no candidate carried the recorded landmark identity`,
607+
{ reason: WAIT_LANDMARK_MISMATCH_REASON, ...landmarkMismatch },
608+
);
609+
}
535610
throw new AppError('COMMAND_FAILED', `wait timed out for selector: ${selectorExpression}`);
536611
}
537612

613+
type LandmarkMatchOutcome =
614+
| { kind: 'satisfied'; node: SnapshotNode }
615+
| { kind: 'identity-mismatch'; evidence: WaitLandmarkMismatchEvidence };
616+
617+
/**
618+
* #1349 landmark check: the wait is satisfied when SOME selector match
619+
* carries the recorded identity (local identity + leaf-anchored ancestry
620+
* prefix). Positional disambiguation signals are deliberately not consulted —
621+
* a destination guard proves the landmark exists on the ready screen, not
622+
* that it kept its list position.
623+
*/
624+
function resolveLandmarkMatch(
625+
nodes: SnapshotNode[],
626+
matchList: SelectorChainMatchList,
627+
recorded: TargetAnnotationV1 | undefined,
628+
): LandmarkMatchOutcome {
629+
const firstMatch = matchList.matchedNodes[0]!;
630+
if (!recorded) return { kind: 'satisfied', node: firstMatch };
631+
const byIndex = buildIndexMap(nodes);
632+
const identitySet = filterIdentitySet(
633+
matchList.matchedNodes,
634+
byIndex,
635+
annotationLocalIdentity(recorded),
636+
recorded.ancestry,
637+
);
638+
const member = identitySet[0];
639+
if (member) return { kind: 'satisfied', node: member };
640+
return {
641+
kind: 'identity-mismatch',
642+
evidence: {
643+
matchCount: matchList.matchedNodes.length,
644+
observed: readNodeLocalIdentity(firstMatch),
645+
observedAncestry: buildAncestryChain(
646+
firstMatch,
647+
byIndex,
648+
Math.max(recorded.ancestry.length, 1),
649+
).chain,
650+
},
651+
};
652+
}
653+
538654
async function waitForText(
539655
runtime: AgentDeviceRuntime,
540656
options: WaitCommandOptions,

‎src/core/command-descriptor/__tests__/parity.test.ts‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -388,3 +388,27 @@ test('recordingEffect resolves request-sensitive observation and mutation subcom
388388
'mutates-app',
389389
);
390390
});
391+
392+
test('targetIdentityVerification pins exactly the evidence-carrying command set (ADR 0012 / #1349)', () => {
393+
const declared = new Map(
394+
RAW_COMMAND_DESCRIPTORS.filter((descriptor) => descriptor.targetIdentityVerification).map(
395+
(descriptor) => [descriptor.name, descriptor.targetIdentityVerification],
396+
),
397+
);
398+
// A new evidence-carrying command must choose its replay verification phase
399+
// here explicitly instead of silently entering the generic pre-dispatch
400+
// path — wait is the only command whose target may legitimately be absent
401+
// when its step starts.
402+
assert.deepEqual(
403+
[...declared.entries()].sort(([a], [b]) => a.localeCompare(b)),
404+
[
405+
['click', 'pre-dispatch'],
406+
['fill', 'pre-dispatch'],
407+
['get', 'pre-dispatch'],
408+
['is', 'pre-dispatch'],
409+
['longpress', 'pre-dispatch'],
410+
['press', 'pre-dispatch'],
411+
['wait', 'post-resolution'],
412+
],
413+
);
414+
});

0 commit comments

Comments
 (0)