Skip to content

Commit e6cbe6b

Browse files
committed
test(replay): cover pre-step artifact ordering and resume-before-mutation (#1555)
Two invariants found during the P5 decomposition pass now have direct counterfactual-verified coverage: - packages/ad-replay/src/internal/__tests__/step-loop.test.ts: a post-dispatch target-binding mismatch (dispatchWithGuard) must report the accumulated PRE-step artifact snapshot it was called with, never the artifacts the failed dispatch itself produced. Verified red by swapping the buildPostDispatchTargetBindingFailure call to outcome.artifactPaths. - src/daemon/handlers/__tests__/session-replay-runtime-plan.test.ts: a rejected --from/--plan-digest resume must never reach prepareReplaySession's coordinator-mutating writes (the R2 ordering invariant) — a pre-armed repair transaction and corrective-resume watermark are asserted byte-for-byte unchanged after rejection. Verified red by calling prepareReplaySession before honoring the plan-validation rejection.
1 parent 7deee4e commit e6cbe6b

2 files changed

Lines changed: 156 additions & 0 deletions

File tree

‎packages/ad-replay/src/internal/__tests__/step-loop.test.ts‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import assert from 'node:assert/strict';
22
import { test } from 'vitest';
33
import { runAdReplay, type AdReplayStepRuntime } from '../step-loop.ts';
44
import type { SessionAction } from '@agent-device/contracts/session';
5+
import type { TargetAnnotationV1 } from '@agent-device/contracts/replay';
56
import type { ReplaySelectorPort } from '../selector-port.ts';
67

78
/**
@@ -121,3 +122,97 @@ test('a close-less plan suppresses nothing and arms every executable step, inclu
121122
// suppressed `close` is still armed, just never dispatched.
122123
assert.equal(armCount(), 2);
123124
});
125+
126+
/**
127+
* #1555 P5 stage R3 invariant: `buildPostDispatchTargetBindingFailure`'s
128+
* `artifactPaths` argument is the run's accumulated PRE-step snapshot — the
129+
* same value `verifyAndDispatchStep`/`dispatchWithGuard` were called with —
130+
* never the artifacts the just-failed dispatch itself produced. A dispatch
131+
* that races a post-resolution refusal (guard-mismatch/landmark-mismatch)
132+
* may have taken its own screenshot as part of resolving (or failing to
133+
* resolve) the action; that capture belongs to the failed attempt, not to
134+
* the divergence report, which describes the screen BEFORE the action ran.
135+
*/
136+
test("a post-dispatch target-binding mismatch reports the pre-step artifact snapshot, not the failed dispatch's own", async () => {
137+
const recorded: TargetAnnotationV1 = {
138+
role: 'button',
139+
ancestry: [],
140+
sibling: 0,
141+
viewportOrder: 0,
142+
verification: 'verified',
143+
};
144+
const openAction = action('open');
145+
const waitAction: SessionAction = {
146+
...action('wait'),
147+
targetEvidence: recorded,
148+
};
149+
150+
let receivedArtifactPaths: readonly string[] | undefined;
151+
const runtime: AdReplayStepRuntime = {
152+
port: {} as ReplaySelectorPort,
153+
// Only `waitAction` carries `targetEvidence`, so this is only ever
154+
// called for it — routed to the #1349 deferred-landmark path, which
155+
// dispatches with a guard WITHOUT any capture/classify round trip.
156+
beginTargetVerification: () => ({ kind: 'post-resolution', isSelectorWait: true }),
157+
captureObservation: async () => {
158+
throw new Error(
159+
'captureObservation: not used — deferred-landmark skips straight to dispatch',
160+
);
161+
},
162+
classifyTarget: () => {
163+
throw new Error('classifyTarget: not used — deferred-landmark skips straight to dispatch');
164+
},
165+
async dispatchStep(dispatchedAction, _index, artifactPaths, _guard) {
166+
if (dispatchedAction.command === 'open') {
167+
return { status: 'ok', artifactPaths: ['open-snapshot.png'] };
168+
}
169+
// The wait's own dispatch attempt produced a DIFFERENT artifact set
170+
// than the pre-step snapshot it was called with (`artifactPaths`,
171+
// asserted below never to leak into the divergence report).
172+
assert.deepEqual(artifactPaths, ['open-snapshot.png']);
173+
return {
174+
status: 'landmark-mismatch',
175+
details: {},
176+
plainFailure: { kind: 'REPLAY_DIVERGENCE', message: 'mismatch', artifactPaths: [] },
177+
artifactPaths: ['open-snapshot.png', 'post-dispatch-only.png'],
178+
};
179+
},
180+
buildRecordedUnverifiableFailure: async () => {
181+
throw new Error('buildRecordedUnverifiableFailure: not used by this fixture');
182+
},
183+
buildTargetBindingFailure: async () => {
184+
throw new Error(
185+
'buildTargetBindingFailure: not used by this fixture (this is a POST-dispatch mismatch)',
186+
);
187+
},
188+
async buildPostDispatchTargetBindingFailure(
189+
_dispatchedAction,
190+
_index,
191+
_evidence,
192+
artifactPaths,
193+
) {
194+
receivedArtifactPaths = artifactPaths;
195+
return { kind: 'REPLAY_DIVERGENCE', message: 'mismatch', artifactPaths: [] };
196+
},
197+
handleActionFailure: async ({ artifactPaths }) => ({
198+
kind: 'REPLAY_DIVERGENCE',
199+
message: 'mismatch',
200+
artifactPaths: [...artifactPaths],
201+
}),
202+
armStep: () => {},
203+
isRepairArmed: () => false,
204+
describeStepValue: () => undefined,
205+
diagnosticsMarker: () => 0,
206+
diagnosticsSince: () => [],
207+
};
208+
209+
const outcome = await runAdReplay(
210+
{ actions: [openAction, waitAction], entryIndex: 0, keepSession: false },
211+
runtime,
212+
);
213+
214+
assert.equal(outcome.status, 'failed');
215+
// The divergence reports the snapshot taken BEFORE the wait's dispatch —
216+
// `open`'s own artifact, nothing the failed dispatch itself produced.
217+
assert.deepEqual(receivedArtifactPaths, ['open-snapshot.png']);
218+
});

‎src/daemon/handlers/__tests__/session-replay-runtime-plan.test.ts‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import os from 'node:os';
1010
import path from 'node:path';
1111
import { runReplayScriptFile } from '../session-replay-runtime.ts';
1212
import { SessionStore } from '../../session-store.ts';
13+
import { createReplayCoordinator } from '../../session-replay-coordinator.ts';
1314
import { dispatchCommand, resolveTargetDevice } from '../../../core/dispatch.ts';
1415
import {
1516
makeAndroidSession,
@@ -145,6 +146,66 @@ test('resume rejects an out-of-range --from before any action', async () => {
145146
expect(response.error.message).toMatch(/out of range/);
146147
});
147148

149+
/**
150+
* R2: `prepareReplayPlan`'s `--from`/`--plan-digest` validation
151+
* (`resolveReplayPlanEntryIndex`, now in `session-replay-runtime-plan.ts`)
152+
* must run — and reject — before `prepareReplaySession`
153+
* (`session-replay-runtime-session.ts`) performs any coordinator-mutating
154+
* write. Were the order reversed, a rejected `--from` would still clear the
155+
* corrective-resume watermark and demote the armed repair transaction before
156+
* the request failed, silently corrupting the very repair state `--from`
157+
* exists to protect.
158+
*/
159+
test("a rejected --from/--plan-digest resume never reaches prepareReplaySession's coordinator-mutating writes", async () => {
160+
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-resume-no-mutate-'));
161+
const sessionStore = new SessionStore(path.join(root, 'sessions'));
162+
const sessionName = 'default';
163+
sessionStore.set(sessionName, makeIosSession(sessionName));
164+
const filePath = writeReplayFile(root, ['open "Demo"', 'click "Continue"', 'click "Save"']);
165+
166+
// Arm a repair transaction and stamp a corrective-resume watermark
167+
// directly, so BOTH of `prepareReplaySession`'s coordinator-mutating
168+
// writes — `consumeReplayResumeState`'s watermark-clear (the `--from 2`
169+
// below matches `expectedFrom`, so it WOULD clear) and
170+
// `prepareSaveScriptSession`'s `demoteForRerunIfArmed` — have something
171+
// real to mutate if this rejected request ever reaches them.
172+
const coordinator = createReplayCoordinator({ sessionStore, sessionName });
173+
coordinator.armStep({ saveScript: true, force: undefined, sourcePath: filePath, firstArm: true });
174+
const armedSession = sessionStore.get(sessionName)!;
175+
// `actionsCountAtDivergence: 999` keeps `describeUnperformedRecordAndHeal`
176+
// from firing first (it needs `sessionActionsLength` to equal this), so
177+
// the rejection below is provably the plan-digest check, not a different one.
178+
armedSession.pendingRecordAndHeal = { expectedFrom: 2, actionsCountAtDivergence: 999 };
179+
sessionStore.set(sessionName, armedSession);
180+
181+
const beforeView = coordinator.view();
182+
const beforeActionsLength = sessionStore.get(sessionName)!.actions.length;
183+
184+
const response = await runReplayScriptFile({
185+
req: baseReq({
186+
positionals: [filePath],
187+
flags: { replayFrom: 2, replayPlanDigest: 'not-the-real-digest' },
188+
}),
189+
sessionName,
190+
logPath: path.join(root, 'daemon.log'),
191+
sessionStore,
192+
invoke: async () => {
193+
throw new Error('must not execute a resume the plan-digest preflight rejected');
194+
},
195+
});
196+
197+
expect(response.ok).toBe(false);
198+
if (response.ok) return;
199+
expect(response.error.code).toBe('INVALID_ARGS');
200+
expect(response.error.message).toMatch(/plan digest/);
201+
202+
// The armed repair boundary and the corrective watermark are
203+
// byte-for-byte unchanged, and no session action was recorded — proof the
204+
// rejection happened before `prepareReplaySession` ran at all.
205+
expect(coordinator.view()).toEqual(beforeView);
206+
expect(sessionStore.get(sessionName)!.actions.length).toBe(beforeActionsLength);
207+
});
208+
148209
test('resume rejects a stale --plan-digest after the script changed', async () => {
149210
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-resume-stale-digest-'));
150211
const sessionStore = new SessionStore(path.join(root, 'sessions'));

0 commit comments

Comments
 (0)