Skip to content

Commit e66ec2a

Browse files
committed
fix(interaction): declare readiness intent and preserve capture facts
1 parent 6b7c8ea commit e66ec2a

6 files changed

Lines changed: 110 additions & 20 deletions

File tree

‎packages/command-runtime/src/commands/interaction/runtime/interactions.ts‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,20 @@ async function tapCommand(
159159
if (!runtime.backend.tap) {
160160
throw new AppError('UNSUPPORTED_OPERATION', 'tap is not supported by this backend');
161161
}
162-
const atRest = await admitResolvedTarget(runtime, options, resolutionRequest, resolved);
162+
const atRest = await admitResolvedTarget(
163+
runtime,
164+
options,
165+
resolutionRequest,
166+
resolved,
167+
(options.count ?? 1) === 1 &&
168+
(options.intervalMs ?? 0) === 0 &&
169+
(options.holdMs ?? 0) === 0 &&
170+
(options.jitterPx ?? 0) === 0 &&
171+
options.doubleTap !== true &&
172+
(options.button ?? 'primary') === 'primary'
173+
? 'when-unknown'
174+
: 'skip',
175+
);
163176
const point = requireResolvedPoint(atRest);
164177
const pointObservation = pointObservationOf(atRest);
165178
let backendResult;

‎packages/command-runtime/src/commands/interaction/runtime/moving-target-gate.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -279,6 +279,7 @@ test('a ref the second observation cannot re-measure is refused, never self-agre
279279
const details = (error as { details?: Record<string, unknown> }).details;
280280
assert.equal(details?.reason, INTERACTION_ERROR_REASONS.targetStillMoving);
281281
assert.equal(details?.dispatched, 'no');
282+
assert.equal(details?.waitedMs, MOVING_TARGET_GATE_SAMPLE_GAP_MS);
282283
// Nothing was measured, so nothing is reported: a caller reading `movedToRect` would take the
283284
// stale frame position for a fresh observation of where the target went.
284285
assert.equal(details?.movedToRect, undefined);

‎packages/command-runtime/src/commands/interaction/runtime/pre-dispatch-admission-readiness.test.ts‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,14 @@ async function exercise(input: {
8080
resultDeadline?: boolean;
8181
missingSource?: boolean;
8282
movingChild?: boolean;
83-
options?: { count?: number; doubleTap?: boolean };
83+
options?: {
84+
count?: number;
85+
doubleTap?: boolean;
86+
intervalMs?: number;
87+
holdMs?: number;
88+
jitterPx?: number;
89+
button?: 'primary' | 'secondary';
90+
};
8491
}) {
8592
const events: string[] = [];
8693
const points: Point[] = [];
@@ -377,7 +384,11 @@ test('a child moving into the old parent point uses the fresh canonically guarde
377384
expect(points[0]!.y).toBeLessThan(166);
378385
});
379386

380-
for (const options of [{ count: 1 }, { doubleTap: false }]) {
387+
for (const options of [
388+
{ count: 1 },
389+
{ doubleTap: false },
390+
{ count: 1, doubleTap: false, intervalMs: 0, holdMs: 0, jitterPx: 0, button: 'primary' },
391+
] as const) {
381392
test(`explicit ordinary tap defaults still await readiness: ${JSON.stringify(options)}`, async () => {
382393
const { run, events, points } = await exercise({ states: ['blocked', 'ready'], options });
383394
await run;
@@ -386,6 +397,22 @@ for (const options of [{ count: 1 }, { doubleTap: false }]) {
386397
});
387398
}
388399

400+
for (const options of [
401+
{ count: 2 },
402+
{ doubleTap: true },
403+
{ intervalMs: 1 },
404+
{ holdMs: 25 },
405+
{ jitterPx: 1 },
406+
{ button: 'secondary' },
407+
] as const) {
408+
test(`non-default tap intent skips readiness: ${JSON.stringify(options)}`, async () => {
409+
const { run, events, points } = await exercise({ states: ['blocked'], options });
410+
await run;
411+
expect(events).not.toContain('read');
412+
expect(points).toHaveLength(1);
413+
});
414+
}
415+
389416
for (const producer of ['simulator-ax-bridge', 'apple-runner'] as const) {
390417
test(`${producer} capture stored and read through the session retains raw native row binding privately`, async () => {
391418
const screen = { x: 0, y: 0, width: 400, height: 800 };

‎packages/command-runtime/src/commands/interaction/runtime/pre-dispatch-admission.ts‎

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ import type {
1212
InteractionTarget,
1313
ResolvedInteractionTarget,
1414
} from '@agent-device/contracts/interaction';
15-
import type { BackendTapOptions } from '@agent-device/contracts/backend';
1615
import type { NativePressReadiness } from '@agent-device/contracts/selector-observation-runtime';
1716
import type { AgentDeviceRuntime, CommandContext } from '@agent-device/contracts/runtime-contract';
1817
import {
@@ -40,24 +39,18 @@ import { INTERACTION_ERROR_REASONS } from '@agent-device/selectors/interaction-e
4039

4140
export async function admitResolvedTarget(
4241
runtime: AgentDeviceRuntime,
43-
options: CommandContext & BackendTapOptions & { target: InteractionTarget },
42+
options: CommandContext & { target: InteractionTarget },
4443
params: ResolveInteractionTargetParams,
4544
resolved: ResolvedInteractionTarget,
45+
nativeReadiness: 'when-unknown' | 'skip',
4646
): Promise<ResolvedInteractionTarget> {
4747
if (resolved.kind === 'point' || !resolved.node?.rect) return resolved;
4848
if (params.expectedResolvedTarget || params.preresolvedTarget) return resolved;
4949
const budget = requestBudget(runtime, options);
5050
const screening = await budget.read(() =>
5151
screenMotion(runtime, options, resolved.node!, resolved.preAction),
5252
);
53-
const native =
54-
!!runtime.backend.readPressReadiness &&
55-
(options.count ?? 1) === 1 &&
56-
(options.intervalMs ?? 0) === 0 &&
57-
(options.holdMs ?? 0) === 0 &&
58-
(options.jitterPx ?? 0) === 0 &&
59-
options.doubleTap !== true &&
60-
(options.button ?? 'primary') === 'primary';
53+
const native = nativeReadiness === 'when-unknown' && !!runtime.backend.readPressReadiness;
6154
if (screening.kind === 'pass' && !native) {
6255
assertWithinBudget(budget);
6356
return resolved;
@@ -266,7 +259,8 @@ async function checkMotion(
266259
state.motion = undefined;
267260
return true;
268261
}
269-
if (verdict.kind === 'refuse') throw motionRefusal(state.resolved, undefined, 0);
262+
if (verdict.kind === 'refuse')
263+
throw motionRefusal(state.resolved, undefined, look.observedAtMs - motion.wait.firstLookAtMs);
270264
if (verdict.kind === 'wait') motion.restingSinceMs = verdict.restingSinceMs;
271265
if (look.observedAtMs < motion.wait.giveUpAtMs)
272266
motion.endsAtMs = Math.max(

‎packages/daemon/src/core/__tests__/snapshot-capture.test.ts‎

Lines changed: 60 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
11
import { expect, test, vi } from 'vitest';
22
import { captureSnapshot, captureSnapshotData } from '../snapshot-capture.ts';
3-
import { summarizeSnapshotDiagnostics } from '@agent-device/contracts/capture';
3+
import {
4+
attachSnapshotClickabilityEvidence,
5+
attachSnapshotOcclusionContextEvidence,
6+
readSnapshotClickabilityEvidence,
7+
readSnapshotOcclusionContextEvidence,
8+
summarizeSnapshotDiagnostics,
9+
} from '@agent-device/contracts/capture';
410
import { buildSnapshotVisibility } from '@agent-device/capture-kit/snapshot-visibility';
511
import { ANDROID_EMULATOR, buildNodes, IOS_SIMULATOR, MACOS_DEVICE } from '@agent-device/test-kit';
612
import {
@@ -68,6 +74,59 @@ test('snapshot capture preserves scope for every other platform projection', asy
6874
]);
6975
});
7076

77+
test('snapshot capture retains Android facts at their normalized node indices', async () => {
78+
const data = {
79+
backend: 'android' as const,
80+
producer: 'android-uiautomator' as const,
81+
nodes: [
82+
{ index: 10, depth: 0, type: 'Button', label: 'Save' },
83+
{ index: 20, depth: 0, type: 'Button', label: 'Cancel' },
84+
],
85+
};
86+
attachSnapshotClickabilityEvidence(data, {
87+
kind: 'exact',
88+
provider: 'android-helper',
89+
clickableByNodeIndex: new Map([
90+
[10, true],
91+
[20, false],
92+
]),
93+
});
94+
attachSnapshotOcclusionContextEvidence(data, {
95+
nodes: [{ index: 100 }, { index: 200 }],
96+
sourceIndexByNodeIndex: new Map([
97+
[10, 100],
98+
[20, 200],
99+
]),
100+
});
101+
102+
const { snapshot } = await captureSnapshot({
103+
device: ANDROID_EMULATOR,
104+
session: undefined,
105+
flags: { snapshotRaw: true },
106+
logPath: '/tmp/snapshot-capture-test.log',
107+
captureData: async () => data,
108+
});
109+
110+
expect(snapshot.nodes.map((node) => [node.index, node.label])).toEqual([
111+
[0, 'Save'],
112+
[1, 'Cancel'],
113+
]);
114+
expect(readSnapshotClickabilityEvidence(snapshot)).toEqual({
115+
kind: 'exact',
116+
provider: 'android-helper',
117+
clickableByNodeIndex: new Map([
118+
[0, true],
119+
[1, false],
120+
]),
121+
});
122+
expect(readSnapshotOcclusionContextEvidence(snapshot)?.sourceIndexByNodeIndex).toEqual(
123+
new Map([
124+
[0, 100],
125+
[1, 200],
126+
]),
127+
);
128+
});
129+
71130
test('buildSnapshotVisibility returns non-partial for empty node list', () => {
72131
const vis = buildSnapshotVisibility({ nodes: [], backend: 'android' });
73132
expect(vis.partial).toBe(false);

‎packages/daemon/src/core/snapshot-capture.ts‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
import type { CommandFlags } from '@agent-device/contracts/command';
22
import {
3-
copySnapshotClickabilityEvidence,
43
recordSnapshotTiming,
54
snapshotCaptureAnnotationsFrom,
65
type SnapshotCaptureAnnotations,
@@ -139,10 +138,7 @@ async function captureSnapshotAttempt(params: CaptureSnapshotParams): Promise<Sn
139138
platform: publicPlatformString(params.device),
140139
});
141140
const annotations = snapshotCaptureAnnotationsFrom(data);
142-
const snapshot = copySnapshotClickabilityEvidence(
143-
data,
144-
buildSnapshotState(data, resolveSnapshotStateFlags(params)),
145-
);
141+
const snapshot = buildSnapshotState(data, resolveSnapshotStateFlags(params));
146142
// The one seam where snapshot state and capture annotations meet: consumers that only keep the
147143
// SnapshotState (selector-backed find/wait, session-stored snapshots) must still learn that the
148144
// capture is an occluding system surface, so the disclosure is not lost with the annotations.

0 commit comments

Comments
 (0)