Skip to content

Commit e86c9d5

Browse files
author
Cameron Chisholm
committed
refactor(gesture): keep cancellation checks at daemon boundaries
1 parent 58eeb12 commit e86c9d5

6 files changed

Lines changed: 116 additions & 15 deletions

File tree

‎packages/daemon/src/execution/interaction/internal/interaction-gesture-tap.test.ts‎

Lines changed: 104 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { afterEach, beforeEach, expect, test, vi } from 'vitest';
22
import {
33
makeIosSession,
4+
makeAndroidSession,
45
authoringPublication,
56
} from '../../../__tests__/support/session-factories.ts';
67
import { makeSessionStore } from '../../../__tests__/support/store-factory.ts';
@@ -11,6 +12,8 @@ import {
1112
markRequestCanceled,
1213
clearRequestCanceled,
1314
} from '@agent-device/host-kit/request';
15+
import type { AndroidObservationAdapter } from '@agent-device/contracts/android-observation';
16+
import { clearAndroidObservationFixture } from '../../../__tests__/support/android-observation-fixture.ts';
1417
import { handleInteractionCommands } from '../index.ts';
1518
import { gestureRuntimeBindingsFixture } from './__tests__/gesture-runtime-bindings.fixtures.ts';
1619

@@ -27,7 +30,7 @@ const perform = vi.fn(async () => ({
2730

2831
beforeEach(() => {
2932
gestures = gestureRuntimeBindingsFixture();
30-
perform.mockClear();
33+
perform.mockReset();
3134
registerRequestAbort(requestId);
3235
});
3336

@@ -39,9 +42,10 @@ afterEach(() => {
3942
async function runGesture(
4043
input: Record<string, unknown>,
4144
command: 'gesture' | 'swipe' = 'gesture',
45+
androidObservation?: AndroidObservationAdapter,
4246
) {
4347
const sessionStore = makeSessionStore();
44-
const session = makeIosSession('tap', {
48+
const session = (androidObservation ? makeAndroidSession : makeIosSession)('tap', {
4549
appBundleId: 'com.example.tap',
4650
scriptPublication: authoringPublication('armed'),
4751
});
@@ -52,6 +56,7 @@ async function runGesture(
5256
return { ...surface, interactions: { ...surface.interactions, gesture: perform } };
5357
},
5458
platformServices: daemonPlatformServicesFixture(),
59+
androidObservation,
5560
req: {
5661
token: 't',
5762
session: 'tap',
@@ -201,3 +206,100 @@ test('cancellation during the last tap records no completed action', async () =>
201206
expect(actions).toEqual([]);
202207
expect(performed).toBe(3);
203208
});
209+
210+
for (const canceled of [false, true]) {
211+
test(`Android post-tap ANR recovery is ${canceled ? 'skipped after cancellation' : 'performed without cancellation'}`, async () => {
212+
let dispatched = false;
213+
const tap = vi.fn(clearAndroidObservationFixture.tap);
214+
const openApp = vi.fn(clearAndroidObservationFixture.openApp);
215+
const readBlockingDialog = vi.fn<AndroidObservationAdapter['readBlockingDialog']>(async () =>
216+
dispatched
217+
? {
218+
status: 'dialog',
219+
focus: {
220+
package: 'com.example.tap',
221+
focusedWindow: 'Application Not Responding',
222+
raw: '',
223+
},
224+
}
225+
: { status: 'clear' },
226+
);
227+
const androidObservation: AndroidObservationAdapter = {
228+
...clearAndroidObservationFixture,
229+
readBlockingDialog,
230+
readSnapshotNodes: async () => [
231+
{
232+
index: 0,
233+
ref: 'e0',
234+
type: 'Button',
235+
label: 'Close app',
236+
rect: { x: 10, y: 20, width: 100, height: 40 },
237+
},
238+
],
239+
tap,
240+
openApp,
241+
};
242+
const result = await perform();
243+
perform.mockClear();
244+
perform.mockImplementation(async () => {
245+
dispatched = true;
246+
if (canceled) markRequestCanceled(requestId);
247+
return result;
248+
});
249+
const { response, actions } = await runGesture(
250+
{
251+
kind: 'tap',
252+
points: [
253+
{ x: 80, y: 300 },
254+
{ x: 200, y: 450 },
255+
],
256+
},
257+
'gesture',
258+
androidObservation,
259+
);
260+
expect(perform).toHaveBeenCalledTimes(1);
261+
expect(response).toMatchObject({ ok: false, error: { code: 'COMMAND_FAILED' } });
262+
if (canceled) {
263+
expect(response).toMatchObject({ error: { details: { reason: 'request_canceled' } } });
264+
expect(readBlockingDialog).toHaveBeenCalledTimes(1);
265+
expect(tap).not.toHaveBeenCalled();
266+
expect(openApp).not.toHaveBeenCalled();
267+
} else {
268+
expect(response && !response.ok ? response.error.details : undefined).not.toHaveProperty(
269+
'reason',
270+
'request_canceled',
271+
);
272+
expect(tap).toHaveBeenCalledTimes(1);
273+
expect(openApp).toHaveBeenCalledTimes(1);
274+
}
275+
expect(actions).toEqual([]);
276+
});
277+
}
278+
279+
test('cancellation during the post-command dialog read prevents recording', async () => {
280+
let reads = 0;
281+
const androidObservation: AndroidObservationAdapter = {
282+
...clearAndroidObservationFixture,
283+
readBlockingDialog: async () => {
284+
if (++reads === 2) markRequestCanceled(requestId);
285+
return { status: 'clear' };
286+
},
287+
};
288+
const { response, actions } = await runGesture(
289+
{
290+
kind: 'tap',
291+
points: [
292+
{ x: 80, y: 300 },
293+
{ x: 200, y: 450 },
294+
],
295+
},
296+
'gesture',
297+
androidObservation,
298+
);
299+
expect(reads).toBe(2);
300+
expect(response).toMatchObject({
301+
ok: false,
302+
error: { code: 'COMMAND_FAILED', details: { reason: 'request_canceled' } },
303+
});
304+
expect(actions).toEqual([]);
305+
});

‎packages/daemon/src/execution/interaction/internal/interaction-gesture.ts‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -152,10 +152,14 @@ function buildGestureOutcome(
152152
const recording = result.kind === 'drag' ? result.recording : undefined;
153153
const sourceTarget = recording?.sourceTarget;
154154
const destinationTarget = recording?.destinationTarget;
155+
const responseResult =
156+
input.kind === 'tap' && (input.count ?? 1) > 1
157+
? { ...result, message: `Tapped ${input.points.length} points ${input.count} times` }
158+
: result;
155159
return {
156160
positionals: dragRecordingPositionals(input, recording),
157161
flags: gestureReplayFlags(input, flags),
158-
responseData: gestureResponseData(result, {
162+
responseData: gestureResponseData(responseResult, {
159163
executionProfile: resolveExecutionProfile(gesture),
160164
...(input.kind === 'tap'
161165
? {
@@ -442,13 +446,9 @@ async function runGestureRepetitions(
442446
gestureAtIndex(index),
443447
params.req.internal,
444448
);
445-
signal?.throwIfAborted();
446449
if (pauseMs > 0 && index + 1 < count) await sleep(pauseMs, signal);
447450
}
448451
if (!result) throw new Error('Gesture orchestration did not execute a gesture.');
449-
if (result.kind === 'tap' && count > 1) {
450-
return { ...result, message: `Tapped ${result.pointerCount} points ${count} times` };
451-
}
452452
return result;
453453
}
454454

‎src/commands/interaction/runtime/gesture-command.ts‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -48,16 +48,12 @@ const coordinateGestureCommand: RuntimeCommand<
4848
}
4949
await assertSupportedInteractionSurface(runtime, options, options.gesture.intent);
5050
const performGesture = runtime.backend.performGesture;
51-
const signal = options.signal ?? runtime.signal;
52-
signal?.throwIfAborted();
5351
const plan = buildGesturePlan(
5452
options.gesture,
5553
await resolveGestureViewport(runtime, options),
5654
runtime.backend.platform,
5755
);
58-
signal?.throwIfAborted();
5956
const backendResult = await performGesture(toBackendContext(runtime, options), plan);
60-
signal?.throwIfAborted();
6157
const formattedBackendResult = toBackendResult(backendResult);
6258
const from = centroidAt(plan.pointers, 0);
6359
const to = centroidAt(plan.pointers, -1);

‎src/commands/interaction/runtime/gestures.test.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -257,13 +257,14 @@ test('each tap plans one repetition against a fresh viewport', async () => {
257257
assert.equal(result.message, 'Tapped 2 points');
258258
});
259259

260-
test('an aborted tap propagates its typed cancellation even when the backend returns normally', async () => {
260+
test('a tap forwards its signal and propagates backend cancellation', async () => {
261261
const controller = new AbortController();
262262
const reason = createRequestCanceledError();
263263
const device = createInteractionDevice(selectorSnapshot(), {
264264
resolveGestureViewport: async () => ({ x: 0, y: 0, width: 400, height: 800 }),
265-
performGesture: async () => {
265+
performGesture: async (context) => {
266266
controller.abort(reason);
267+
context.signal?.throwIfAborted();
267268
},
268269
});
269270
await assert.rejects(

‎src/commands/schema/cli-help-gestures.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,10 @@ Shapes:
2323
agent-device gesture tap 80 300 200 450 310 600 --count 20 --pause-ms 100
2424
scroll <dir> --until <selector> repeats scroll-and-check passes until that element is on screen, then stops: one request instead of a scroll-then-snapshot loop, and it stops on the target rather than overshooting it. It reports the passes it spent, fails when the content runs out first, and is refused on top/bottom, which already stop themselves. The default limit is ${DEFAULT_SCROLL_UNTIL_PASSES} gestures. For a longer list, add --max-passes <n> (1–${SCROLL_UNTIL_MAX_PASSES}); it requires --until. Large pass budgets can hit the 90-second request timeout; inspect the current screen and continue with another search if that happens.
2525
longpress accepts coordinates, @refs, or selectors; prefer @ref/selector, coordinates only as a fallback. Duration and gesture scale/center are positional. gesture pan is one finger by default; add --pointer-count 2 for a parallel two-finger pan. Keep count/pause/pattern on one swipe: --count (cap 200), --pause-ms (cap 10000ms), --pattern ping-pong; the combined swipe/pause schedule is capped at 60000ms.
26-
For repeated iOS smoke checks: press <x> <y> --count <n> --jitter-px <n> for tap series, swipe <x1> <y1> <x2> <y2> --count <n> for drag series. gesture tap touches 2 to 5 distinct points and holds them for --hold-ms (default 50, min 16); --count repeats the same contacts (cap 200) with --pause-ms between, the series must fit 60000ms counting about 500ms per repetition, and a repetition costs about 450ms on a physical iPhone. Cancellation interrupts pauses, returns COMMAND_FAILED with details.reason=request_canceled, and does not record the action. Use press for one finger.
26+
For repeated iOS smoke checks: press <x> <y> --count <n> --jitter-px <n> for tap series, swipe <x1> <y1> <x2> <y2> --count <n> for drag series. gesture tap touches 2 to 5 distinct points and holds them for --hold-ms (default 50, min 16); --count repeats the same contacts (cap 200) with --pause-ms between, the series must fit 60000ms counting about 500ms per repetition, and a repetition costs about 450ms on a physical iPhone. Use press for one finger.
2727
2828
Platform quirks:
29+
Cancelled gesture and swipe requests return COMMAND_FAILED with details.reason=request_canceled and record no action. Repetition pauses are interruptible.
2930
iOS transform/pinch/rotate use private XCTest synthesis for a continuous two-finger pan/scale/rotation path; verify app metrics instead of assuming requested values map exactly to recognizer output. A gesture tap of three or more fingers can trigger iOS system gestures (undo/redo, the edit menu) and fast repeats can trigger double-tap recognizers, so verify app state; iOS touches the contacts down and lifts them together; Android touches them down up to 8ms apart and lifts them 8ms apart in reverse order.
3031
Android transform injects a geometric two-finger path; app recognizers may report non-exact pan/scale/rotation -- verify semantic app state or coarse per-component effects instead of exact numeric deltas unless the app exposes stable metrics. If Android needs exact values, prefer isolated gesture pan --pointer-count 2, gesture pinch, or gesture rotate over one combined transform:
3132
agent-device gesture transform 200 420 80 -40 2 35 700 --platform android

‎website/docs/docs/commands.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -492,7 +492,8 @@ agent-device wait 'id="field-phone" value="00 062 91 77"'
492492
- `gesture fling up|down|left|right x y [distance]` is a fast directional throw. Neither `swipe` nor `gesture fling` takes a duration, and `gesture rotate` takes no velocity; see [Rewrite Old Gesture Commands](/docs/migrating-gestures) to update old scripts.
493493
- `gesture drag source destination [sourceHoldMs] [moveMs] [destinationHoldMs]` holds one pointer down from source to destination. Endpoints are selectors or refs, resolved before dispatch and recorded as selector chains. Defaults are 800, 500, and 0ms, capped at 10000ms in total. It works on Android touch devices and iOS/iPadOS; other backends refuse it before touching the screen.
494494
- `gesture pinch <scale> [x] [y]`, `gesture rotate <degrees> [x] [y]` (the sign sets direction), `gesture transform x y dx dy scale degrees [durationMs]`, and two-finger `gesture pan` work in Android, iOS, and iPadOS app sessions, on simulators and physical devices. On Apple they use the XCTest runner, so iOS sessions reached through a WebDriver provider refuse them. One-finger `gesture pan` works wherever coordinate drags do. A path that does not fit the viewport fails before injection instead of being clamped. Use `orientation` to rotate the device.
495-
- `gesture tap x1 y1 x2 y2 [... x5 y5]` touches 2 to 5 distinct points and holds them for `--hold-ms` (default 50ms, at least 16ms) before lifting. `--count` repeats the same contacts up to 200 times with `--pause-ms` between repetitions; the series must fit in 60000ms, counting about 500ms of overhead per repetition. It works in Android and iOS/iPadOS app sessions; WebDriver providers refuse it unless they declare `tapGesture` support. On iOS the contacts touch down and lift together; on Android they touch down up to 8ms apart and lift 8ms apart in reverse order. Three or more fingers can trigger iOS system gestures such as undo, so verify app state afterwards. Cancellation interrupts pauses, returns `COMMAND_FAILED` with `details.reason=request_canceled`, and does not record the action. Use `press` for one finger.
495+
- `gesture tap x1 y1 x2 y2 [... x5 y5]` touches 2 to 5 distinct points and holds them for `--hold-ms` (default 50ms, at least 16ms) before lifting. `--count` repeats the same contacts up to 200 times with `--pause-ms` between repetitions; the series must fit in 60000ms, counting about 500ms of overhead per repetition. It works in Android and iOS/iPadOS app sessions; WebDriver providers refuse it unless they declare `tapGesture` support. On iOS the contacts touch down and lift together; on Android they touch down up to 8ms apart and lift 8ms apart in reverse order. Three or more fingers can trigger iOS system gestures such as undo, so verify app state afterwards. Use `press` for one finger.
496+
- Cancelled `gesture` and `swipe` requests return `COMMAND_FAILED` with `details.reason=request_canceled` and record no action. Repetition pauses are interruptible.
496497
- App recognizers may not report exactly the requested pan, scale, and rotation. Verify app-level state such as `pan changed yes`, and prefer separate `gesture pan`, `gesture pinch`, or `gesture rotate` commands when exact values matter. See `agent-device help gestures`.
497498
498499
### Scroll

0 commit comments

Comments
 (0)