Skip to content

Commit e844582

Browse files
author
Cameron Chisholm
committed
feat(apple): admit two-finger gestures on physical iOS devices
1 parent 507ef23 commit e844582

16 files changed

Lines changed: 87 additions & 92 deletions

‎docs/adr/0013-unified-gesture-plans.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ Platform adapters consume the canonical plan:
142142
macOS lowers a one-contact plan to its drag executor.
143143
tvOS lowers a one-contact plan to remote direction.
144144
Core admission and the Apple adapter both consume the same shared multi-touch support policy.
145-
Multi-touch remains capability-gated to iOS simulators.
145+
Multi-touch is capability-gated to iOS and iPadOS, on simulators and physical devices, through the XCTest runner.
146146

147147
iOS deliberately retains three one-contact motion schedules behind the shared private-XCTest event bridge.
148148
Apple's public drag API models the phases independently as an [initial hold, movement velocity, and destination hold](https://developer.apple.com/documentation/xcuiautomation/xcuicoordinate/press%28forduration%3Athendragto%3Awithvelocity%3Athenholdforduration%3A%29).

‎packages/contracts/src/gesture-admission.ts‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -29,16 +29,12 @@ export const TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT =
2929

3030
/** The hint the Android owner states for a TV target, which has no touch input at all. */
3131
export const ANDROID_TV_MULTI_TOUCH_UNSUPPORTED_HINT =
32-
'Android TV has no touch input — this gesture is supported on Android phones, tablets, and the iOS simulator only.';
33-
34-
/** The hint the Apple owner states for a physical iOS/iPadOS device. */
35-
export const PHYSICAL_IOS_MULTI_TOUCH_UNSUPPORTED_HINT =
36-
'Two-finger gesture synthesis is iOS-simulator only — not available on physical iOS devices.';
32+
'Android TV has no touch input — this gesture is supported on Android phones, tablets, iOS, and iPadOS.';
3733

3834
/**
3935
* How a refused cell names itself, reproducing every subject the retired admission produced.
4036
*
41-
* Four owner-specific subjects, then the plain platform name. The special cases resolve the
37+
* Three owner-specific subjects, then the plain platform name. The special cases resolve the
4238
* Apple OS the way `assertAppleMultiTouchSupported` does; the default reads `appleOs` raw, the way
4339
* the retired `gesturePlatformMessage` did — an Apple device with no declared OS therefore still
4440
* reports `apple`, exactly as before.
@@ -54,14 +50,14 @@ function gestureRefusalSubject(device: DeviceInfo, tier: GestureRuntimeTier): st
5450
}
5551

5652
/**
57-
* The three owner-specific subjects two-contact synthesis produced, or `undefined` where the
58-
* retired admission fell through to the plain platform name.
53+
* The owner-specific subjects two-contact synthesis refusals use, or `undefined` for the plain
54+
* platform name. iOS and iPadOS have none: their owner admits multi-touch on simulators and
55+
* devices, so only a provider can refuse it there, and its hint says why.
5956
*/
6057
function multiTouchRefusalSubject(device: DeviceInfo): string | undefined {
6158
if (device.platform === 'android') return device.target === 'tv' ? 'Android TV' : undefined;
6259
if (!isApplePlatform(device.platform)) return undefined;
6360
const appleOs = resolveDeviceAppleOs(device);
64-
if (appleOs === 'ios' || appleOs === 'ipados') return 'physical iOS devices';
6561
if (appleOs === 'macos' || appleOs === 'tvos' || appleOs === 'visionos') {
6662
return APPLE_OS_DISPLAY_NAMES[appleOs];
6763
}

‎packages/contracts/src/gesture-runtime.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ export type GestureViewportInput = Readonly<{
4646
* - `performDirectionalFlingPlan` — `gesture fling --direction`, whose speed semantics Linux
4747
* cannot honor even though it executes coordinate flings through its drag primitive.
4848
* - `performMultiTouchGesturePlan` — pinch/rotate/transform and two-pointer pan. On Apple this is
49-
* the two-finger XCTest synthesis, which is iOS/iPadOS **simulator** only.
49+
* the two-finger XCTest synthesis, which runs on iOS/iPadOS simulators and physical devices.
5050
* - `performTargetAuthoredDrag` — `gesture drag`, which needs an adapter preserving source hold,
5151
* timed movement, and destination hold.
5252
*/

‎packages/daemon/src/core/gesture-runtime-admission.test.ts‎

Lines changed: 24 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -20,26 +20,10 @@ const refusalCases = [
2020
fact: {
2121
available: false,
2222
reason: 'unsupported-platform-leaf',
23-
hint: 'Android TV has no touch input — this gesture is supported on Android phones, tablets, and the iOS simulator only.',
23+
hint: 'Android TV has no touch input — this gesture is supported on Android phones, tablets, iOS, and iPadOS.',
2424
},
2525
hint: /Android TV has no touch input/,
26-
},
27-
{
28-
name: 'physical iOS',
29-
device: {
30-
platform: 'apple',
31-
appleOs: 'ios',
32-
id: 'iphone-device',
33-
name: 'iPhone',
34-
kind: 'device',
35-
target: 'mobile',
36-
},
37-
fact: {
38-
available: false,
39-
reason: 'unsupported-device-kind',
40-
hint: 'Two-finger gesture synthesis is iOS-simulator only — not available on physical iOS devices.',
41-
},
42-
hint: /physical iOS devices/,
26+
message: 'gesture pinch is not supported on Android TV',
4327
},
4428
{
4529
name: 'tvOS',
@@ -57,12 +41,32 @@ const refusalCases = [
5741
hint: 'tvOS does not support multi-touch gesture plans.',
5842
},
5943
hint: /tvOS/,
44+
message: 'gesture pinch is not supported on tvOS',
45+
},
46+
{
47+
// The Apple owner admits physical iOS; a provider without the XCTest runner can still refuse.
48+
name: 'provider-refused physical iOS',
49+
device: {
50+
platform: 'apple',
51+
appleOs: 'ios',
52+
id: 'ios-device',
53+
name: 'iPhone',
54+
kind: 'device',
55+
target: 'mobile',
56+
},
57+
fact: {
58+
available: false,
59+
reason: 'unsupported-provider-mode',
60+
hint: 'Two-finger gesture synthesis is not available on iOS devices reached through a WebDriver provider.',
61+
},
62+
hint: /WebDriver provider/,
63+
message: 'gesture pinch is not supported on ios',
6064
},
6165
] as const;
6266

6367
test.each(refusalCases)(
6468
'$name preserves multi-touch admission details without binding',
65-
async ({ device, fact, hint }) => {
69+
async ({ device, fact, hint, message }) => {
6670
const target = device as DeviceInfo;
6771
const owner = localRuntimeOwner(target.platform);
6872
const base = createUnavailableRuntimeFactsForTest(target, owner);
@@ -89,6 +93,7 @@ test.each(refusalCases)(
8993
expect(resolved.ok).toBe(false);
9094
if (!resolved.ok) {
9195
expect(resolved.response.error.code).toBe('UNSUPPORTED_OPERATION');
96+
expect(resolved.response.error.message).toBe(message);
9297
expect(resolved.response.error.hint).toMatch(hint);
9398
}
9499
expect(inspectFacts).toHaveBeenCalledOnce();

‎packages/platform-android/src/runtime.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -483,7 +483,7 @@ test('carries the retired Android TV hints verbatim', async () => {
483483
expect(facts.operations.performMultiTouchGesturePlan).toEqual({
484484
available: false,
485485
reason: 'unsupported-platform-leaf',
486-
hint: 'Android TV has no touch input — this gesture is supported on Android phones, tablets, and the iOS simulator only.',
486+
hint: 'Android TV has no touch input — this gesture is supported on Android phones, tablets, iOS, and iPadOS.',
487487
});
488488
expect(facts.operations.performTargetAuthoredDrag).toMatchObject({
489489
available: false,

‎packages/platform-apple/src/gesture-facts.test.ts‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ test.each([
3939
appleDevice({ kind: 'device', iosPhysicalDeviceBackend: 'coredevice' }),
4040
true,
4141
true,
42-
false,
42+
true,
4343
true,
4444
true,
4545
true,
@@ -64,14 +64,8 @@ test.each([
6464
},
6565
);
6666

67-
test('carries the retired multi-touch hints verbatim on every Apple leaf that refused', async () => {
67+
test('carries the retired multi-touch hints on every Apple leaf that refused', async () => {
6868
const runtime = createApplePlatformRuntime(platformRuntimeHostFixture());
69-
const physical = await runtime.inspectFacts(appleDevice({ kind: 'device' }));
70-
expect(physical.operations.performMultiTouchGesturePlan).toEqual({
71-
available: false,
72-
reason: 'unsupported-device-kind',
73-
hint: 'Two-finger gesture synthesis is iOS-simulator only — not available on physical iOS devices.',
74-
});
7569
const macos = await runtime.inspectFacts(leaves.macos);
7670
expect(macos.operations.performMultiTouchGesturePlan).toMatchObject({
7771
available: false,
@@ -107,5 +101,8 @@ test('binds only the gesture tiers the leaf admitted', async () => {
107101
expect(simulator.operations.scrollDirection).toBeTypeOf('function');
108102
const physical = await bind(appleDevice({ kind: 'device' }));
109103
expect(physical.operations.performGesturePlan).toBeTypeOf('function');
110-
expect(physical.operations.performMultiTouchGesturePlan).toBeUndefined();
104+
expect(physical.operations.performMultiTouchGesturePlan).toBeTypeOf('function');
105+
const macos = await bind(leaves.macos);
106+
expect(macos.operations.performGesturePlan).toBeTypeOf('function');
107+
expect(macos.operations.performMultiTouchGesturePlan).toBeUndefined();
111108
});

‎packages/platform-apple/src/gesture-facts.ts‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,5 @@
11
import { APPLE_MULTI_TOUCH_UNSUPPORTED_HINTS } from './multitouch-support.ts';
2-
import {
3-
PHYSICAL_IOS_MULTI_TOUCH_UNSUPPORTED_HINT,
4-
TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT,
5-
} from '@agent-device/contracts/gesture-admission';
2+
import { TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT } from '@agent-device/contracts/gesture-admission';
63
import { gestureRuntimeOperationFacts } from '@agent-device/contracts/gesture-runtime';
74
import type {
85
RuntimeOperationFact,
@@ -19,11 +16,6 @@ const gestureLeafUnavailable = Object.freeze({
1916
const gestureKindUnavailable = unsupportedAppleDeviceKind(
2017
'Gestures are supported only for Apple simulators and devices.',
2118
);
22-
const physicalIosMultiTouchUnavailable = Object.freeze({
23-
available: false,
24-
reason: 'unsupported-device-kind',
25-
hint: PHYSICAL_IOS_MULTI_TOUCH_UNSUPPORTED_HINT,
26-
} as const);
2719
const targetAuthoredDragUnavailable = Object.freeze({
2820
available: false,
2921
reason: 'unsupported-platform-leaf',
@@ -76,7 +68,7 @@ function appleMultiTouchGestureFact(device: DeviceInfo): RuntimeOperationFact {
7668
...(hint === undefined ? {} : { hint }),
7769
} as const);
7870
}
79-
return device.kind === 'simulator' ? available : physicalIosMultiTouchUnavailable;
71+
return available;
8072
}
8173

8274
function appleTargetAuthoredDragFact(device: DeviceInfo): RuntimeOperationFact {

‎packages/platform-apple/src/interactions.test.ts‎

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,8 @@ import {
99
} from './interactions.ts';
1010
import { runAppleRunnerCommand } from './core/runner-client.ts';
1111
import { AppError } from '@agent-device/kernel/errors';
12-
import {
13-
gestureRefusalMessage,
14-
PHYSICAL_IOS_MULTI_TOUCH_UNSUPPORTED_HINT,
15-
} from '@agent-device/contracts/gesture-admission';
12+
import { gestureRefusalMessage } from '@agent-device/contracts/gesture-admission';
13+
import { APPLE_MULTI_TOUCH_UNSUPPORTED_HINTS } from './multitouch-support.ts';
1614
import type { GesturePlan } from '@agent-device/contracts/gesture-plan-types';
1715
import type { RunnerCommand } from './runner/index.ts';
1816
import { TEXT_ENTRY_ROUTES } from '@agent-device/contracts/interactor-types';
@@ -415,19 +413,38 @@ test('performGestureApple sends exact two-pointer pan samples through gesture',
415413
});
416414
});
417415

416+
test('performGestureApple sends two-pointer samples to a physical iOS device', async () => {
417+
mockRunAppleRunnerCommand.mockResolvedValue({ transformed: true });
418+
const plan = twoFingerPanPlan();
419+
420+
const result = await performGestureApple(
421+
IOS_TEST_DEVICE,
422+
{ appBundleId: 'com.example.App' },
423+
{},
424+
plan,
425+
);
426+
427+
assert.deepEqual(result, { transformed: true });
428+
assert.deepEqual(mockRunAppleRunnerCommand.mock.calls[0]?.[1], {
429+
command: 'gesture',
430+
gesturePlan: plan,
431+
appBundleId: 'com.example.App',
432+
});
433+
});
434+
418435
test('Apple admission and execution share the same multi-touch refusal', async () => {
419436
// Admission is now the Apple owner's `performMultiTouchGesturePlan` fact composed with the
420437
// shared refusal wording (R42); execution keeps its defensive adapter check. The two must
421438
// still say the same thing, which is what this pins.
422-
const admissionMessage = gestureRefusalMessage(IOS_TEST_DEVICE, 'multi-touch', 'pan');
439+
const admissionMessage = gestureRefusalMessage(MACOS_TEST_DEVICE, 'multi-touch', 'pan');
423440

424441
await assert.rejects(
425-
() => performGestureApple(IOS_TEST_DEVICE, {}, {}, twoFingerPanPlan()),
442+
() => performGestureApple(MACOS_TEST_DEVICE, {}, {}, twoFingerPanPlan()),
426443
(error: unknown) =>
427444
error instanceof AppError &&
428445
error.code === 'UNSUPPORTED_OPERATION' &&
429446
error.message === admissionMessage &&
430-
error.details?.hint === PHYSICAL_IOS_MULTI_TOUCH_UNSUPPORTED_HINT,
447+
error.details?.hint === APPLE_MULTI_TOUCH_UNSUPPORTED_HINTS.macos,
431448
);
432449
assert.equal(mockRunAppleRunnerCommand.mock.calls.length, 0);
433450
});

‎packages/platform-apple/src/multitouch-support.ts‎

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,9 @@ import type { GesturePlan } from '@agent-device/contracts/gesture-plan-types';
1010

1111
export const APPLE_MULTI_TOUCH_UNSUPPORTED_HINTS: Partial<Record<AppleOS, string>> = {
1212
visionos: 'visionOS uses spatial input and does not support two-finger touch synthesis.',
13-
tvos: 'tvOS has no touch input — this gesture is supported on Android and the iOS simulator only.',
13+
tvos: 'tvOS has no touch input — this gesture is supported on Android, iOS, and iPadOS.',
1414
macos:
15-
'macOS automation has no multi-touch input — this gesture is supported on Android and the iOS simulator only.',
15+
'macOS automation has no multi-touch input — this gesture is supported on Android, iOS, and iPadOS.',
1616
};
1717

1818
/** One policy shared by command admission and the defensive Apple adapter check. */
@@ -24,16 +24,7 @@ export function assertAppleMultiTouchSupported(
2424
throw unsupported(intent, device.platform);
2525
}
2626
const appleOs = resolveDeviceAppleOs(device);
27-
if ((appleOs === 'ios' || appleOs === 'ipados') && device.kind === 'simulator') return;
28-
if (appleOs === 'ios' || appleOs === 'ipados') {
29-
throw new AppError(
30-
'UNSUPPORTED_OPERATION',
31-
`gesture ${intent} is not supported on physical iOS devices`,
32-
{
33-
hint: 'Two-finger gesture synthesis is iOS-simulator only — not available on physical iOS devices.',
34-
},
35-
);
36-
}
27+
if (appleOs === 'ios' || appleOs === 'ipados') return;
3728
throw unsupported(
3829
intent,
3930
APPLE_OS_DISPLAY_NAMES[appleOs],

‎packages/provider-webdriver/src/platform-runtime.test.ts‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -513,8 +513,8 @@ function host(run: PlatformRuntimeHost['commands']['run']): PlatformRuntimeHost
513513
}
514514

515515
// R52/R53: gestures and scrolling ride the same reachability gate the captures do. The one extra
516-
// gate is the retired multi-touch policy — this provider owns physical devices only, and
517-
// two-finger synthesis on a physical iOS device was refused before this migration too.
516+
// gate is multi-touch: this provider reaches iOS through its own WebDriver interactor, which has no
517+
// two-finger synthesis.
518518
test.each([
519519
['Android physical', device, true],
520520
['iOS physical', { ...device, platform: 'apple' as const, appleOs: 'ios' as const }, false],
@@ -534,8 +534,10 @@ test.each([
534534
expect(facts.operations.scrollDirection).toEqual({ available: true });
535535
expect(facts.operations.performMultiTouchGesturePlan.available).toBe(multiTouch);
536536
if (!multiTouch) {
537-
expect(facts.operations.performMultiTouchGesturePlan).toMatchObject({
538-
hint: 'Two-finger gesture synthesis is iOS-simulator only — not available on physical iOS devices.',
537+
expect(facts.operations.performMultiTouchGesturePlan).toEqual({
538+
available: false,
539+
reason: 'unsupported-provider-mode',
540+
hint: 'Two-finger gesture synthesis is not available on iOS devices reached through a WebDriver provider.',
539541
});
540542
}
541543
});

0 commit comments

Comments
 (0)