Skip to content

Commit 0c92a5b

Browse files
strzhaoclaude
andcommitted
feat(harmonyos): inject single-pointer human-trajectory pans through the panel evdev node
Motivation: HarmonyOS lowering dropped every interior sample of the shared gesture plan: performHarmonyGesture collapsed PointerTrajectorySample[] onto a straight uitest uiInput swipe/fling, so flows that verify trajectory shape (slider controls, drag verification) had no non-linear path on this platform. Approach: - contracts: gesture pan accepts an optional trajectory: 'human' opt-in. buildSinglePointerPlan then densifies the canonical endpoints into a deterministic human-hand stroke (slow onset, bounded overshoot, settle-back, seeded wobble, >= 20 samples) and marks the plan, so transports receive the full trajectory model instead of an endpoint pair. - platform-harmonyos: the marked plan lowers through the touchscreen evdev node as a Type-B sendevent sequence paced to the sample timeline, carried in one hdc shell round trip. The release frame reports the final point so the stroke ends on the destination. Plans without the opt-in keep the byte-identical endpoint swipe transport. - CLI: --trajectory human on gesture pan, threaded through the flag registry, CLI reader, command fields, .ad replay preflight, and integration model. Test evidence: - contracts: generator invariants (determinism, exact endpoints, monotonic offsets, slow onset, capped overshoot, viewport clamp), plan marking, and payload/positional normalization (60 focused tests). - platform-harmonyos: evdev batch lowering, transport sample cap, script rendering and pacing, node resolution with its typed refusal, dispatch wiring; the legacy uiInput assertions are unchanged. - On-device trajectory verification is tracked in the contribution ledger. Docs impact: the help gestures topic gains the HarmonyOS trajectory quirk line. Co-Authored-By: Claude Code <[email protected]>
1 parent a253440 commit 0c92a5b

19 files changed

Lines changed: 1222 additions & 11 deletions

‎packages/command-registry/src/flag-definitions-action.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,17 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [
3636
projectConfig: true,
3737
recorded: true,
3838
},
39+
{
40+
key: 'trajectory',
41+
names: ['--trajectory'],
42+
type: 'enum',
43+
enumValues: ['human'],
44+
usageLabel: '--trajectory <human>',
45+
usageDescription:
46+
'Gesture pan: lower the stroke as a sampled human-hand path (slow onset, overshoot, settle; HarmonyOS)',
47+
projectConfig: true,
48+
recorded: true,
49+
},
3950
{
4051
key: 'fps',
4152
names: ['--fps'],

‎packages/command-surface/src/interaction/gesture.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ function gestureInputFromCli(positionals: string[], flags: CliFlags): Record<str
2525
holdMs: flags.holdMs,
2626
count: flags.count,
2727
pauseMs: flags.pauseMs,
28+
trajectory: flags.trajectory,
2829
}),
2930
};
3031
}

‎packages/command-surface/src/interaction/metadata.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { CLICK_BUTTONS } from '@agent-device/contracts/click-button';
22
import {
33
GESTURE_KINDS,
4+
HUMAN_TRAJECTORY_MODES,
45
type GesturePayload,
56
readGesturePayload,
67
} from '@agent-device/contracts/gesture-input';
@@ -230,6 +231,10 @@ const gestureFields = {
230231
degrees: numberField('Rotation in degrees.'),
231232
durationMs: integerField('Pan/transform duration.', { min: 16, max: 10_000 }),
232233
pointerCount: integerField('Pan touch pointer count (1 or 2).', { min: 1, max: 2 }),
234+
trajectory: enumField(
235+
HUMAN_TRAJECTORY_MODES,
236+
'Pan stroke transport path; human samples a hand-like curve.',
237+
),
233238
points: pointArrayField('Tap contact points, held together for holdMs.', {
234239
min: GESTURE_TAP_MIN_CONTACTS,
235240
max: GESTURE_MAX_CONTACTS,

‎packages/contracts/src/cli-flags.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import type { RecordingExportQuality } from './recording-export-quality.ts';
33
import type { BackMode } from './back-mode.ts';
44
import type { ClickButton } from './click-button.ts';
55
import type { SwipePattern } from './scroll-gesture.ts';
6+
import type { HumanTrajectoryMode } from './gesture-plan-types.ts';
67
import type { BootPickPolicy, DeviceTarget, PlatformSelector } from '@agent-device/kernel/device';
78
import type { SnapshotPreferredBackend } from '@agent-device/kernel/snapshot';
89
import type {
@@ -103,6 +104,7 @@ export type CliFlags = CloudProviderProfileFields &
103104
appsFilter?: 'user-installed' | 'all';
104105
count?: number;
105106
pointerCount?: number;
107+
trajectory?: HumanTrajectoryMode;
106108
fps?: number;
107109
quality?: RecordingExportQuality | string;
108110
hideTouches?: boolean;

‎packages/contracts/src/gesture-input.test.ts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,3 +166,42 @@ test('tap-only inputs are refused on other gesture kinds', () => {
166166
});
167167
}
168168
});
169+
170+
test('pan reads the human trajectory opt-in and every other kind refuses it', () => {
171+
assert.deepEqual(
172+
readGesturePayload({
173+
kind: 'pan',
174+
origin: { x: 1, y: 2 },
175+
delta: { x: 3, y: 4 },
176+
trajectory: 'human',
177+
}),
178+
{
179+
kind: 'pan',
180+
origin: { x: 1, y: 2 },
181+
delta: { x: 3, y: 4 },
182+
pointerCount: undefined,
183+
durationMs: undefined,
184+
trajectory: 'human',
185+
},
186+
);
187+
assert.throws(
188+
() =>
189+
readGesturePayload({
190+
kind: 'pan',
191+
origin: { x: 1, y: 2 },
192+
delta: { x: 3, y: 4 },
193+
trajectory: 'fast',
194+
}),
195+
/one of: human/,
196+
);
197+
assert.throws(
198+
() =>
199+
readGesturePayload({
200+
kind: 'fling',
201+
direction: 'up',
202+
origin: { x: 1, y: 2 },
203+
trajectory: 'human',
204+
}),
205+
/trajectory is supported only for gesture pan/,
206+
);
207+
});

‎packages/contracts/src/gesture-input.ts‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import {
2020
GESTURE_MAX_CONTACTS,
2121
GESTURE_TAP_MIN_CONTACTS,
2222
type GesturePointerCount,
23+
type HumanTrajectoryMode,
2324
} from './gesture-plan-types.ts';
2425

2526
export const COORDINATE_GESTURE_KINDS = [
@@ -33,6 +34,9 @@ export const COORDINATE_GESTURE_KINDS = [
3334
] as const;
3435
export const GESTURE_KINDS = [...COORDINATE_GESTURE_KINDS, 'drag'] as const;
3536

37+
/** The transport paths `gesture pan` can request for its stroke. */
38+
export const HUMAN_TRAJECTORY_MODES = ['human'] as const;
39+
3640
export type DragGesturePayload = {
3741
kind: 'drag';
3842
source: string;
@@ -48,6 +52,8 @@ export type PanGesturePayload = {
4852
delta: Point;
4953
pointerCount?: GesturePointerCount;
5054
durationMs?: number;
55+
/** Lowers the pan as a sampled human-hand stroke instead of one straight transport segment. */
56+
trajectory?: HumanTrajectoryMode;
5157
};
5258

5359
export type FlingGesturePayload = {
@@ -125,6 +131,7 @@ export function readGesturePayload(input: unknown): GesturePayload {
125131
| GesturePointerCount
126132
| undefined,
127133
durationMs: readOptionalGestureDuration(record),
134+
trajectory: readOptionalPanTrajectory(record),
128135
};
129136
}
130137
if (kind === 'drag') {
@@ -133,6 +140,7 @@ export function readGesturePayload(input: unknown): GesturePayload {
133140
if (record.pointerCount !== undefined) {
134141
throw new AppError('INVALID_ARGS', 'pointerCount is supported only for gesture pan');
135142
}
143+
assertPanOnlyTrajectory(kind, record);
136144
if (kind === 'fling') {
137145
if (record.durationMs !== undefined) {
138146
throw new AppError(
@@ -308,6 +316,21 @@ function readOptionalGestureDuration(record: Record<string, unknown>): number |
308316
return durationMs;
309317
}
310318

319+
function readOptionalPanTrajectory(
320+
record: Record<string, unknown>,
321+
): HumanTrajectoryMode | undefined {
322+
return record.trajectory === undefined
323+
? undefined
324+
: readEnum(record, 'trajectory', HUMAN_TRAJECTORY_MODES);
325+
}
326+
327+
function assertPanOnlyTrajectory(kind: string, record: Record<string, unknown>): void {
328+
if (kind === 'pan' || record.trajectory === undefined) return;
329+
throw new AppError('INVALID_ARGS', 'trajectory is supported only for gesture pan', {
330+
hint: 'A sampled human-hand path is a timed single-pointer motion.',
331+
});
332+
}
333+
311334
function readRecord(input: unknown): Record<string, unknown> {
312335
if (!input || typeof input !== 'object' || Array.isArray(input)) {
313336
throw new AppError('INVALID_ARGS', 'gesture requires structured object input');

‎packages/contracts/src/gesture-normalization.test.ts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import assert from 'node:assert/strict';
22
import { test } from 'vitest';
33
import {
44
describeReplayGestureArityError,
5+
describeReplayGestureFlagError,
56
gesturePayloadFromPositionals,
67
gesturePayloadToPositionals,
78
normalizeGestureCommandInput,
@@ -19,6 +20,7 @@ test('CLI and .ad positionals normalize at one explicit syntax seam', () => {
1920
delta: { x: 30, y: -40 },
2021
durationMs: 500,
2122
pointerCount: 2,
23+
trajectory: undefined,
2224
},
2325
);
2426
});
@@ -334,3 +336,30 @@ test('an .ad tap line with an odd coordinate count reports its source', () => {
334336
'gesture tap accepts 4 to 10 arguments: x1 y1 x2 y2 [x3 y3 [x4 y4 [x5 y5]]] (line 4).',
335337
);
336338
});
339+
340+
test('the pan trajectory rides the flag channel: parse, preflight, and normalization agree', () => {
341+
const payload = gesturePayloadFromPositionals(['pan', '10', '20', '5', '6', '300'], {
342+
trajectory: 'human',
343+
});
344+
assert.equal(payload.kind === 'pan' ? payload.trajectory : undefined, 'human');
345+
assert.equal(
346+
describeReplayGestureFlagError(
347+
'gesture',
348+
['fling', 'up', '1', '2'],
349+
{ trajectory: 'human' },
350+
'line 3',
351+
),
352+
'gesture fling does not read --trajectory (line 3).',
353+
);
354+
assert.equal(
355+
describeReplayGestureFlagError(
356+
'gesture',
357+
['pan', '10', '20', '5', '6'],
358+
{ trajectory: 'human' },
359+
'line 4',
360+
),
361+
undefined,
362+
);
363+
const normalized = normalizeGestureCommandInput(payload);
364+
assert.equal(normalized.intent === 'pan' ? normalized.trajectory : undefined, 'human');
365+
});

‎packages/contracts/src/gesture-normalization.ts‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,11 @@
11
import type { Point } from '@agent-device/kernel/snapshot';
22
import { AppError } from '@agent-device/kernel/errors';
33
import { readGesturePayload, type GESTURE_KINDS, type GesturePayload } from './gesture-input.ts';
4-
import type { GestureCommandInput, GestureSemanticInput } from './gesture-plan-types.ts';
4+
import type {
5+
GestureCommandInput,
6+
GestureSemanticInput,
7+
HumanTrajectoryMode,
8+
} from './gesture-plan-types.ts';
59
import {
610
DEFAULT_DRAG_DESTINATION_HOLD_MS,
711
DEFAULT_DRAG_MOVE_MS,
@@ -20,13 +24,14 @@ export type SwipePayload = {
2024
pattern?: 'one-way' | 'ping-pong';
2125
};
2226

23-
type GestureFlagKey = 'pointerCount' | 'holdMs' | 'count' | 'pauseMs';
27+
type GestureFlagKey = 'pointerCount' | 'holdMs' | 'count' | 'pauseMs' | 'trajectory';
2428

2529
const GESTURE_FLAG_NAMES: Record<GestureFlagKey, string> = {
2630
pointerCount: '--pointer-count',
2731
holdMs: '--hold-ms',
2832
count: '--count',
2933
pauseMs: '--pause-ms',
34+
trajectory: '--trajectory',
3035
};
3136

3237
/**
@@ -35,7 +40,7 @@ const GESTURE_FLAG_NAMES: Record<GestureFlagKey, string> = {
3540
* CLI would reject and have it silently dropped.
3641
*/
3742
export const GESTURE_FLAGS_BY_KIND = {
38-
pan: ['pointerCount'],
43+
pan: ['pointerCount', 'trajectory'],
3944
tap: ['holdMs', 'count', 'pauseMs'],
4045
fling: [],
4146
swipe: [],
@@ -269,6 +274,7 @@ export type GesturePositionalFlags = {
269274
holdMs?: number;
270275
count?: number;
271276
pauseMs?: number;
277+
trajectory?: HumanTrajectoryMode;
272278
};
273279

274280
/** The explicit parser for the public CLI and `.ad` gesture syntax. */
@@ -290,6 +296,7 @@ export function gesturePayloadFromPositionals(
290296
delta,
291297
pointerCount,
292298
durationMs: optionalPositionNumber(args[4]),
299+
trajectory: flags.trajectory,
293300
});
294301
}
295302
case 'fling': {
@@ -447,6 +454,7 @@ export function normalizePublicGesture(input: GesturePayload): NormalizedPublicG
447454
delta: input.delta,
448455
pointerCount: input.pointerCount,
449456
durationMs: input.durationMs,
457+
trajectory: input.trajectory,
450458
},
451459
};
452460
case 'fling':

0 commit comments

Comments
 (0)