Skip to content

Commit a3df604

Browse files
only21milclaudecodexmaria-rcks
authored
fix(web): preserve the active pointer in the flat Device view (#21430)
Co-authored-by: Claude Opus 5.5 <[email protected]> Co-authored-by: GPT-6.1 Sol <[email protected]> Co-authored-by: maria-rcks <[email protected]> Co-authored-by: maria-rcks <[email protected]>
1 parent 98a7353 commit a3df604

2 files changed

Lines changed: 233 additions & 17 deletions

File tree

Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
import { RegistryContext } from "@effect/atom-react";
2+
import { AtomRegistry } from "effect/reactivity";
3+
import "~/index.css";
4+
import { EnvironmentId } from "@t3tools/contracts";
5+
import { expect, it, vi } from "vite-plus/test";
6+
7+
import { act, render, stubGlobal } from "../../../test/render";
8+
9+
vi.mock(import("~/state/device"), async (original) => ({
10+
...(await original()),
11+
useDeviceHubAccess: () => access,
12+
}));
13+
vi.mock(import("./DevicePhoneViewport"), () => ({
14+
DevicePhoneViewport: () => <></>,
15+
}));
16+
import { DeviceStreamView } from "./DeviceStreamView";
17+
18+
const access = { httpBase: "http://test", wsBase: "ws://test", query: {}, credentials: true };
19+
function setup() {
20+
const registry = AtomRegistry.make();
21+
const touches: unknown[] = [];
22+
const image = vi.spyOn(HTMLImageElement.prototype, "src", "set").mockImplementation(() => {});
23+
// Select MJPEG without starting a WebCodecs decoder. Input uses the real stream client.
24+
const decoder = stubGlobal("VideoDecoder", undefined);
25+
Reflect.deleteProperty(globalThis, "VideoDecoder");
26+
const fetch = stubGlobal("fetch", () => Promise.resolve(new Response("prime")));
27+
// The client opens its input socket only after the MJPEG prime resolves.
28+
type InputSocket = {
29+
onmessage?: (event: { data: ArrayBuffer }) => void;
30+
};
31+
let open!: (socket: InputSocket) => void;
32+
const opened = new Promise<InputSocket>((resolve) => {
33+
open = resolve;
34+
});
35+
const socket = stubGlobal(
36+
"WebSocket",
37+
class {
38+
static OPEN = 1;
39+
readyState = 1;
40+
constructor() {
41+
open(this);
42+
}
43+
onmessage?: (event: { data: ArrayBuffer }) => void;
44+
send(message: string | Uint8Array) {
45+
if (typeof message !== "string" && message[0] === 0x03) {
46+
touches.push(JSON.parse(new TextDecoder().decode(message.subarray(1))));
47+
}
48+
}
49+
close() {}
50+
},
51+
);
52+
return {
53+
view: () => (
54+
<RegistryContext.Provider value={registry}>
55+
<div style={{ width: 400, height: 800 }}>{view()}</div>
56+
</RegistryContext.Provider>
57+
),
58+
[Symbol.dispose]() {
59+
registry.dispose();
60+
image.mockRestore();
61+
socket[Symbol.dispose]();
62+
fetch[Symbol.dispose]();
63+
decoder[Symbol.dispose]();
64+
},
65+
touches,
66+
async configure(inputUnavailable = false) {
67+
const json = new TextEncoder().encode(
68+
JSON.stringify({ width: 400, height: 800, orientation: "portrait", inputUnavailable }),
69+
);
70+
const packet = new Uint8Array(1 + json.length);
71+
packet[0] = 0x82;
72+
packet.set(json, 1);
73+
const input = await opened;
74+
input.onmessage?.({ data: Uint8Array.of(0x83).buffer });
75+
input.onmessage?.({ data: packet.buffer });
76+
},
77+
};
78+
}
79+
80+
function view() {
81+
return (
82+
<DeviceStreamView
83+
hostId="local"
84+
environmentId={EnvironmentId.make("test")}
85+
deviceId="test"
86+
platform="ios"
87+
visible
88+
allowPhoneView={false}
89+
renderControls={(controls) => (
90+
<output>
91+
{controls.handle?.inputConnected ? "Input connected" : "Input disconnected"}
92+
</output>
93+
)}
94+
/>
95+
);
96+
}
97+
98+
// router: false because the device view renders no links and never navigates.
99+
const options = { router: false } as const;
100+
101+
function flatPointer(container: HTMLElement) {
102+
const surface = container.querySelector("canvas")!.parentElement!.parentElement!;
103+
// Synthetic IDs exercise concurrent touches independently of the browser's capture state.
104+
surface.setPointerCapture = () => {};
105+
const rect = new DOMRect(0, 0, 400, 800);
106+
surface.getBoundingClientRect = () => rect;
107+
return {
108+
send(type: string, pointerId: number, x = 0.25, y = 0.25) {
109+
surface.dispatchEvent(
110+
new PointerEvent(type, {
111+
bubbles: true,
112+
pointerId,
113+
pointerType: "touch",
114+
clientX: rect.left + rect.width * x,
115+
clientY: rect.top + rect.height * y,
116+
}),
117+
);
118+
},
119+
};
120+
}
121+
122+
it("keeps the first flat pointer active when a second finger moves and lifts", async () => {
123+
using input = setup();
124+
const { configure, touches } = input;
125+
await using screen = await render(input.view(), options);
126+
await act(async () => configure());
127+
await expect.element(screen.getByText("Input connected", { exact: true })).toBeVisible();
128+
const { send } = flatPointer(screen.container);
129+
send("pointerdown", 1);
130+
send("pointerdown", 2, 0.75, 0.75);
131+
send("pointermove", 2, 0.875, 0.875);
132+
send("pointerup", 2);
133+
send("pointercancel", 2);
134+
send("pointermove", 1, 0.5, 0.5);
135+
send("pointerup", 1, 0.75, 0.75);
136+
send("pointermove", 1);
137+
expect(touches).toEqual([
138+
{ type: "begin", x: 0.25, y: 0.25 },
139+
{ type: "move", x: 0.5, y: 0.5 },
140+
{ type: "end", x: 0.75, y: 0.75 },
141+
]);
142+
});
143+
144+
it("ignores a flat pointer that goes down before input connects", async () => {
145+
using input = setup();
146+
const { configure, touches } = input;
147+
await using screen = await render(input.view(), options);
148+
const { send } = flatPointer(screen.container);
149+
send("pointerdown", 1);
150+
await act(async () => configure());
151+
await expect.element(screen.getByText("Input connected", { exact: true })).toBeVisible();
152+
send("pointerdown", 2, 0.5, 0.5);
153+
send("pointerup", 2, 0.75, 0.75);
154+
expect(touches).toEqual([
155+
{ type: "begin", x: 0.5, y: 0.5 },
156+
{ type: "end", x: 0.75, y: 0.75 },
157+
]);
158+
});
159+
160+
it.each(["pointerup", "pointercancel", "lostpointercapture"])(
161+
"ends the owned flat pointer exactly once on %s, then accepts a new pointer",
162+
async (ending) => {
163+
using input = setup();
164+
const { configure, touches } = input;
165+
await using screen = await render(input.view(), options);
166+
await act(async () => configure());
167+
await expect.element(screen.getByText("Input connected", { exact: true })).toBeVisible();
168+
const { send } = flatPointer(screen.container);
169+
send("pointerdown", 7);
170+
send("pointermove", 7, 0.5, 0.5);
171+
send("lostpointercapture", 8, 0, 0);
172+
// Capture loss reports its own point; the end must still use the last move.
173+
const at = ending === "lostpointercapture" ? 0 : 0.5;
174+
send(ending, 7, at, at);
175+
send("lostpointercapture", 7, 0, 0);
176+
send("pointerup", 7);
177+
send("pointermove", 7);
178+
send("pointerdown", 9);
179+
send("pointerup", 9);
180+
expect(touches).toEqual([
181+
{ type: "begin", x: 0.25, y: 0.25 },
182+
{ type: "move", x: 0.5, y: 0.5 },
183+
{ type: "end", x: 0.5, y: 0.5 },
184+
{ type: "begin", x: 0.25, y: 0.25 },
185+
{ type: "end", x: 0.25, y: 0.25 },
186+
]);
187+
},
188+
);
189+
190+
it("releases flat pointer ownership when input disconnects", async () => {
191+
using input = setup();
192+
await using screen = await render(input.view(), options);
193+
await act(() => input.configure());
194+
await expect.element(screen.getByText("Input connected", { exact: true })).toBeVisible();
195+
const { send } = flatPointer(screen.container);
196+
send("pointerdown", 1);
197+
await act(() => input.configure(true));
198+
await expect.element(screen.getByText("Input disconnected", { exact: true })).toBeVisible();
199+
await act(() => input.configure());
200+
await expect.element(screen.getByText("Input connected", { exact: true })).toBeVisible();
201+
send("pointerdown", 2, 0.5, 0.5);
202+
send("pointerup", 2, 0.75, 0.75);
203+
expect(input.touches).toEqual([
204+
{ type: "begin", x: 0.25, y: 0.25 },
205+
{ type: "begin", x: 0.5, y: 0.5 },
206+
{ type: "end", x: 0.75, y: 0.75 },
207+
]);
208+
});

‎apps/web/src/components/device/DeviceStreamView.tsx‎

Lines changed: 25 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import type { DevicePlatform, EnvironmentId } from "@t3tools/contracts";
22
import { useAtomValue } from "@effect/atom-react";
33
import { AsyncResult, Atom } from "effect/reactivity";
4-
import { useId, useRef, useState, type ReactNode } from "react";
4+
import { useEffect, useId, useRef, useState, type ReactNode } from "react";
55

66
import { cn } from "~/lib/utils";
77
import { Button } from "~/components/ui/button";
@@ -201,14 +201,29 @@ export function DeviceStreamView(props: {
201201
);
202202
const axElements = AsyncResult.getOrElse(axResult, () => NO_AX_ELEMENTS);
203203

204-
const pointerActive = useRef(false);
204+
const activePointer = useRef<{ id: number; x: number; y: number } | null>(null);
205+
// oxlint-disable-next-line t3code/no-unreasoned-state-sync -- clears pointer ownership when its device input connection disconnects or is replaced
206+
useEffect(() => {
207+
if (!client || !inputState.connected) return;
208+
return () => {
209+
activePointer.current = null;
210+
};
211+
}, [client, inputState.connected]);
205212
const normalizedPoint = (event: React.PointerEvent<HTMLElement>) => {
206213
const rect = event.currentTarget.getBoundingClientRect();
207214
const x = (event.clientX - rect.left) / rect.width;
208215
const y = (event.clientY - rect.top) / rect.height;
209216
return { x: Math.min(1, Math.max(0, x)), y: Math.min(1, Math.max(0, y)) };
210217
};
211218

219+
const endPointer = (event: React.PointerEvent<HTMLElement>) => {
220+
const pointer = activePointer.current;
221+
if (pointer?.id !== event.pointerId) return;
222+
activePointer.current = null;
223+
const { x, y } = event.type === "lostpointercapture" ? pointer : normalizedPoint(event);
224+
client?.sendTouch("end", x, y);
225+
};
226+
212227
const phoneUnavailableReason =
213228
isDuo && !screen?.supportsHingeAngle
214229
? "iPhone Duo 3D requires Device Hub 0.11.0 or newer"
@@ -335,30 +350,23 @@ export function DeviceStreamView(props: {
335350
className={cn("relative select-none", showPhone && "invisible pointer-events-none")}
336351
style={{ width: frame.width, height: frame.height }}
337352
onPointerDown={(event) => {
338-
if (event.button !== 0) return;
353+
if (event.button !== 0 || !inputState.connected || activePointer.current !== null)
354+
return;
339355
event.currentTarget.setPointerCapture(event.pointerId);
340356
(event.currentTarget.parentElement as HTMLElement | null)?.focus();
341-
pointerActive.current = true;
342357
const { x, y } = normalizedPoint(event);
358+
activePointer.current = { id: event.pointerId, x, y };
343359
client?.sendTouch("begin", x, y);
344360
}}
345361
onPointerMove={(event) => {
346-
if (!pointerActive.current) return;
362+
if (activePointer.current?.id !== event.pointerId) return;
347363
const { x, y } = normalizedPoint(event);
364+
activePointer.current = { id: event.pointerId, x, y };
348365
client?.sendTouch("move", x, y);
349366
}}
350-
onPointerUp={(event) => {
351-
if (!pointerActive.current) return;
352-
pointerActive.current = false;
353-
const { x, y } = normalizedPoint(event);
354-
client?.sendTouch("end", x, y);
355-
}}
356-
onPointerCancel={(event) => {
357-
if (!pointerActive.current) return;
358-
pointerActive.current = false;
359-
const { x, y } = normalizedPoint(event);
360-
client?.sendTouch("end", x, y);
361-
}}
367+
onPointerUp={endPointer}
368+
onPointerCancel={endPointer}
369+
onLostPointerCapture={endPointer}
362370
>
363371
<div
364372
ref={mountCanvas}

0 commit comments

Comments
 (0)