Skip to content

Commit 35dcdd6

Browse files
shivamhwpPR Batch Testerjztmanylclaude
authored
fix(server): phone taps, typing and scrolling reach desktop browser tabs again (pingdotgg#19456)
Co-authored-by: PR Batch Tester <[email protected]> Co-authored-by: Max Nylander <[email protected]> Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent a077fdd commit 35dcdd6

2 files changed

Lines changed: 293 additions & 25 deletions

File tree

‎apps/server/src/preview/ServerBrowser.test.ts‎

Lines changed: 204 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import * as Path from "effect/Path";
2424
import * as PubSub from "effect/PubSub";
2525
import * as Queue from "effect/Queue";
2626
import * as Stream from "effect/Stream";
27+
import * as TestClock from "effect/testing/TestClock";
2728
import type { BrowserContext, Page } from "playwright-core";
2829
import { beforeEach, expect, vi } from "vite-plus/test";
2930

@@ -56,6 +57,7 @@ vi.mock("./ServerBrowserContexts.ts", () => ({
5657
}
5758
async connectDesktopPage(endpoint: string) {
5859
const context = makeContext();
60+
context.page.viewportSize.mockReturnValue(null);
5961
desktopConnections.push({ endpoint, context });
6062
return { browser: { close: async () => {} }, page: context.page as unknown as Page };
6163
}
@@ -83,11 +85,16 @@ const PNG_HEADER = (() => {
8385
return header.toString("base64");
8486
})();
8587

86-
function makeSession(frame?: ReturnType<typeof makeFrame>) {
88+
function makeSession(
89+
frame: ReturnType<typeof makeFrame> | undefined,
90+
viewport: { width: number; height: number },
91+
) {
8792
const events = new NodeEvents.EventEmitter();
8893
return {
89-
on: vi.fn((name: string, listener: () => void) => events.on(name, listener)),
90-
emit: (name: string) => events.emit(name),
94+
on: vi.fn((name: string, callback: (...args: unknown[]) => void) => events.on(name, callback)),
95+
emit: (name: string, ...args: unknown[]) => events.emit(name, ...args),
96+
emitAsync: (name: string, ...args: unknown[]) =>
97+
Promise.all(events.listeners(name).map((listener) => listener(...args))),
9198
detach: vi.fn(async () => {}),
9299
send: vi.fn(async (method: string, input?: unknown): Promise<Record<string, unknown>> => {
93100
if (method === "Page.getNavigationHistory") return { currentIndex: 0, entries: [{}] };
@@ -101,7 +108,12 @@ function makeSession(frame?: ReturnType<typeof makeFrame>) {
101108
if (method === "Page.getFrameTree") return { frameTree: { frame: { id: "main" } } };
102109
if (method === "Page.createIsolatedWorld") {
103110
return {
104-
executionContextId: (input as { worldName?: string }).worldName === "t3-preview" ? 1 : 7,
111+
executionContextId:
112+
(input as { worldName?: string }).worldName === "t3-preview-viewport"
113+
? 9
114+
: (input as { worldName?: string }).worldName === "t3-preview"
115+
? 1
116+
: 7,
105117
};
106118
}
107119
if (method === "Runtime.evaluate" && (input as { contextId?: number }).contextId === 1) {
@@ -118,6 +130,8 @@ function makeSession(frame?: ReturnType<typeof makeFrame>) {
118130
else pageCopy = "";
119131
return { result: { value } };
120132
}
133+
if (method === "Runtime.evaluate" && (input as { contextId?: number }).contextId === 9)
134+
return desktopSizeAnswer?.() ?? { result: { value: { ...viewport } } };
121135
return { result: { value: "evaluated" } };
122136
}),
123137
};
@@ -129,7 +143,8 @@ function makeContext(onClose?: (context: BrowserContext) => void) {
129143
const setChooserFiles = vi.fn(async (_files: unknown) => {});
130144
const sessions: ReturnType<typeof makeSession>[] = [];
131145
let url = "about:blank";
132-
let viewport = { width: 1280, height: 800 };
146+
let viewport: ReturnType<Page["viewportSize"]> = { width: 1280, height: 800 };
147+
const cssViewport = { width: 1920, height: 1080 };
133148
let closed = false;
134149
let contextClosed = false;
135150
const page = {
@@ -159,7 +174,7 @@ function makeContext(onClose?: (context: BrowserContext) => void) {
159174
frames: () => [page],
160175
url: () => url,
161176
title: vi.fn(async () => "test page"),
162-
viewportSize: () => viewport,
177+
viewportSize: vi.fn(() => viewport),
163178
setViewportSize: vi.fn(async (size: typeof viewport) => {
164179
viewport = size;
165180
}),
@@ -197,11 +212,13 @@ function makeContext(onClose?: (context: BrowserContext) => void) {
197212
const context = {
198213
page,
199214
sessions,
215+
cssViewport,
200216
newPage: async () => page as unknown as Page,
201217
grantPermissions: vi.fn(async () => {}),
202218
newCDPSession: async (target: unknown) => {
203219
const session = makeSession(
204220
target === page ? undefined : (target as ReturnType<typeof makeFrame>),
221+
cssViewport,
205222
);
206223
sessions.push(session);
207224
return session;
@@ -268,7 +285,11 @@ let focusedFrame: ReturnType<typeof makeFrame> | null = null;
268285
let contextFailure: Error | null = null;
269286
let initialNavigation: (() => Promise<void>) | null = null;
270287
let desktopRendersNext = false;
288+
/** Replaces the answer to the desktop page's size reads. */
289+
let desktopSizeAnswer: (() => Promise<Record<string, unknown>>) | null = null;
271290
const desktopDetaches = new NodeEvents.EventEmitter();
291+
let desktopDetachReady = Promise.withResolvers<void>();
292+
let desktopReleased = Promise.withResolvers<void>();
272293
const desktopTabs = new Set<string>();
273294
const desktopRenders = (tabId: string) => {
274295
if (desktopRendersNext) {
@@ -317,6 +338,7 @@ const dependencies = Layer.mergeAll(
317338
const onDetach = (key: { threadId: string; tabId: string }) =>
318339
Queue.offerUnsafe(queue, key);
319340
desktopDetaches.on("detach", onDetach);
341+
desktopDetachReady.resolve();
320342
return onDetach;
321343
}),
322344
(onDetach) => Effect.sync(() => desktopDetaches.off("detach", onDetach)),
@@ -336,7 +358,10 @@ const dependencies = Layer.mergeAll(
336358
isAttached: (key) => Effect.sync(() => desktopRenders(key.tabId)),
337359
endpoint: (key) =>
338360
Effect.acquireRelease(Effect.succeed(`ws://desktop/${key.tabId}`), () =>
339-
Effect.sync(() => releasedDesktopTabs.push(key.tabId)),
361+
Effect.sync(() => {
362+
releasedDesktopTabs.push(key.tabId);
363+
desktopReleased.resolve();
364+
}),
340365
),
341366
pointer: () => Effect.void,
342367
turn: (threadId, active) => Effect.sync(() => desktopTurns.emit("turn", threadId, active)),
@@ -367,6 +392,33 @@ const viewerInput = (tabId: string, canOperate: boolean) => ({
367392
maxHeight: 800,
368393
quality: 70,
369394
});
395+
/** Fills a viewer's output with frames; resolves once every one of them is acknowledged. */
396+
const stallViewer = (viewer: ServerBrowser.ServerBrowserViewer) => {
397+
const output = viewer.output as unknown as Queue.Queue<ServerBrowser.ServerBrowserViewerOutput>;
398+
const acked = Promise.withResolvers<void>();
399+
let unacked = 0;
400+
const ack = Effect.sync(() => {
401+
if (--unacked === 0) acked.resolve();
402+
});
403+
while (Queue.offerUnsafe(output, { _tag: "frame", data: Buffer.alloc(0), ack })) unacked++;
404+
return Effect.promise(() => acked.promise);
405+
};
406+
const viewportSizes = (items: ReadonlyArray<ServerBrowser.ServerBrowserViewerOutput>) =>
407+
items.flatMap((item) => (item._tag === "viewport" ? [`${item.width}x${item.height}`] : []));
408+
const openDesktopViewer = Effect.gen(function* () {
409+
const browser = yield* ServerBrowser.ServerBrowser;
410+
const manager = yield* Manager.PreviewManager;
411+
desktopRendersNext = true;
412+
const { tabId } = yield* manager.open({ threadId: scope.thread.threadId, runtime: "server" });
413+
const viewer = yield* browser.attachViewer(viewerInput(tabId, true));
414+
const context = desktopConnections[0]!.context;
415+
const cdp = context.sessions[0]!;
416+
const resize = (width: number, height: number) => {
417+
Object.assign(context.cssViewport, { width, height });
418+
return Effect.promise(() => cdp.emitAsync("Page.frameResized"));
419+
};
420+
return { tabId, viewer, context, cdp, resize };
421+
});
370422

371423
beforeEach(() => {
372424
contexts.length = 0;
@@ -380,6 +432,9 @@ beforeEach(() => {
380432
focusedFrame = null;
381433
desktopTabs.clear();
382434
desktopRendersNext = false;
435+
desktopSizeAnswer = null;
436+
desktopDetachReady = Promise.withResolvers<void>();
437+
desktopReleased = Promise.withResolvers<void>();
383438
releasedDesktopTabs.length = 0;
384439
desktopConnections.length = 0;
385440
});
@@ -1144,6 +1199,148 @@ it.live("only the current controller can answer a file picker", () =>
11441199
).pipe(Effect.provide(layer)),
11451200
);
11461201

1202+
it.effect(
1203+
"desktop viewers get the page's CSS size and keep the last one a read cannot measure",
1204+
() =>
1205+
Effect.scoped(
1206+
Effect.gen(function* () {
1207+
// Each page size read waits here for its answer.
1208+
const reads = yield* Queue.unbounded<PromiseWithResolvers<Record<string, unknown>>>();
1209+
const deferReads = () => {
1210+
desktopSizeAnswer = () => {
1211+
const read = Promise.withResolvers<Record<string, unknown>>();
1212+
Queue.offerUnsafe(reads, read);
1213+
return read.promise;
1214+
};
1215+
};
1216+
deferReads();
1217+
const { viewer, cdp, context } = yield* openDesktopViewer;
1218+
const answer = (width: number, height: number) => ({
1219+
result: { value: { width, height } },
1220+
});
1221+
const resized = () => cdp.emitAsync("Page.frameResized");
1222+
// The read on attach never answers and gives up with no size to send.
1223+
const blocked = yield* Queue.take(reads);
1224+
yield* TestClock.adjust("1 second");
1225+
// The next page load measures again.
1226+
desktopSizeAnswer = null;
1227+
yield* Effect.promise(() => cdp.emitAsync("Page.loadEventFired"));
1228+
expect(viewportSizes(yield* Queue.takeAll(viewer.output))).toEqual(["1920x1080"]);
1229+
// While a dialog is open nothing reads. The host closing it clears it, and since
1230+
// closing resizes nothing, the close itself remeasures.
1231+
deferReads();
1232+
context.page.emit("dialog", {
1233+
type: () => "alert",
1234+
message: () => "",
1235+
defaultValue: () => "",
1236+
});
1237+
yield* Effect.promise(resized);
1238+
expect(yield* Queue.size(reads)).toBe(0);
1239+
desktopSizeAnswer = null;
1240+
yield* Effect.promise(() => cdp.emitAsync("Page.javascriptDialogClosed"));
1241+
expect(viewportSizes(yield* Queue.takeAll(viewer.output))).toEqual([
1242+
"1920x1080",
1243+
"1920x1080",
1244+
]);
1245+
// The late answer, a read a navigation breaks and a hidden webview keep the last size.
1246+
blocked.resolve(answer(100, 100));
1247+
deferReads();
1248+
const navigated = resized();
1249+
(yield* Queue.take(reads)).reject(new Error("Cannot find context with specified id"));
1250+
yield* Effect.promise(() => navigated);
1251+
const hidden = resized();
1252+
(yield* Queue.take(reads)).resolve(answer(0, 0));
1253+
yield* Effect.promise(() => hidden);
1254+
expect(viewportSizes(yield* Queue.takeAll(viewer.output))).toEqual([
1255+
"1920x1080",
1256+
"1920x1080",
1257+
]);
1258+
// Resizes during a read share one more read, so the newest size arrives last.
1259+
const first = resized();
1260+
const firstRead = yield* Queue.take(reads);
1261+
const burst = Promise.all([resized(), resized()]);
1262+
firstRead.resolve(answer(800, 500));
1263+
(yield* Queue.take(reads)).resolve(answer(1000, 600));
1264+
expect(yield* Queue.size(reads)).toBe(0);
1265+
yield* Effect.promise(() => Promise.all([first, burst]));
1266+
expect(viewportSizes(yield* Queue.takeAll(viewer.output))).toEqual([
1267+
"800x500",
1268+
"1000x600",
1269+
"1000x600",
1270+
]);
1271+
}),
1272+
).pipe(Effect.provide(layer)),
1273+
);
1274+
1275+
it.effect("a stalled desktop viewer still gets its size, control and reconnect", () =>
1276+
Effect.scoped(
1277+
Effect.gen(function* () {
1278+
const { tabId, viewer, context, resize } = yield* openDesktopViewer;
1279+
const output =
1280+
viewer.output as unknown as Queue.Queue<ServerBrowser.ServerBrowserViewerOutput>;
1281+
// Ready once the sizes measured on attach and on a first resize both arrived.
1282+
yield* resize(1920, 1080);
1283+
expect(viewportSizes(yield* Queue.takeAll(viewer.output))).toEqual([
1284+
"1920x1080",
1285+
"1920x1080",
1286+
]);
1287+
yield* Effect.promise(() => desktopDetachReady.promise);
1288+
yield* viewer.input({ type: "takeControl" });
1289+
context.page.emit("filechooser", {
1290+
isMultiple: () => false,
1291+
element: () => ({ getAttribute: async () => ".csv" }),
1292+
setFiles: vi.fn(async () => {}),
1293+
});
1294+
let chooser = yield* Queue.take(viewer.output);
1295+
while (chooser._tag !== "fileChooser") chooser = yield* Queue.take(viewer.output);
1296+
yield* Queue.clear(viewer.output);
1297+
// A resize compacts a full backlog but keeps control, the open picker and
1298+
// one-time notifications in order.
1299+
yield* viewer.input({ type: "takeControl" });
1300+
const notifications: Array<ServerBrowser.ServerBrowserViewerOutput> = [
1301+
{ _tag: "fileChooserClosed", id: "chooser-0" },
1302+
{ _tag: "popup", tabId: "popup-1" },
1303+
{ _tag: "download", id: "download-1", fileName: "report.csv", sizeBytes: 3 },
1304+
{ _tag: "clipboard", text: "copied" },
1305+
];
1306+
for (const item of notifications) Queue.offerUnsafe(output, item);
1307+
let acked = stallViewer(viewer);
1308+
yield* resize(1000, 600);
1309+
expect(yield* Queue.takeAll(viewer.output)).toEqual([
1310+
chooser,
1311+
...notifications,
1312+
expect.objectContaining({ _tag: "control", controller: "you" }),
1313+
{ _tag: "viewport", width: 1000, height: 600 },
1314+
]);
1315+
yield* acked;
1316+
// A control change keeps only the newest size.
1317+
yield* resize(900, 500);
1318+
yield* resize(1200, 700);
1319+
acked = stallViewer(viewer);
1320+
yield* viewer.input({ type: "releaseControl" });
1321+
expect(yield* Queue.takeAll(viewer.output)).toEqual([
1322+
expect.objectContaining({ _tag: "control", controller: "unclaimed" }),
1323+
{ _tag: "viewport", width: 1200, height: 700 },
1324+
]);
1325+
yield* acked;
1326+
// Notifications that no longer fit go oldest first, never the size.
1327+
let copies = 0;
1328+
while (Queue.offerUnsafe(output, { _tag: "clipboard", text: `copy ${copies}` })) copies++;
1329+
yield* resize(1000, 600);
1330+
const compacted = yield* Queue.takeAll(viewer.output);
1331+
expect(compacted.at(0)).toEqual({ _tag: "clipboard", text: "copy 1" });
1332+
expect(compacted.at(-1)).toEqual({ _tag: "viewport", width: 1000, height: 600 });
1333+
// The desktop takes the page back; a resize after it cannot bury the reconnect.
1334+
acked = stallViewer(viewer);
1335+
desktopDetaches.emit("detach", { threadId: scope.thread.threadId, tabId });
1336+
yield* Effect.promise(() => desktopReleased.promise);
1337+
yield* resize(1000, 600);
1338+
expect(yield* Queue.takeAll(viewer.output)).toEqual([{ _tag: "reconnect" }]);
1339+
yield* acked;
1340+
}),
1341+
).pipe(Effect.provide(layer)),
1342+
);
1343+
11471344
it.live("agents read a human's tab with no arguments and act on it only while nobody drives", () =>
11481345
Effect.scoped(
11491346
Effect.gen(function* () {

0 commit comments

Comments
 (0)