Skip to content

Commit 38072bd

Browse files
fix(desktop): new tabs the agent opens from links no longer steal composer focus (#36300)
1 parent 1da05a8 commit 38072bd

4 files changed

Lines changed: 128 additions & 8 deletions

File tree

‎apps/desktop/src/preview/DesktopBrowserHost.test.ts‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1298,6 +1298,41 @@ describe("DesktopBrowserHost", () => {
12981298
}),
12991299
);
13001300

1301+
it.effect("stops crediting the agent once a person acts through a viewer", () =>
1302+
Effect.gen(function* () {
1303+
const host = yield* DesktopBrowserHost.make;
1304+
const debuggee = makeDebuggee();
1305+
host.attach(key, debuggee.tab);
1306+
const replies = yield* watchReplies(host);
1307+
const raw = (message: Record<string, unknown>) =>
1308+
host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) }));
1309+
const viewer = (id: number, method: string, type: string) =>
1310+
raw({ id, method, params: { type }, sessionId: "t3-preview-page-1" });
1311+
const led = () => host.agentLedPage(debuggee.tab.webContents);
1312+
1313+
yield* command(host, 1, "DOM.getDocument");
1314+
expect(led()).toBe(false);
1315+
yield* raw({ id: 2, method: "Target.attachToTarget", params: { targetId: "GUEST" } });
1316+
expect(yield* Queue.take(replies)).toBe(2);
1317+
yield* raw({ id: 3, method: "Page.startScreencast", sessionId: "t3-preview-page-1" });
1318+
1319+
for (const [method, type] of [
1320+
["Input.dispatchKeyEvent", "rawKeyDown"],
1321+
["Input.dispatchMouseEvent", "mousePressed"],
1322+
["Input.dispatchTouchEvent", "touchStart"],
1323+
] as const) {
1324+
yield* command(host, 4, "Runtime.evaluate");
1325+
expect(led()).toBe(true);
1326+
// The viewer's pointer only passes over the page the agent drives.
1327+
yield* viewer(5, "Input.dispatchMouseEvent", "mouseMoved");
1328+
yield* viewer(6, "Input.dispatchMouseEvent", "mouseWheel");
1329+
expect(led()).toBe(true);
1330+
yield* viewer(7, method, type);
1331+
expect(led()).toBe(false);
1332+
}
1333+
}).pipe(Effect.scoped),
1334+
);
1335+
13011336
it.effect("detaches only the tab whose CDP event is too large to relay", () =>
13021337
Effect.gen(function* () {
13031338
const host = yield* DesktopBrowserHost.make;

‎apps/desktop/src/preview/DesktopBrowserHost.ts‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,9 @@ const AGENT_INPUT_COMMAND =
2626
/"method":"(?:Input\.|Page\.navigate"|Runtime\.(?:evaluate|callFunctionOn)")/;
2727
/** The same commands as above, by method, for the relay's sends. */
2828
const AGENT_INPUT_METHOD = /^(?:Input\.|Page\.navigate$|Runtime\.(?:evaluate|callFunctionOn)$)/;
29-
const AGENT_DOWNLOAD_WINDOW_MS = 5_000;
29+
const AGENT_INPUT_WINDOW_MS = 5_000;
30+
/** A viewer's pointer travelling over the stream activates nothing on the page. */
31+
const POINTER_TRAVEL = new Set<unknown>(["mouseMoved", "mouseWheel"]);
3032
const FOCUS_IDLE = "5 seconds";
3133
/** Well inside the server's 10 s DESKTOP_ATTACH_TIMEOUT, for a first page that never reports it is up. */
3234
const FIRST_PAGE_TIMEOUT = "5 seconds";
@@ -60,6 +62,8 @@ interface AttachedTab {
6062
downloadDirectory: string | null;
6163
pendingDownloadGuid: string | null;
6264
agentInputAt: number;
65+
/** When the agent last drove the page with no person's input through a viewer since. */
66+
agentLedAt: number;
6367
focusEmulated: boolean;
6468
focusChange: Promise<void> | null;
6569
focusUses: number;
@@ -98,6 +102,8 @@ export class DesktopBrowserHost extends Context.Service<
98102
readonly loadFailed: (key: DesktopBrowserTabKey, failure: PreviewAutomationLoadError) => void;
99103
readonly setShown: (key: DesktopBrowserTabKey, shown: boolean | undefined) => void;
100104
readonly humanStartedDownload: (source: Electron.WebContents) => boolean;
105+
/** Whether what the page does now follows the agent's input, not a viewer's. */
106+
readonly agentLedPage: (source: Electron.WebContents) => boolean;
101107
readonly placeDownload: (source: Electron.WebContents, item: Electron.DownloadItem) => boolean;
102108
readonly pointers: Stream.Stream<{
103109
readonly key: DesktopBrowserTabKey;
@@ -351,11 +357,13 @@ export const make = Effect.gen(function* () {
351357
if (tab.screencasts.size > 0) return Promise.resolve({});
352358
}
353359
// A person's input through a viewer, or Playwright's own housekeeping, is not the agent's.
354-
if (
355-
AGENT_INPUT_METHOD.test(method) &&
356-
(owner === undefined || !streamingOwners.has(owner))
357-
) {
358-
tab.agentUsed = true;
360+
if (AGENT_INPUT_METHOD.test(method)) {
361+
if (owner === undefined || !streamingOwners.has(owner)) {
362+
tab.agentUsed = true;
363+
tab.agentLedAt = performance.now();
364+
} else if (method.startsWith("Input.") && !POINTER_TRAVEL.has(params["type"])) {
365+
tab.agentLedAt = Number.NEGATIVE_INFINITY;
366+
}
359367
}
360368
// Waits for focus emulation; insertText and other non-CDP input would not queue behind it.
361369
const forward = (current?: () => boolean) => {
@@ -490,6 +498,7 @@ export const make = Effect.gen(function* () {
490498
downloadDirectory: null,
491499
pendingDownloadGuid: null,
492500
agentInputAt: Number.NEGATIVE_INFINITY,
501+
agentLedAt: Number.NEGATIVE_INFINITY,
493502
focusEmulated: false,
494503
focusChange: null,
495504
focusUses: 0,
@@ -604,7 +613,12 @@ export const make = Effect.gen(function* () {
604613

605614
const humanStartedDownload = (source: Electron.WebContents) => {
606615
const tab = [...tabs.values()].find((candidate) => candidate.debuggee.webContents === source);
607-
return tab !== undefined && performance.now() - tab.agentInputAt > AGENT_DOWNLOAD_WINDOW_MS;
616+
return tab !== undefined && performance.now() - tab.agentInputAt > AGENT_INPUT_WINDOW_MS;
617+
};
618+
619+
const agentLedPage = (source: Electron.WebContents) => {
620+
const tab = [...tabs.values()].find((candidate) => candidate.debuggee.webContents === source);
621+
return tab !== undefined && performance.now() - tab.agentLedAt <= AGENT_INPUT_WINDOW_MS;
608622
};
609623

610624
const announceAll = Effect.suspend(() => {
@@ -647,6 +661,7 @@ export const make = Effect.gen(function* () {
647661
setShown,
648662
placeDownload,
649663
humanStartedDownload,
664+
agentLedPage,
650665
});
651666
});
652667

‎apps/desktop/src/preview/Manager.test.ts‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -619,6 +619,7 @@ describe("PreviewManager", () => {
619619
setShown: () => {},
620620
loadFailed: () => {},
621621
humanStartedDownload: () => true,
622+
agentLedPage: () => false,
622623
placeDownload: () => true,
623624
}),
624625
Layer.mock(PreviewPasskeys.PreviewPasskeys)({
@@ -2196,6 +2197,71 @@ describe("PreviewManager", () => {
21962197
}).pipe(Effect.provide(managerLayer())),
21972198
);
21982199

2200+
effectIt.effect(
2201+
"opens a new-tab link in the background only when the agent's input led to it",
2202+
() =>
2203+
Effect.gen(function* () {
2204+
const host = yield* DesktopBrowserHost.DesktopBrowserHost;
2205+
const serverTab = { threadId: "thread-1", tabId: "server-tab-1" };
2206+
const contents = Object.assign(
2207+
makeTestPreviewWebContents(async () => {
2208+
throw new Error("unused");
2209+
}, 42),
2210+
{
2211+
isDevToolsOpened: () => false,
2212+
getUserAgent: () => "Electron",
2213+
focusedFrame: null,
2214+
},
2215+
);
2216+
fromId.mockReturnValue(contents);
2217+
const manager = yield* PreviewManager.PreviewManager;
2218+
const openLinks = yield* Queue.unbounded<boolean>();
2219+
yield* manager.subscribeOpenLinks(({ background }) =>
2220+
Queue.offer(openLinks, background).pipe(Effect.asVoid),
2221+
);
2222+
const attached = yield* Deferred.make<void>();
2223+
yield* host.events.pipe(
2224+
Stream.runForEach((line) =>
2225+
new TextDecoder().decode(line).includes('"type":"attached"')
2226+
? Deferred.succeed(attached, undefined)
2227+
: Effect.void,
2228+
),
2229+
Effect.forkScoped,
2230+
);
2231+
yield* manager.createTab("tab_1", { serverTab });
2232+
yield* manager.registerWebview("tab_1", 42);
2233+
yield* Deferred.await(attached);
2234+
const openWindow = vi.mocked(contents.setWindowOpenHandler).mock.calls[0]![0];
2235+
const openBlankLink = Effect.suspend(() => {
2236+
openWindow({
2237+
url: "https://example.com/next",
2238+
disposition: "foreground-tab",
2239+
} as Electron.HandlerDetails);
2240+
return Queue.take(openLinks);
2241+
});
2242+
2243+
expect(yield* openBlankLink).toBe(false);
2244+
2245+
yield* host.handleCommandLine(
2246+
JSON.stringify({
2247+
type: "cdp",
2248+
...serverTab,
2249+
message: JSON.stringify({
2250+
id: 1,
2251+
method: "Runtime.evaluate",
2252+
params: { expression: "document.querySelector('a').click()" },
2253+
sessionId: "t3-preview-page",
2254+
}),
2255+
}),
2256+
);
2257+
expect(yield* openBlankLink).toBe(true);
2258+
2259+
// A person clicking in the guest itself gives it a focused frame.
2260+
Object.assign(contents, { focusedFrame: {} });
2261+
expect(yield* openBlankLink).toBe(false);
2262+
}).pipe(Effect.provide(managerLayer()), Effect.scoped),
2263+
);
2264+
21992265
effectIt.effect("reports early and subsequent load failures for the current guest", () =>
22002266
Effect.gen(function* () {
22012267
const host = yield* DesktopBrowserHost.DesktopBrowserHost;

‎apps/desktop/src/preview/Manager.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1656,7 +1656,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
16561656
emitOpenLink({
16571657
tabId,
16581658
url: details.url,
1659-
background: details.disposition === "background-tab",
1659+
// A person's own click or key leaves the guest a focused frame; GuestInput hands
1660+
// focus back after the agent's press, so a foreground tab would take it again.
1661+
background:
1662+
details.disposition === "background-tab" ||
1663+
(wc.focusedFrame === null && browserHost.agentLedPage(wc)),
16601664
}),
16611665
);
16621666
return { action: "deny" };

0 commit comments

Comments
 (0)