Skip to content

Commit 0f2d5d3

Browse files
committed
fix(preview): make in-app browser automation work on real app pages
1 parent 2f486ab commit 0f2d5d3

15 files changed

Lines changed: 515 additions & 48 deletions

File tree

‎apps/desktop/src/ipc/channels.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ export const PREVIEW_AUTOMATION_PRESS_CHANNEL = "desktop:preview-automation-pres
7575
export const PREVIEW_AUTOMATION_SCROLL_CHANNEL = "desktop:preview-automation-scroll";
7676
export const PREVIEW_AUTOMATION_EVALUATE_CHANNEL = "desktop:preview-automation-evaluate";
7777
export const PREVIEW_AUTOMATION_WAIT_FOR_CHANNEL = "desktop:preview-automation-wait-for";
78+
export const PREVIEW_AUTOMATION_SET_VIEWPORT_CHANNEL = "desktop:preview-automation-set-viewport";
7879
export const PREVIEW_RECORDING_START_CHANNEL = "desktop:preview-recording-start";
7980
export const PREVIEW_RECORDING_STOP_CHANNEL = "desktop:preview-recording-stop";
8081
export const PREVIEW_RECORDING_SAVE_CHANNEL = "desktop:preview-recording-save";

‎apps/desktop/src/ipc/methods/preview.ts‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ import {
22
DesktopPreviewAnnotationThemeInputSchema,
33
DesktopPreviewArtifactInputSchema,
44
DesktopPreviewAutomationClickInputSchema,
5+
DesktopPreviewAutomationSetViewportInputSchema,
6+
DesktopPreviewAutomationSnapshotInputSchema,
57
DesktopPreviewAutomationEvaluateInputSchema,
68
DesktopPreviewAutomationPressInputSchema,
79
DesktopPreviewAutomationScrollInputSchema,
@@ -276,11 +278,31 @@ export const automationStatus = DesktopIpc.makeIpcMethod({
276278

277279
export const automationSnapshot = DesktopIpc.makeIpcMethod({
278280
channel: IpcChannels.PREVIEW_AUTOMATION_SNAPSHOT_CHANNEL,
279-
payload: DesktopPreviewTabInputSchema,
281+
payload: DesktopPreviewAutomationSnapshotInputSchema,
280282
result: PreviewAutomationSnapshot,
281-
handler: Effect.fn("desktop.ipc.preview.automationSnapshot")(function* ({ tabId }) {
283+
handler: Effect.fn("desktop.ipc.preview.automationSnapshot")(function* ({ tabId, include }) {
284+
const manager = yield* PreviewManager.PreviewManager;
285+
return yield* manager.automationSnapshot(tabId, include ?? []);
286+
}),
287+
});
288+
289+
export const automationSetViewport = DesktopIpc.makeIpcMethod({
290+
channel: IpcChannels.PREVIEW_AUTOMATION_SET_VIEWPORT_CHANNEL,
291+
payload: DesktopPreviewAutomationSetViewportInputSchema,
292+
result: Schema.Void,
293+
handler: Effect.fn("desktop.ipc.preview.automationSetViewport")(function* ({
294+
tabId,
295+
width,
296+
height,
297+
clear,
298+
}) {
282299
const manager = yield* PreviewManager.PreviewManager;
283-
return yield* manager.automationSnapshot(tabId);
300+
yield* manager.automationSetViewport(
301+
tabId,
302+
clear === true || width === undefined || height === undefined
303+
? { clear: true }
304+
: { width, height },
305+
);
284306
}),
285307
});
286308

@@ -381,6 +403,7 @@ export const methods = [
381403
closePictureInPicture,
382404
automationStatus,
383405
automationSnapshot,
406+
automationSetViewport,
384407
automationClick,
385408
automationType,
386409
automationPress,

‎apps/desktop/src/preload.ts‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -225,8 +225,16 @@ contextBridge.exposeInMainWorld("desktopBridge", {
225225
automation: {
226226
status: (tabId) =>
227227
ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_STATUS_CHANNEL, { tabId }),
228-
snapshot: (tabId) =>
229-
ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_SNAPSHOT_CHANNEL, { tabId }),
228+
snapshot: (tabId, include) =>
229+
ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_SNAPSHOT_CHANNEL, {
230+
tabId,
231+
...(include === undefined ? {} : { include }),
232+
}),
233+
setViewport: (tabId, input) =>
234+
ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_SET_VIEWPORT_CHANNEL, {
235+
tabId,
236+
...input,
237+
}),
230238
click: (tabId, input) =>
231239
ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_CLICK_CHANNEL, { tabId, input }),
232240
type: (tabId, input) =>

‎apps/desktop/src/preview/Manager.test.ts‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2839,4 +2839,36 @@ describe("Preview automation diagnostics", () => {
28392839
expect(JSON.stringify(error)).not.toContain(selector);
28402840
expect("locator" in error).toBe(false);
28412841
});
2842+
2843+
it("names hidden, disabled, and ambiguous click failures without leaking the locator", () => {
2844+
const selector = "role=button[name='target-secret']";
2845+
const hidden = new PreviewManager.PreviewAutomationTargetNotFoundError({
2846+
operation: "click",
2847+
tabId: "tab_1",
2848+
selectorKind: "locator",
2849+
selectorLength: selector.length,
2850+
failureKind: "hidden",
2851+
});
2852+
const disabled = new PreviewManager.PreviewAutomationTargetNotFoundError({
2853+
operation: "click",
2854+
tabId: "tab_1",
2855+
selectorKind: "locator",
2856+
selectorLength: selector.length,
2857+
failureKind: "disabled",
2858+
});
2859+
const ambiguous = new PreviewManager.PreviewAutomationTargetNotFoundError({
2860+
operation: "click",
2861+
tabId: "tab_1",
2862+
selectorKind: "locator",
2863+
selectorLength: selector.length,
2864+
failureKind: "ambiguous",
2865+
matchCount: 3,
2866+
});
2867+
expect(hidden.message).toContain("not visible");
2868+
expect(disabled.message).toContain("disabled");
2869+
expect(ambiguous.message).toContain("matched 3 elements");
2870+
expect(hidden.message).not.toContain("secret");
2871+
expect(disabled.message).not.toContain("secret");
2872+
expect(ambiguous.message).not.toContain("secret");
2873+
});
28422874
});

0 commit comments

Comments
 (0)