Skip to content

Commit 01d6ea1

Browse files
steipeteGRD-Chang
steipete
and
GRD-Chang
authored
feat(browser): expose agent download actions (#101369)
* feat(browser): expose agent download actions Co-authored-by: changcy <[email protected]> * chore: keep release notes in PR context --------- Co-authored-by: changcy <[email protected]>
1 parent 473b3cf commit 01d6ea1

17 files changed

Lines changed: 434 additions & 6 deletions

‎docs/tools/browser-control.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -264,6 +264,11 @@ openclaw browser set device "iPhone 14"
264264
265265
Notes:
266266
267+
- The agent-facing `browser` tool exposes `action=download` (required `ref` and
268+
`path`) and `action=waitfordownload` (optional `path`). Both return the saved
269+
download URL, suggested filename, and guarded local path. Explicit download
270+
interception is available for managed Playwright profiles; existing-session
271+
profiles return an unsupported-operation error.
267272
- `upload` and `dialog` are **arming** calls; run them before the click/press that triggers the chooser/dialog. If an action opens a modal, the action response includes `blockedByDialog` and `browserState.dialogs.pending`; pass that `dialogId` to respond directly. Dialogs handled outside OpenClaw appear under `browserState.dialogs.recent`.
268273
- `click`/`type`/etc require a `ref` from `snapshot` (numeric `12`, role ref `e12`, or actionable ARIA ref `ax12`). CSS selectors are intentionally not supported for actions. Use `click-coords` when the visible viewport position is the only reliable target.
269274
- Download and trace paths are constrained to OpenClaw temp roots: `/tmp/openclaw{,/downloads}` (fallback: `${os.tmpdir()}/openclaw/...`).

‎extensions/browser/plugin-registration.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ function createLazyBrowserTool(opts?: {
7575
label: "Browser",
7676
name: "browser",
7777
description: [
78-
"Control the browser via OpenClaw's browser control server (status/start/stop/profiles/tabs/open/snapshot/screenshot/actions).",
78+
"Control the browser via OpenClaw's browser control server (status/start/stop/profiles/tabs/open/snapshot/screenshot/download/actions).",
7979
"Browser choice: omit profile by default for the isolated OpenClaw-managed browser (`openclaw`).",
8080
'For the logged-in user browser, use profile="user". A supported Chromium-based browser (v144+) must be running on the selected host or browser node. Use only when existing logins/cookies matter and the user is present.',
8181
'For profile="user" or other existing-session profiles, omit timeoutMs on act:type, evaluate, hover, scrollIntoView, drag, select, and fill; that driver rejects per-call timeout overrides for those actions.',

‎extensions/browser/src/browser-tool.actions.ts‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,16 @@ import {
1313
DEFAULT_AI_SNAPSHOT_MAX_CHARS,
1414
browserAct,
1515
browserConsoleMessages,
16+
browserDownload,
1617
browserSnapshot,
1718
browserTabs,
19+
browserWaitForDownload,
1820
getBrowserProfileCapabilities,
1921
getRuntimeConfig,
2022
imageResultFromFile,
2123
jsonResult,
2224
normalizeOptionalString,
25+
readStringParam,
2326
readStringValue,
2427
resolveBrowserConfig,
2528
resolveProfile,
@@ -28,20 +31,24 @@ import {
2831
} from "./browser-tool.runtime.js";
2932
import {
3033
DEFAULT_BROWSER_ACTION_TIMEOUT_MS,
34+
DEFAULT_BROWSER_DOWNLOAD_TIMEOUT_MS,
3135
DEFAULT_BROWSER_SNAPSHOT_TIMEOUT_MS,
3236
} from "./browser/constants.js";
3337
import { neutralizeMediaDirectives } from "./browser/vision.js";
3438

3539
const browserToolActionDeps = {
3640
browserAct,
3741
browserConsoleMessages,
42+
browserDownload,
3843
browserSnapshot,
3944
browserTabs,
45+
browserWaitForDownload,
4046
getRuntimeConfig,
4147
imageResultFromFile,
4248
};
4349

4450
const BROWSER_ACT_REQUEST_TIMEOUT_SLACK_MS = 5_000;
51+
const BROWSER_DOWNLOAD_REQUEST_TIMEOUT_SLACK_MS = 5_000;
4552

4653
type BrowserActRequest = Parameters<typeof browserAct>[1];
4754
type BrowserActRequestWithTimeout = BrowserActRequest & { timeoutMs?: number };
@@ -142,17 +149,22 @@ export const testing = {
142149
overrides: Partial<{
143150
browserAct: typeof browserAct;
144151
browserConsoleMessages: typeof browserConsoleMessages;
152+
browserDownload: typeof browserDownload;
145153
browserSnapshot: typeof browserSnapshot;
146154
browserTabs: typeof browserTabs;
155+
browserWaitForDownload: typeof browserWaitForDownload;
147156
imageResultFromFile: typeof imageResultFromFile;
148157
getRuntimeConfig: typeof getRuntimeConfig;
149158
}> | null,
150159
) {
151160
browserToolActionDeps.browserAct = overrides?.browserAct ?? browserAct;
152161
browserToolActionDeps.browserConsoleMessages =
153162
overrides?.browserConsoleMessages ?? browserConsoleMessages;
163+
browserToolActionDeps.browserDownload = overrides?.browserDownload ?? browserDownload;
154164
browserToolActionDeps.browserSnapshot = overrides?.browserSnapshot ?? browserSnapshot;
155165
browserToolActionDeps.browserTabs = overrides?.browserTabs ?? browserTabs;
166+
browserToolActionDeps.browserWaitForDownload =
167+
overrides?.browserWaitForDownload ?? browserWaitForDownload;
156168
browserToolActionDeps.imageResultFromFile =
157169
overrides?.imageResultFromFile ?? imageResultFromFile;
158170
browserToolActionDeps.getRuntimeConfig = overrides?.getRuntimeConfig ?? getRuntimeConfig;
@@ -570,6 +582,78 @@ export async function executeConsoleAction(params: {
570582
return formatConsoleToolResult(result);
571583
}
572584

585+
function resolveDownloadProxyTimeoutMs(timeoutMs: number | undefined): number {
586+
const waitTimeoutMs = timeoutMs ?? DEFAULT_BROWSER_DOWNLOAD_TIMEOUT_MS;
587+
// The node proxy must outlive the browser-server request; callBrowserProxy
588+
// adds a second grace window for the outer Gateway node.invoke call.
589+
return waitTimeoutMs + BROWSER_DOWNLOAD_REQUEST_TIMEOUT_SLACK_MS;
590+
}
591+
592+
type BrowserDownloadRequest =
593+
| { action: "download"; route: "/download"; ref: string; path: string }
594+
| { action: "waitfordownload"; route: "/wait/download"; path?: string };
595+
596+
function readBrowserDownloadRequest(
597+
action: BrowserDownloadRequest["action"],
598+
input: Record<string, unknown>,
599+
): BrowserDownloadRequest {
600+
if (action === "download") {
601+
return {
602+
action,
603+
route: "/download",
604+
ref: readStringParam(input, "ref", { required: true }),
605+
path: readStringParam(input, "path", { required: true }),
606+
};
607+
}
608+
return {
609+
action,
610+
route: "/wait/download",
611+
path: readStringParam(input, "path"),
612+
};
613+
}
614+
615+
/** Execute explicit Browser download operations through the local or node-host path. */
616+
export async function executeDownloadAction(params: {
617+
action: "download" | "waitfordownload";
618+
input: Record<string, unknown>;
619+
baseUrl?: string;
620+
profile?: string;
621+
proxyRequest: BrowserProxyRequest | null;
622+
onTabActivity?: (targetId: string | undefined) => void;
623+
}): Promise<AgentToolResult<unknown>> {
624+
const { action, input, baseUrl, profile, proxyRequest } = params;
625+
const targetId = normalizeOptionalString(input.targetId);
626+
const timeoutMs = normalizePositiveTimeoutMs(input.timeoutMs);
627+
const request = readBrowserDownloadRequest(action, input);
628+
const result = proxyRequest
629+
? await proxyRequest({
630+
method: "POST",
631+
path: request.route,
632+
profile,
633+
timeoutMs: resolveDownloadProxyTimeoutMs(timeoutMs),
634+
body:
635+
request.action === "download"
636+
? { ref: request.ref, path: request.path, targetId, timeoutMs }
637+
: { path: request.path, targetId, timeoutMs },
638+
})
639+
: request.action === "download"
640+
? await browserToolActionDeps.browserDownload(baseUrl, {
641+
ref: request.ref,
642+
path: request.path,
643+
targetId,
644+
timeoutMs,
645+
profile,
646+
})
647+
: await browserToolActionDeps.browserWaitForDownload(baseUrl, {
648+
path: request.path,
649+
targetId,
650+
timeoutMs,
651+
profile,
652+
});
653+
params.onTabActivity?.(readStringValue((result as { targetId?: unknown }).targetId) ?? targetId);
654+
return jsonResult(result);
655+
}
656+
573657
/** Execute browser actions with profile-aware timeout defaults and stale-tab recovery. */
574658
export async function executeActAction(params: {
575659
request: BrowserActRequest;

‎extensions/browser/src/browser-tool.runtime.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,9 +39,11 @@ export {
3939
browserArmDialog,
4040
browserArmFileChooser,
4141
browserConsoleMessages,
42+
browserDownload,
4243
browserNavigate,
4344
browserPdfSave,
4445
browserScreenshotAction,
46+
browserWaitForDownload,
4547
} from "./browser/client-actions.js";
4648
export {
4749
browserCloseTab,

‎extensions/browser/src/browser-tool.schema.test.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { ACT_MAX_VIEWPORT_DIMENSION } from "./browser/act-policy.js";
66
type SchemaRecord = Record<string, { maximum?: number; properties?: SchemaRecord }>;
77
type SchemaProperty = {
88
description?: string;
9+
enum?: string[];
910
maximum?: number;
1011
properties?: SchemaRecord;
1112
};
@@ -30,4 +31,11 @@ describe("browser tool schema", () => {
3031
expect(properties.targetId.description).toContain("raw CDP targetId");
3132
expect(requestProperties.targetId.description).toBe(properties.targetId.description);
3233
});
34+
35+
it("exposes explicit download actions and their output path", () => {
36+
const properties = BrowserToolSchema.properties as BrowserSchemaRecord;
37+
38+
expect(properties.action.enum).toEqual(expect.arrayContaining(["download", "waitfordownload"]));
39+
expect(properties.path).toBeDefined();
40+
});
3341
});

‎extensions/browser/src/browser-tool.schema.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ const BROWSER_TOOL_ACTIONS = [
4444
"navigate",
4545
"console",
4646
"pdf",
47+
"download",
48+
"waitfordownload",
4749
"upload",
4850
"dialog",
4951
"act",
@@ -129,6 +131,7 @@ export const BrowserToolSchema = Type.Object({
129131
urls: Type.Optional(Type.Boolean()),
130132
fullPage: Type.Optional(Type.Boolean()),
131133
ref: Type.Optional(Type.String()),
134+
path: Type.Optional(Type.String()),
132135
element: Type.Optional(Type.String()),
133136
type: optionalStringEnum(BROWSER_IMAGE_TYPES),
134137
level: Type.Optional(Type.String()),

‎extensions/browser/src/browser-tool.test.ts‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,26 @@ const browserActionsMocks = vi.hoisted(() => ({
5959
],
6060
})),
6161
browserNavigate: vi.fn(async () => ({ ok: true })),
62+
browserDownload: vi.fn(async () => ({
63+
ok: true,
64+
targetId: "tab-1",
65+
download: {
66+
path: "/tmp/openclaw/downloads/report.pdf",
67+
suggestedFilename: "report.pdf",
68+
url: "https://example.com/report.pdf",
69+
},
70+
})),
6271
browserPdfSave: vi.fn(async () => ({ ok: true, path: "/tmp/test.pdf" })),
6372
browserScreenshotAction: vi.fn(async () => ({ ok: true, path: "/tmp/test.png" })),
73+
browserWaitForDownload: vi.fn(async () => ({
74+
ok: true,
75+
targetId: "tab-1",
76+
download: {
77+
path: "/tmp/openclaw/downloads/export.csv",
78+
suggestedFilename: "export.csv",
79+
url: "https://example.com/export.csv",
80+
},
81+
})),
6482
}));
6583
vi.mock("./browser/client-actions.js", () => browserActionsMocks);
6684

@@ -324,8 +342,10 @@ function resetBrowserToolMocks() {
324342
browserToolActionsTesting.setDepsForTest({
325343
browserAct: browserActionsMocks.browserAct as never,
326344
browserConsoleMessages: browserActionsMocks.browserConsoleMessages as never,
345+
browserDownload: browserActionsMocks.browserDownload as never,
327346
browserSnapshot: browserClientMocks.browserSnapshot as never,
328347
browserTabs: browserClientMocks.browserTabs as never,
348+
browserWaitForDownload: browserActionsMocks.browserWaitForDownload as never,
329349
getRuntimeConfig: configMocks.loadConfig as never,
330350
imageResultFromFile: toolCommonMocks.imageResultFromFile as never,
331351
});
@@ -483,6 +503,126 @@ describe("browser tool description", () => {
483503
});
484504
});
485505

506+
describe("browser tool download actions", () => {
507+
registerBrowserToolAfterEachReset();
508+
509+
it("downloads a snapshot ref through the host client", async () => {
510+
const tool = createBrowserTool({ agentSessionKey: "agent:main:main" });
511+
512+
const result = await tool.execute?.("call-1", {
513+
action: "download",
514+
target: "host",
515+
profile: "openclaw",
516+
ref: "e12",
517+
path: "report.pdf",
518+
targetId: "tab-1",
519+
timeoutMs: "30000",
520+
});
521+
522+
expect(browserActionsMocks.browserDownload).toHaveBeenCalledWith(undefined, {
523+
ref: "e12",
524+
path: "report.pdf",
525+
targetId: "tab-1",
526+
timeoutMs: 30_000,
527+
profile: "openclaw",
528+
});
529+
expect(result?.details).toMatchObject({
530+
download: { path: "/tmp/openclaw/downloads/report.pdf" },
531+
});
532+
expect(sessionTabRegistryMocks.touchSessionBrowserTab).toHaveBeenCalledWith(
533+
expect.objectContaining({ sessionKey: "agent:main:main", targetId: "tab-1" }),
534+
);
535+
});
536+
537+
it("waits for the next host download without requiring a path", async () => {
538+
const tool = createBrowserTool();
539+
540+
await tool.execute?.("call-1", {
541+
action: "waitfordownload",
542+
target: "host",
543+
targetId: "tab-1",
544+
});
545+
546+
expect(browserActionsMocks.browserWaitForDownload).toHaveBeenCalledWith(undefined, {
547+
path: undefined,
548+
targetId: "tab-1",
549+
timeoutMs: undefined,
550+
profile: undefined,
551+
});
552+
});
553+
554+
it("keeps requested download waits open across node and Gateway timeouts", async () => {
555+
mockSingleBrowserProxyNode();
556+
gatewayMocks.callGatewayTool.mockResolvedValueOnce({
557+
ok: true,
558+
payload: {
559+
result: {
560+
ok: true,
561+
targetId: "tab-1",
562+
download: { path: "/tmp/openclaw/downloads/export.csv" },
563+
},
564+
},
565+
});
566+
const tool = createBrowserTool();
567+
568+
await tool.execute?.("call-1", {
569+
action: "waitfordownload",
570+
target: "node",
571+
path: "export.csv",
572+
targetId: "tab-1",
573+
timeoutMs: 30_000,
574+
});
575+
576+
const { options, request } = lastNodeInvokeCall();
577+
expect(options.timeoutMs).toBe(40_000);
578+
expect(request.params?.path).toBe("/wait/download");
579+
expect(request.params?.timeoutMs).toBe(35_000);
580+
expect(request.params?.body).toEqual({
581+
path: "export.csv",
582+
targetId: "tab-1",
583+
timeoutMs: 30_000,
584+
});
585+
});
586+
587+
it("keeps the default node download wait beyond the legacy proxy ceiling", async () => {
588+
mockSingleBrowserProxyNode();
589+
gatewayMocks.callGatewayTool.mockResolvedValueOnce({
590+
ok: true,
591+
payload: {
592+
result: {
593+
ok: true,
594+
targetId: "tab-1",
595+
download: { path: "/tmp/openclaw/downloads/report.pdf" },
596+
},
597+
},
598+
});
599+
const tool = createBrowserTool();
600+
601+
await tool.execute?.("call-1", {
602+
action: "download",
603+
target: "node",
604+
ref: "e12",
605+
path: "report.pdf",
606+
});
607+
608+
const { options, request } = lastNodeInvokeCall();
609+
expect(options.timeoutMs).toBe(130_000);
610+
expect(request.params?.timeoutMs).toBe(125_000);
611+
expect(request.params?.path).toBe("/download");
612+
expect(request.params?.body).toMatchObject({ ref: "e12", path: "report.pdf" });
613+
});
614+
615+
it.each([
616+
[{ action: "download", ref: "e12" }, "path required"],
617+
[{ action: "download", path: "report.pdf" }, "ref required"],
618+
])("rejects incomplete download input %#", async (input, message) => {
619+
const tool = createBrowserTool();
620+
621+
await expect(tool.execute?.("call-1", { target: "host", ...input })).rejects.toThrow(message);
622+
expect(browserActionsMocks.browserDownload).not.toHaveBeenCalled();
623+
});
624+
});
625+
486626
describe("browser tool snapshot maxChars", () => {
487627
registerBrowserToolAfterEachReset();
488628

0 commit comments

Comments
 (0)