Skip to content

Commit cef4a0e

Browse files
Merge remote-tracking branch 'origin/main' into fix/issue-32827
2 parents e45ed8b + 3cd97a1 commit cef4a0e

10 files changed

Lines changed: 142 additions & 12 deletions

File tree

‎apps/mobile/src/components/FilePreview.tsx‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import ImageViewing from "react-native-image-viewing";
66
import { useSafeAreaInsets } from "react-native-safe-area-context";
77

88
import { openAttachmentInViewer } from "../lib/attachmentDownload";
9+
import { useMediaActions } from "../state/mediaActions";
910
import type {
1011
ResolvedFilePreviewDocument,
1112
ResolvedFilePreviewSource,
@@ -20,6 +21,8 @@ function PdfDocumentPreview(props: {
2021
readonly onRequestClose: () => void;
2122
}) {
2223
const insets = useSafeAreaInsets();
24+
const media = useMediaActions(props.source.actionsSource);
25+
const save = media.actions.find((action) => action.id === "save");
2326
return (
2427
<Modal
2528
animationType="slide"
@@ -35,6 +38,19 @@ function PdfDocumentPreview(props: {
3538
presentation="inline"
3639
title={props.source.name ?? "PDF"}
3740
onClose={props.onRequestClose}
41+
actions={
42+
save
43+
? [
44+
{
45+
accessibilityLabel: "Save or share",
46+
icon: "square.and.arrow.up",
47+
disabled: save.disabled,
48+
loading: media.sharing,
49+
onPress: () => void save.run(),
50+
},
51+
]
52+
: []
53+
}
3854
/>
3955
<PdfPreview uri={props.source.uri} />
4056
</View>

‎apps/mobile/src/components/FilePreviewModal.tsx‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import { refreshAssetUrl } from "../state/assets";
1414
import { filePreviewGallery, filePreviewKey, filePreviewName } from "../lib/filePreviewGallery";
1515
import { FilePreviewGallery } from "./FilePreviewGallery";
1616
import { FilePreview } from "./FilePreview";
17+
import type { MediaActionsSource } from "../lib/mediaActionsSource";
1718
import type { FilePreviewItem, FilePreviewSource } from "./FilePreviewModal.types";
1819

1920
export type {
@@ -67,6 +68,18 @@ async function resolvePreviewUri(
6768
return { uri: url + (item.srcFragment ?? "") };
6869
}
6970

71+
// Shares from the unresolved item so remote URLs refresh and draft files take their own lease.
72+
function pdfActionsSource(item: FilePreviewItem): MediaActionsSource {
73+
const metadata = {
74+
name: filePreviewName(item),
75+
mimeType: item.mimeType ?? "application/pdf",
76+
...(item.sourceIdentifier ? { sourceIdentifier: item.sourceIdentifier } : {}),
77+
};
78+
if ("uri" in item) return { ...metadata, uri: item.uri };
79+
if ("attachment" in item) return { ...metadata, attachment: item.attachment };
80+
return { ...metadata, environmentId: item.environmentId, resource: item.resource };
81+
}
82+
7083
function ResolvedFilePreview(props: {
7184
readonly source: FilePreviewSource;
7285
readonly onRequestClose: () => void;
@@ -166,9 +179,16 @@ function ResolvedFilePreview(props: {
166179
: []),
167180
]);
168181
};
182+
const actionsSource =
183+
selected.kind === "pdf" ? (selected.actionsSource ?? pdfActionsSource(selected)) : undefined;
169184
return (
170185
<FilePreview
171-
source={{ ...selected, kind: selected.kind, uri }}
186+
source={{
187+
...selected,
188+
kind: selected.kind,
189+
uri,
190+
...(actionsSource ? { actionsSource } : {}),
191+
}}
172192
onRequestClose={props.onRequestClose}
173193
onOpenError={onOpenError}
174194
/>

‎apps/mobile/src/features/files/ThreadFilesRouteScreen.tsx‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -845,6 +845,16 @@ export function ThreadFileScreen(props: ThreadFileRouteScreenProps) {
845845
kind: "pdf",
846846
uri: previewUri,
847847
name: basename(relativePath),
848+
...(environmentId !== null && assetPreview.resource !== null
849+
? {
850+
actionsSource: {
851+
name: basename(relativePath),
852+
mimeType: "application/pdf",
853+
environmentId,
854+
resource: assetPreview.resource,
855+
},
856+
}
857+
: {}),
848858
}),
849859
} as const)
850860
: null,

‎apps/server/src/mcp/McpHttpServer.test.ts‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import {
2020
import * as Context from "effect/Context";
2121
import * as Deferred from "effect/Deferred";
2222
import * as Effect from "effect/Effect";
23+
import * as FileSystem from "effect/FileSystem";
2324
import * as Layer from "effect/Layer";
2425
import * as Logger from "effect/Logger";
2526
import * as Schema from "effect/Schema";
@@ -467,6 +468,57 @@ it.effect.each([
467468
).pipe(Effect.provide(layerTest)),
468469
);
469470

471+
it.effect("saves the uncapped screenshot while the agent sees the capped one", () =>
472+
Effect.scoped(
473+
Effect.gen(function* () {
474+
const server = yield* McpServer.McpServer;
475+
const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker;
476+
const connected = yield* Deferred.make<void>();
477+
const capped =
478+
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jRZkAAAAASUVORK5CYII=";
479+
const full =
480+
"iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAAAAABX3VL4AAAADklEQVR4nGNgYGBgYAAAAAYAAXKCmeoAAAAASUVORK5CYII=";
481+
const events = yield* broker.connect({ clientId: "mcp-save-client", environmentId });
482+
yield* Stream.runForEach(events, (event) => {
483+
if (event.type === "connected") return Deferred.succeed(connected, undefined);
484+
expect(event.request.input).toEqual({ fullScreenshot: true });
485+
return broker.respond({
486+
clientId: "mcp-save-client",
487+
connectionId: event.connectionId,
488+
requestId: event.request.requestId,
489+
ok: true,
490+
result: {
491+
...snapshotResult,
492+
screenshot: { mimeType: "image/png", data: capped, width: 1, height: 1 },
493+
fullScreenshot: { data: full, width: 2, height: 2 },
494+
},
495+
});
496+
}).pipe(Effect.forkScoped);
497+
yield* Deferred.await(connected);
498+
499+
const snapshot = yield* server
500+
.callTool({
501+
name: "preview_snapshot",
502+
arguments: { tabId: alternateTabId, save: true, includeImage: true },
503+
})
504+
.pipe(
505+
Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),
506+
Effect.provideService(McpSchema.McpServerClient, client),
507+
);
508+
const { screenshotPath } = snapshot.structuredContent as { readonly screenshotPath: string };
509+
const saved = yield* (yield* FileSystem.FileSystem).readFile(screenshotPath);
510+
511+
expect(Buffer.from(saved).toString("base64")).toBe(full);
512+
expect(snapshot.structuredContent).not.toHaveProperty("fullScreenshot");
513+
expect(snapshot.content.at(-1)).toEqual({
514+
type: "image",
515+
mimeType: "image/png",
516+
data: new Uint8Array(Buffer.from(capped, "base64")),
517+
});
518+
}),
519+
).pipe(Effect.provide(layerTest)),
520+
);
521+
470522
it.effect("refuses preview tools to a client outside a thread before they run", () =>
471523
Effect.gen(function* () {
472524
const server = yield* McpServer.McpServer;

‎apps/server/src/mcp/McpHttpServer.ts‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -557,11 +557,19 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
557557
readonly width: number;
558558
readonly height: number;
559559
};
560+
readonly fullScreenshot?: { readonly data: string };
560561
};
561-
const { screenshot, ...page } = snapshot;
562+
const { screenshot, fullScreenshot, ...page } = snapshot;
562563
const png = new Uint8Array(Buffer.from(screenshot.data, "base64"));
563564
const screenshotPath =
564-
payload?.save === true ? yield* saveScreenshot(snapshot.url, png) : undefined;
565+
payload?.save === true
566+
? yield* saveScreenshot(
567+
snapshot.url,
568+
fullScreenshot
569+
? new Uint8Array(Buffer.from(fullScreenshot.data, "base64"))
570+
: png,
571+
)
572+
: undefined;
565573
// Providers replay images on later requests, and some reject them; include images only when requested.
566574
const includeImage = payload?.includeImage === true;
567575
if (screenshotPath !== undefined && !includeImage) {

‎apps/server/src/mcp/toolkits/preview/handlers.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -221,8 +221,12 @@ const handlers = {
221221
invokeTargeted<PreviewAutomationSetColorSchemeResult>("setColorScheme", input),
222222
),
223223
preview_snapshot: McpToolAccess.readsAsCaller((input) => {
224-
const { includeImage: _includeImage, save: _save, ...operationInput } = input ?? {};
225-
return invokeTargeted<PreviewAutomationSnapshot>("snapshot", operationInput);
224+
const { includeImage: _includeImage, save, ...operationInput } = input ?? {};
225+
// The saved file is for people, so it keeps the resolution the agent's image is capped below.
226+
return invokeTargeted<PreviewAutomationSnapshot>("snapshot", {
227+
...operationInput,
228+
...(save === true ? { fullScreenshot: true } : {}),
229+
});
226230
}),
227231
preview_act: McpToolAccess.actsAsCaller(({ action, ...input }) =>
228232
decodeActInput[action](input).pipe(

‎apps/server/src/preview/ServerBrowser.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import {
3030
type PreviewAutomationScrollInput,
3131
type PreviewAutomationSelectInput,
3232
type PreviewAutomationSetColorSchemeInput,
33+
type PreviewAutomationSnapshotInput,
3334
type PreviewAutomationStatus,
3435
type PreviewAutomationTypeInput,
3536
type PreviewAutomationUploadInput,
@@ -2241,6 +2242,7 @@ const make = Effect.gen(function* () {
22412242
ServerBrowserPage.snapshot({
22422243
...tab,
22432244
renderScale: tab.desktop ? "display" : RENDER_SCALE,
2245+
fullScreenshot: (input as PreviewAutomationSnapshotInput | undefined)?.fullScreenshot,
22442246
}),
22452247
);
22462248
}

‎apps/server/src/preview/ServerBrowserPage.test.ts‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -309,10 +309,11 @@ describe("server browser snapshot screenshot", () => {
309309
const screenshotOf = async (html: string) => {
310310
const page = await context.newPage();
311311
await page.setContent(html);
312-
const { screenshot } = await ServerBrowserPage.snapshot({
312+
const { screenshot, fullScreenshot } = await ServerBrowserPage.snapshot({
313313
page,
314314
cdp: await context.newCDPSession(page),
315315
renderScale: 2,
316+
fullScreenshot: true,
316317
consoleEntries: [],
317318
networkEntries: [],
318319
actionTimeline: [],
@@ -335,7 +336,7 @@ describe("server browser snapshot screenshot", () => {
335336
Array.from(context.getImageData(x * bitmap.width, y * bitmap.height, 1, 1).data),
336337
);
337338
})()`)) as Array<[number, number, number, number]>;
338-
return { screenshot, pixels };
339+
return { screenshot, fullScreenshot, pixels };
339340
};
340341
const colorOf = ([red, green, blue]: readonly [number, number, number, number]) =>
341342
blue - red > 100 ? "blue" : red - blue > 100 ? "red" : `rgb(${red}, ${green}, ${blue})`;
@@ -350,12 +351,13 @@ describe("server browser snapshot screenshot", () => {
350351
});
351352

352353
it("captures a plain page at the shrunk size with its content in place", async () => {
353-
const { screenshot, pixels } = await screenshotOf(halves(""));
354+
const { screenshot, fullScreenshot, pixels } = await screenshotOf(halves(""));
354355

355356
expect({
356357
size: [screenshot.width, screenshot.height],
358+
fullSize: [fullScreenshot?.width, fullScreenshot?.height],
357359
colors: pixels.map(colorOf),
358-
}).toEqual({ size: [1280, 800], colors: ["blue", "red", "red"] });
360+
}).toEqual({ size: [1280, 800], fullSize: [2560, 1600], colors: ["blue", "red", "red"] });
359361
});
360362

361363
it("shrinks the screenshot of an XML document, whose createElement makes no canvas", async () => {

‎apps/server/src/preview/ServerBrowserPage.ts‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -519,6 +519,7 @@ type SnapshotInput = {
519519
readonly cdp: CDPSession;
520520
/** Headless pages render at the scale the server set; a desktop page at its display's. */
521521
readonly renderScale: number | "display";
522+
readonly fullScreenshot?: boolean | undefined;
522523
readonly consoleEntries: ReadonlyArray<PreviewAutomationConsoleEntry>;
523524
readonly networkEntries: ReadonlyArray<PreviewAutomationNetworkEntry>;
524525
readonly actionTimeline: PreviewAutomationSnapshot["actionTimeline"];
@@ -562,8 +563,13 @@ const capture = async (
562563
"url" | "title" | "loading" | "visibleText" | "interactiveElements"
563564
>
564565
>,
565-
captureViewport(input.cdp, { format: "png" }).then((png) =>
566-
scale < 1 ? shrinkPng(input.cdp, png, scale) : png,
566+
captureViewport(input.cdp, { format: "png" }).then(async (full) =>
567+
scale < 1
568+
? {
569+
data: await shrinkPng(input.cdp, full, scale),
570+
full: input.fullScreenshot ? full : undefined,
571+
}
572+
: { data: full, full: undefined },
567573
),
568574
]);
569575
void pageAndScreenshot.catch(constVoid);
@@ -588,7 +594,7 @@ const capture = async (
588594
void iframe.dispose().catch(constVoid);
589595
if (frame && unread.includes(frame)) throw tooBusy();
590596
}
591-
const [page, data] = await pageAndScreenshot;
597+
const [page, { full, data }] = await pageAndScreenshot;
592598
if (state.generation !== generation) throw pageChanged();
593599
const mask = (text: string) => (dotted ? text.replace(dotted, MASKED_VALUE) : text);
594600
// Redacted before the cut, so a payload crossing it can't leave its start showing, and before
@@ -615,6 +621,7 @@ const capture = async (
615621
data,
616622
...pngSize(data),
617623
},
624+
...(full !== undefined ? { fullScreenshot: { data: full, ...pngSize(full) } } : {}),
618625
};
619626
};
620627

‎packages/contracts/src/previewAutomation.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -552,9 +552,18 @@ export const PreviewAutomationSnapshot = Schema.Struct({
552552
width: Schema.Int,
553553
height: Schema.Int,
554554
}),
555+
/** The capture before `screenshot` was capped for the agent, when requested and larger. */
556+
fullScreenshot: Schema.optional(
557+
Schema.Struct({ data: Schema.String, width: Schema.Int, height: Schema.Int }),
558+
),
555559
});
556560
export type PreviewAutomationSnapshot = typeof PreviewAutomationSnapshot.Type;
557561

562+
export const PreviewAutomationSnapshotInput = Schema.Struct({
563+
fullScreenshot: Schema.optional(Schema.Boolean),
564+
});
565+
export type PreviewAutomationSnapshotInput = typeof PreviewAutomationSnapshotInput.Type;
566+
558567
export const PreviewAutomationRecordingStatus = Schema.Struct({
559568
tabId: PreviewTabId,
560569
recording: Schema.Boolean,

0 commit comments

Comments
 (0)