Skip to content

Commit 6940ca3

Browse files
committed
feat(web): open html render images in the image viewer
A plain click on an image in an agent's html render now opens it in the same expanded image viewer chat images use. The render bootstrap posts a t3/notifications/open-image message for images outside links and buttons when the page did not prevent the click, and the client opens the viewer under the same focus and user-activation gate as links.
1 parent ede3f2d commit 6940ca3

3 files changed

Lines changed: 99 additions & 23 deletions

File tree

‎apps/web/src/components/files/BrowserDocumentFrame.tsx‎

Lines changed: 47 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import {
1313
htmlRenderThemeMessage,
1414
htmlRenderResult,
1515
readHtmlRenderContentHeight,
16+
readHtmlRenderImageRequest,
1617
readHtmlRenderKeyDown,
1718
readHtmlRenderLinkRequest,
1819
} from "@t3tools/shared/htmlRender";
@@ -24,6 +25,8 @@ import {
2425
import { useEffect, useEffectEvent, useLayoutEffect, useMemo, useRef, useState } from "react";
2526

2627
import { useOpenLink } from "~/browser/useOpenLink";
28+
import { ExpandedImageDialog } from "~/components/chat/ExpandedImageDialog";
29+
import type { ExpandedImagePreview } from "~/components/chat/ExpandedImagePreview";
2730
import { useHtmlRenderTheme } from "~/hooks/useHtmlRenderTheme";
2831
import { shortcutConflictKey } from "~/keybindings";
2932
import { useOpenChangeRequestLink } from "~/lib/openPullRequestLink";
@@ -129,6 +132,7 @@ export function HtmlRenderDocument(props: {
129132
const frameRef = useRef<HTMLIFrameElement>(null);
130133
const [src] = useState(() => `${props.src.split("#", 1)[0]}${htmlRenderThemeFragment(theme)}`);
131134
const [loaded, setLoaded] = useState(false);
135+
const [expandedImage, setExpandedImage] = useState<ExpandedImagePreview | null>(null);
132136
const postTheme = () => {
133137
frameRef.current?.contentWindow?.postMessage(htmlRenderThemeMessage(theme), "*");
134138
};
@@ -265,6 +269,25 @@ export function HtmlRenderDocument(props: {
265269
window.addEventListener("message", replayKey);
266270
return () => window.removeEventListener("message", replayKey);
267271
}, []);
272+
// A clicked page image opens in the same viewer as chat images, gated like a link.
273+
useEffect(() => {
274+
const openImage = (event: MessageEvent) => {
275+
const frame = frameRef.current;
276+
const request = readHtmlRenderImageRequest(event.data);
277+
if (
278+
request === undefined ||
279+
frame === null ||
280+
event.source !== frame.contentWindow ||
281+
document.activeElement !== frame ||
282+
navigator.userActivation?.isActive === false
283+
) {
284+
return;
285+
}
286+
setExpandedImage({ images: [{ src: request.src, name: request.name }], index: 0 });
287+
};
288+
window.addEventListener("message", openImage);
289+
return () => window.removeEventListener("message", openImage);
290+
}, []);
268291
const { onContentHeight } = props;
269292
// A page posts its height once per change, so listen from the commit that
270293
// inserts the frame; a passive effect could run after a fast page's first post.
@@ -280,24 +303,29 @@ export function HtmlRenderDocument(props: {
280303
return () => window.removeEventListener("message", resize);
281304
}, [onContentHeight]);
282305
return (
283-
<iframe
284-
ref={frameRef}
285-
src={src}
286-
title={props.title}
287-
// Never allow-same-origin: the opaque origin keeps the page out of the app's session.
288-
sandbox="allow-scripts allow-forms"
289-
loading="lazy"
290-
onLoad={() => {
291-
setLoaded(true);
292-
// Covers a theme change that landed while the page was loading.
293-
postTheme();
294-
postShortcuts();
295-
}}
296-
// A frame whose color scheme differs from its document's paints an opaque
297-
// canvas, so the blank document a frame starts with would flash white in
298-
// dark mode. Once the page is in, its prefers-color-scheme follows the app.
299-
className={cn("border-0 scheme-light", props.className)}
300-
style={loaded ? { colorScheme: theme.appearance } : undefined}
301-
/>
306+
<>
307+
<iframe
308+
ref={frameRef}
309+
src={src}
310+
title={props.title}
311+
// Never allow-same-origin: the opaque origin keeps the page out of the app's session.
312+
sandbox="allow-scripts allow-forms"
313+
loading="lazy"
314+
onLoad={() => {
315+
setLoaded(true);
316+
// Covers a theme change that landed while the page was loading.
317+
postTheme();
318+
postShortcuts();
319+
}}
320+
// A frame whose color scheme differs from its document's paints an opaque
321+
// canvas, so the blank document a frame starts with would flash white in
322+
// dark mode. Once the page is in, its prefers-color-scheme follows the app.
323+
className={cn("border-0 scheme-light", props.className)}
324+
style={loaded ? { colorScheme: theme.appearance } : undefined}
325+
/>
326+
{expandedImage !== null ? (
327+
<ExpandedImageDialog preview={expandedImage} onClose={() => setExpandedImage(null)} />
328+
) : null}
329+
</>
302330
);
303331
}

‎packages/shared/src/htmlRender.test.ts‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import {
99
injectHtmlRenderBootstrap,
1010
htmlRenderThemeMessage,
1111
readHtmlRenderContentHeight,
12+
readHtmlRenderImageRequest,
1213
readHtmlRenderKeyDown,
1314
readHtmlRenderLinkRequest,
1415
readHtmlRenderReference,
@@ -107,6 +108,29 @@ describe("readHtmlRenderLinkRequest", () => {
107108
});
108109
});
109110

111+
describe("readHtmlRenderImageRequest", () => {
112+
it("accepts http(s) and inline image URLs in an open-image notification", () => {
113+
const image = (params: unknown) => ({
114+
jsonrpc: "2.0",
115+
method: "t3/notifications/open-image",
116+
params,
117+
});
118+
expect(
119+
readHtmlRenderImageRequest(image({ src: "data:image/png;base64,AA", name: " Chart " })),
120+
).toEqual({
121+
src: "data:image/png;base64,AA",
122+
name: "Chart",
123+
});
124+
expect(readHtmlRenderImageRequest(image({ src: "https://example.com/a.png" }))).toEqual({
125+
src: "https://example.com/a.png",
126+
name: "Image",
127+
});
128+
expect(readHtmlRenderImageRequest(image({ src: "javascript:alert(1)" }))).toBeUndefined();
129+
expect(readHtmlRenderImageRequest(image({ src: "data:text/html,<p>" }))).toBeUndefined();
130+
expect(readHtmlRenderImageRequest(image({ src: "blob:null/1" }))).toBeUndefined();
131+
});
132+
});
133+
110134
describe("readHtmlRenderKeyDown", () => {
111135
it("reads the key of a keydown notification and nothing else", () => {
112136
const notification = (params: unknown) => ({

‎packages/shared/src/htmlRender.ts‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -280,6 +280,8 @@ const SIZE_CHANGED_METHOD = "ui/notifications/size-changed";
280280
// those on.
281281
const SHORTCUTS_METHOD = "t3/notifications/shortcuts";
282282
const KEY_DOWN_METHOD = "t3/notifications/keydown";
283+
// Not part of MCP Apps: a click on a page image opens it in the app's image viewer.
284+
const OPEN_IMAGE_METHOD = "t3/notifications/open-image";
283285

284286
/** The content height in a framed render's `ui/notifications/size-changed` notification. */
285287
export function readHtmlRenderContentHeight(data: unknown): number | undefined {
@@ -317,6 +319,24 @@ export function readHtmlRenderLinkRequest(data: unknown):
317319
: undefined;
318320
}
319321

322+
/**
323+
* The image a framed render's reader clicked, if `data` is an open-image
324+
* notification for an http(s) or inline image URL. Server-side publishing
325+
* inlines local images, so most arrive as `data:image/` URLs.
326+
*/
327+
export function readHtmlRenderImageRequest(
328+
data: unknown,
329+
): { readonly src: string; readonly name: string } | undefined {
330+
if (typeof data !== "object" || data === null) return undefined;
331+
const { jsonrpc, method, params } = data as Record<string, unknown>;
332+
if (jsonrpc !== "2.0" || method !== OPEN_IMAGE_METHOD) return undefined;
333+
if (typeof params !== "object" || params === null) return undefined;
334+
const { src, name } = params as Record<string, unknown>;
335+
if (typeof src !== "string" || !/^(?:https?:\/\/|data:image\/)/i.test(src)) return undefined;
336+
const label = typeof name === "string" ? name.trim().slice(0, HTML_RENDER_MAX_TITLE_LENGTH) : "";
337+
return { src, name: label || "Image" };
338+
}
339+
320340
/** The key a framed render passed on in a `t3/notifications/keydown` notification. */
321341
export function readHtmlRenderKeyDown(data: unknown):
322342
| {
@@ -398,7 +418,9 @@ const BASE_CSS =
398418
"html{background:var(--background);color:var(--foreground);font-family:var(--font-sans);font-size:14px;line-height:1.5;-webkit-font-smoothing:antialiased;-webkit-text-size-adjust:100%;scrollbar-width:none}" +
399419
"html::-webkit-scrollbar{display:none}body{margin:0}code,kbd,pre,samp{font-family:var(--font-mono)}" +
400420
// Focus outlines paint inside the element, so the frame edge never clips them.
401-
":where(:focus-visible){outline:2px solid var(--ring)}:focus-visible{outline-width:2px!important;outline-offset:-2px!important}";
421+
":where(:focus-visible){outline:2px solid var(--ring)}:focus-visible{outline-width:2px!important;outline-offset:-2px!important}" +
422+
// An image outside a link or button opens in the app's viewer, as chat images do.
423+
":where(img:not(a[href] img,button img)){cursor:zoom-in}";
402424

403425
function rootRule(theme: HtmlRenderTheme): string {
404426
const declarations = Object.entries(theme.variables)
@@ -413,14 +435,16 @@ function rootRule(theme: HtmlRenderTheme): string {
413435
// then drops the fragment so a page's own hash routing never sees it. A link
414436
// the reader clicks to another page never replaces the page inside the thread:
415437
// a framed page asks its client to open it, and a top-level page (mobile)
416-
// opens it as a new window, which the client sends to the browser. A framed
417-
// page also reports its content height, measured as the server measures it,
438+
// opens it as a new window, which the client sends to the browser. A plain
439+
// left click on a framed page's image, outside a link or button and not
440+
// prevented by any of the page's listeners, asks the client to open the image
441+
// in its viewer. A framed page also reports its content height, measured as the server measures it,
418442
// so its client can fit the frame to the page. A chord the client named as an
419443
// app shortcut is passed on rather than left to the browser, unless a text
420444
// field has focus or the page handled it: the deciding listener is added while
421445
// the key dispatches, so it runs after every listener the page has. Keys
422446
// match as resolveEventKeys reads them, so the client accepts what it claims.
423-
const BOOTSTRAP_SCRIPT = `(function(){var s=document.getElementById("t3-theme"),n=0;if(!s)return;var b=${JSON.stringify(BASE_CSS)};function a(t){if(!t||typeof t!=="object"||!t.variables||typeof t.variables!=="object")return;var c=":root{color-scheme:"+(t.appearance==="light"?"light":"dark")+";";for(var k in t.variables){if(/^--[a-z0-9-]+$/.test(k))c+=k+":"+String(t.variables[k]).replace(/[;{}<>]/g,"")+";";}s.textContent=c+"}"+b;}try{var m=/[#&]${THEME_FRAGMENT_KEY}=([^&]*)/.exec(location.hash);if(m){a(JSON.parse(decodeURIComponent(m[1])));history.replaceState(history.state,"",location.pathname+location.search);}}catch(e){}window.addEventListener("message",function(e){var d=e.data,p=d&&d.params;if(d&&d.jsonrpc==="2.0"&&d.method===${JSON.stringify(HOST_CONTEXT_CHANGED_METHOD)}&&p&&p.styles)a({appearance:p.theme,variables:p.styles.variables});});document.addEventListener("click",function(e){var l=e.isTrusted?e.composedPath().find(function(t){return t&&t.matches&&t.matches("a[href]");}):null,u;if(!l)return;try{u=new URL(l.getAttribute("href"),document.baseURI);}catch(x){return;}if(!/^https?:$/.test(u.protocol)||u.href.split("#")[0]===location.href.split("#")[0])return;if(window.parent!==window){e.preventDefault();window.parent.postMessage({jsonrpc:"2.0",id:"t3-link-"+(++n),method:${JSON.stringify(OPEN_LINK_METHOD)},params:{url:u.href,metaKey:e.metaKey,ctrlKey:e.ctrlKey}},"*");}else{l.setAttribute("target","_blank");l.setAttribute("rel","noopener");}},true);if(window.parent!==window){var h,o,z=function(){var r=document.documentElement,v=Math.ceil(r.scrollHeight>r.clientHeight?r.scrollHeight:r.getBoundingClientRect().height);if(v===h)return;h=v;window.parent.postMessage({jsonrpc:"2.0",method:${JSON.stringify(SIZE_CHANGED_METHOD)},params:{height:v}},"*");};if(window.ResizeObserver){o=new ResizeObserver(z);o.observe(document.documentElement);}document.addEventListener("DOMContentLoaded",function(){if(o&&document.body)o.observe(document.body);z();});window.addEventListener("load",z);var K=[],B=[],C=${JSON.stringify(EVENT_CODE_SHORTCUT_KEYS)};window.addEventListener("message",function(e){var d=e.data,p=d&&d.params;if(e.source===window.parent&&d&&d.jsonrpc==="2.0"&&d.method===${JSON.stringify(SHORTCUTS_METHOD)}&&p&&Array.isArray(p.shortcuts)&&Array.isArray(p.blocked)){K=p.shortcuts;B=p.blocked;}});window.addEventListener("keydown",function(e){var t=e.composedPath()[0],c=e.code||"",l=e.key.toLowerCase(),x=/^[a-z]$/.test(l)?[l]:[l,/^Key[A-Z]$/.test(c)?c.slice(3).toLowerCase():"",C.hasOwnProperty(c)?C[c]:""],f,m=function(s){return s&&s.metaKey===e.metaKey&&s.ctrlKey===e.ctrlKey&&s.shiftKey===e.shiftKey&&s.altKey===e.altKey&&x.indexOf(s.key)>-1;};if(!e.isTrusted||e.isComposing||e.getModifierState("AltGraph")&&!/^[a-z0-9]$/i.test(e.key)||t&&(t.isContentEditable||/^(?:INPUT|TEXTAREA|SELECT)$/.test(t.tagName))||!K.some(m)||B.some(m))return;f=function(v){window.removeEventListener("keydown",f);if(v!==e||e.defaultPrevented)return;e.preventDefault();window.parent.postMessage({jsonrpc:"2.0",method:${JSON.stringify(KEY_DOWN_METHOD)},params:{key:e.key,code:e.code,metaKey:e.metaKey,ctrlKey:e.ctrlKey,shiftKey:e.shiftKey,altKey:e.altKey,repeat:e.repeat}},"*");};window.addEventListener("keydown",f);setTimeout(function(){window.removeEventListener("keydown",f);});},true);}})();`;
447+
const BOOTSTRAP_SCRIPT = `(function(){var s=document.getElementById("t3-theme"),n=0;if(!s)return;var b=${JSON.stringify(BASE_CSS)};function a(t){if(!t||typeof t!=="object"||!t.variables||typeof t.variables!=="object")return;var c=":root{color-scheme:"+(t.appearance==="light"?"light":"dark")+";";for(var k in t.variables){if(/^--[a-z0-9-]+$/.test(k))c+=k+":"+String(t.variables[k]).replace(/[;{}<>]/g,"")+";";}s.textContent=c+"}"+b;}try{var m=/[#&]${THEME_FRAGMENT_KEY}=([^&]*)/.exec(location.hash);if(m){a(JSON.parse(decodeURIComponent(m[1])));history.replaceState(history.state,"",location.pathname+location.search);}}catch(e){}window.addEventListener("message",function(e){var d=e.data,p=d&&d.params;if(d&&d.jsonrpc==="2.0"&&d.method===${JSON.stringify(HOST_CONTEXT_CHANGED_METHOD)}&&p&&p.styles)a({appearance:p.theme,variables:p.styles.variables});});document.addEventListener("click",function(e){var l=e.isTrusted?e.composedPath().find(function(t){return t&&t.matches&&t.matches("a[href]");}):null,u;if(!l)return;try{u=new URL(l.getAttribute("href"),document.baseURI);}catch(x){return;}if(!/^https?:$/.test(u.protocol)||u.href.split("#")[0]===location.href.split("#")[0])return;if(window.parent!==window){e.preventDefault();window.parent.postMessage({jsonrpc:"2.0",id:"t3-link-"+(++n),method:${JSON.stringify(OPEN_LINK_METHOD)},params:{url:u.href,metaKey:e.metaKey,ctrlKey:e.ctrlKey}},"*");}else{l.setAttribute("target","_blank");l.setAttribute("rel","noopener");}},true);if(window.parent!==window){var h,o,z=function(){var r=document.documentElement,v=Math.ceil(r.scrollHeight>r.clientHeight?r.scrollHeight:r.getBoundingClientRect().height);if(v===h)return;h=v;window.parent.postMessage({jsonrpc:"2.0",method:${JSON.stringify(SIZE_CHANGED_METHOD)},params:{height:v}},"*");};if(window.ResizeObserver){o=new ResizeObserver(z);o.observe(document.documentElement);}document.addEventListener("DOMContentLoaded",function(){if(o&&document.body)o.observe(document.body);z();});window.addEventListener("load",z);window.addEventListener("click",function(e){var t=e.composedPath()[0],u;if(!e.isTrusted||e.defaultPrevented||e.button!==0||e.metaKey||e.ctrlKey||e.shiftKey||e.altKey||!t||t.tagName!=="IMG"||t.closest("a[href],button"))return;u=t.currentSrc||t.src;if(u)setTimeout(function(){if(!e.defaultPrevented)window.parent.postMessage({jsonrpc:"2.0",method:${JSON.stringify(OPEN_IMAGE_METHOD)},params:{src:u,name:t.alt||t.title||""}},"*");});});var K=[],B=[],C=${JSON.stringify(EVENT_CODE_SHORTCUT_KEYS)};window.addEventListener("message",function(e){var d=e.data,p=d&&d.params;if(e.source===window.parent&&d&&d.jsonrpc==="2.0"&&d.method===${JSON.stringify(SHORTCUTS_METHOD)}&&p&&Array.isArray(p.shortcuts)&&Array.isArray(p.blocked)){K=p.shortcuts;B=p.blocked;}});window.addEventListener("keydown",function(e){var t=e.composedPath()[0],c=e.code||"",l=e.key.toLowerCase(),x=/^[a-z]$/.test(l)?[l]:[l,/^Key[A-Z]$/.test(c)?c.slice(3).toLowerCase():"",C.hasOwnProperty(c)?C[c]:""],f,m=function(s){return s&&s.metaKey===e.metaKey&&s.ctrlKey===e.ctrlKey&&s.shiftKey===e.shiftKey&&s.altKey===e.altKey&&x.indexOf(s.key)>-1;};if(!e.isTrusted||e.isComposing||e.getModifierState("AltGraph")&&!/^[a-z0-9]$/i.test(e.key)||t&&(t.isContentEditable||/^(?:INPUT|TEXTAREA|SELECT)$/.test(t.tagName))||!K.some(m)||B.some(m))return;f=function(v){window.removeEventListener("keydown",f);if(v!==e||e.defaultPrevented)return;e.preventDefault();window.parent.postMessage({jsonrpc:"2.0",method:${JSON.stringify(KEY_DOWN_METHOD)},params:{key:e.key,code:e.code,metaKey:e.metaKey,ctrlKey:e.ctrlKey,shiftKey:e.shiftKey,altKey:e.altKey,repeat:e.repeat}},"*");};window.addEventListener("keydown",f);setTimeout(function(){window.removeEventListener("keydown",f);});},true);}})();`;
424448

425449
function bootstrapMarkup(markup: string): string {
426450
const dark = htmlRenderTheme(T3_CODE_DARK_THEME_COLORS, "dark");

0 commit comments

Comments
 (0)