Skip to content

Commit bf00290

Browse files
committed
fix(chat): render HTML in tool call details
1 parent 5d4c202 commit bf00290

18 files changed

Lines changed: 570 additions & 86 deletions

File tree

‎apps/mobile/src/features/threads/HtmlRenderWebView.tsx‎

Lines changed: 24 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
import { AppActivityIndicator } from "../../components/ui/AppActivityIndicator";
22
import { useNavigation } from "@react-navigation/native";
3-
import type { EnvironmentId, ThreadId } from "@t3tools/contracts";
3+
import type { AssetResource, EnvironmentId, ThreadId } from "@t3tools/contracts";
44
import {
5+
HTML_RENDER_COLUMN_WIDTH,
56
htmlRenderFileName,
67
htmlRenderFrameHeight,
78
htmlRenderThemeFragment,
@@ -180,23 +181,31 @@ export function HtmlRenderWebView(props: {
180181
export function ThreadHtmlRender(props: {
181182
readonly environmentId: EnvironmentId;
182183
readonly threadId: ThreadId;
183-
readonly render: HtmlRenderReference;
184-
readonly frameWidth: number;
185-
readonly iconColor: ColorValue;
184+
readonly render: Pick<HtmlRenderReference, "title" | "height" | "heights"> &
185+
(
186+
| { readonly attachmentId: string }
187+
| { readonly resource: Extract<AssetResource, { readonly _tag: "tool-input-html" }> }
188+
);
189+
readonly frameWidth?: number;
190+
readonly iconColor?: ColorValue;
186191
}) {
187192
const navigation = useNavigation();
188-
const { attachmentId, title } = props.render;
189-
const height = htmlRenderFrameHeight(props.render, props.frameWidth);
193+
const { title } = props.render;
194+
const attachmentId = "attachmentId" in props.render ? props.render.attachmentId : null;
195+
const height = htmlRenderFrameHeight(props.render, props.frameWidth ?? HTML_RENDER_COLUMN_WIDTH);
190196
const fileName = htmlRenderFileName(title);
191197
const resource = useMemo(
192-
() => ({
193-
_tag: "attachment" as const,
194-
attachmentId,
195-
fileName,
196-
mimeType: "text/html",
197-
disposition: "inline" as const,
198-
}),
199-
[attachmentId, fileName],
198+
() =>
199+
"resource" in props.render
200+
? props.render.resource
201+
: {
202+
_tag: "attachment" as const,
203+
attachmentId: props.render.attachmentId,
204+
fileName,
205+
mimeType: "text/html",
206+
disposition: "inline" as const,
207+
},
208+
[props.render, fileName],
200209
);
201210
const asset = useAssetUrlState(props.environmentId, resource);
202211
const refresh = useRefreshAssetUrl(props.environmentId, resource);
@@ -248,7 +257,7 @@ export function ThreadHtmlRender(props: {
248257
<AppActivityIndicator />
249258
</View>
250259
)}
251-
{uri !== null && !failed ? (
260+
{uri !== null && !failed && attachmentId !== null ? (
252261
<Pressable
253262
accessibilityRole="button"
254263
accessibilityLabel={`Open ${title}`}

‎apps/mobile/src/features/threads/thread-work-log.tsx‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ import {
7272
} from "../../lib/threadActivity";
7373
import {
7474
toolCallLines,
75+
turnItemDetailRevision,
7576
turnItemOutputImages,
7677
turnItemOutputText,
7778
} from "@t3tools/client-runtime/work-log/item-detail";
@@ -93,6 +94,7 @@ import type { MarkdownImageRenderer } from "../../native/SelectableMarkdownText"
9394
import type { FilePreviewSource } from "../../components/FilePreviewModal";
9495
import { FilePreviewGallery } from "../../components/FilePreviewGallery";
9596
import { ThreadMarkdownImage } from "./ThreadMarkdownImage";
97+
import { ThreadHtmlRender } from "./HtmlRenderWebView";
9698
import Animated, {
9799
FadeIn,
98100
FadeOut,
@@ -970,7 +972,12 @@ const ThreadWorkLogRow = function ThreadWorkLogRow(
970972
expanded && !isRead && shownItem.type === "command_execution"
971973
? toolCallLines({ command: shownItem.input })
972974
: expanded && !isRead && shownItem.type === "dynamic_tool"
973-
? toolCallLines({ args: shownItem.input })
975+
? toolCallLines({
976+
args: shownItem.input,
977+
toolName: shownItem.toolName,
978+
status: shownItem.status,
979+
output: shownItem.output,
980+
})
974981
: expanded && shownItem.type === "file_search"
975982
? toolCallLines({ args: { pattern: shownItem.pattern } })
976983
: expanded && shownItem.type === "web_search"
@@ -1168,6 +1175,23 @@ const ThreadWorkLogRow = function ThreadWorkLogRow(
11681175
layout={WORK_LOG_LAYOUT_TRANSITION}
11691176
className={reasoning ? "ml-7 py-1" : "pb-1 pt-0.5"}
11701177
>
1178+
{call?.html ? (
1179+
<ThreadHtmlRender
1180+
key={`${props.environmentId}:${row.projectedItem.sourceThreadId}:${row.projectedItem.sourceItemId}:${turnItemDetailRevision(shownItem)}`}
1181+
environmentId={props.environmentId}
1182+
threadId={row.projectedItem.sourceThreadId}
1183+
render={{
1184+
title: "HTML preview",
1185+
height: 400,
1186+
resource: {
1187+
_tag: "tool-input-html",
1188+
threadId: row.projectedItem.sourceThreadId,
1189+
itemId: row.projectedItem.sourceItemId,
1190+
revision: turnItemDetailRevision(shownItem),
1191+
},
1192+
}}
1193+
/>
1194+
) : null}
11711195
{row.workEntry.questionAnswer ? (
11721196
<QuestionAnswerHistory
11731197
environmentId={props.environmentId}

‎apps/server/src/assets/AssetAccess.test.ts‎

Lines changed: 85 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -99,16 +99,37 @@ const oversizedScreenshotItem = {
9999
},
100100
};
101101

102+
const htmlItem = {
103+
...screenshotItem,
104+
id: TurnItemId.make("tool-html"),
105+
toolName: "mcp__t3-code__html_preview",
106+
input: {
107+
html:
108+
'<!doctype html><script src="https://cdn.example/chart.js"></script><main>' +
109+
"x".repeat(300_000) +
110+
"Complete document</main>",
111+
},
112+
};
113+
const otherHtmlItem = {
114+
...htmlItem,
115+
id: TurnItemId.make("other-html"),
116+
toolName: "other.html_preview",
117+
};
118+
102119
const layerTest = Layer.mergeAll(
103120
NodeHttpPlatform.layer,
104121
Layer.mock(Orchestrator.OrchestratorV2)({
105122
getTurnItem: ({ itemId }) =>
106123
Effect.succeed(
107124
itemId === screenshotItem.id
108125
? screenshotItem
109-
: itemId === oversizedScreenshotItem.id
110-
? oversizedScreenshotItem
111-
: null,
126+
: itemId === htmlItem.id
127+
? htmlItem
128+
: itemId === otherHtmlItem.id
129+
? otherHtmlItem
130+
: itemId === oversizedScreenshotItem.id
131+
? oversizedScreenshotItem
132+
: null,
112133
),
113134
}),
114135
layerConfig,
@@ -122,6 +143,67 @@ const layerTest = Layer.mergeAll(
122143
).pipe(Layer.provideMerge(NodeServices.layer));
123144

124145
describe("AssetAccess", () => {
146+
it.effect("inlines local images in tool HTML with the existing page preparation", () =>
147+
Effect.gen(function* () {
148+
const fs = yield* FileSystem.FileSystem;
149+
const path = yield* Path.Path;
150+
const directory = yield* fs.makeTempDirectoryScoped({ prefix: "t3-tool-html-" });
151+
const image = path.join(directory, "shot.png");
152+
yield* fs.writeFile(image, screenshotPng);
153+
const orchestrator = yield* Orchestrator.OrchestratorV2;
154+
yield* Effect.gen(function* () {
155+
const resource = {
156+
_tag: "tool-input-html" as const,
157+
threadId: htmlItem.threadId,
158+
itemId: htmlItem.id,
159+
revision: "local-image",
160+
};
161+
const result = yield* issueAssetUrl({ resource });
162+
const suffix = result.relativeUrl.slice(`${ASSET_ROUTE_PREFIX}/`.length);
163+
const separator = suffix.indexOf("/");
164+
const asset = yield* resolveAsset(suffix.slice(0, separator), suffix.slice(separator + 1));
165+
expect(asset?.kind).toBe("bytes");
166+
if (asset?.kind !== "bytes") return;
167+
expect(new TextDecoder().decode(asset.bytes)).toContain(
168+
`<img src="data:image/png;base64,${Buffer.from(screenshotPng).toString("base64")}">`,
169+
);
170+
}).pipe(
171+
Effect.provideService(Orchestrator.OrchestratorV2, {
172+
...orchestrator,
173+
getTurnItem: () =>
174+
Effect.succeed({ ...htmlItem, input: { html: `<img src="${image}">` } }),
175+
}),
176+
);
177+
}).pipe(Effect.provide(layerTest)),
178+
);
179+
180+
it.effect("serves the complete HTML tool input through a signed document URL", () =>
181+
Effect.gen(function* () {
182+
const resource = {
183+
_tag: "tool-input-html" as const,
184+
threadId: htmlItem.threadId,
185+
itemId: htmlItem.id,
186+
revision: "first",
187+
};
188+
const result = yield* issueAssetUrl({ resource });
189+
const suffix = result.relativeUrl.slice(`${ASSET_ROUTE_PREFIX}/`.length);
190+
const separator = suffix.indexOf("/");
191+
const asset = yield* resolveAsset(suffix.slice(0, separator), suffix.slice(separator + 1));
192+
expect(asset?.kind).toBe("bytes");
193+
if (asset?.kind !== "bytes") return;
194+
expect(asset.mimeType).toBe("text/html");
195+
const html = new TextDecoder().decode(asset.bytes);
196+
expect(html).toContain('<script src="https://cdn.example/chart.js"></script>');
197+
expect(html).toContain("Complete document</main>");
198+
expect(html).toContain("t3-theme");
199+
const revised = yield* issueAssetUrl({ resource: { ...resource, revision: "second" } });
200+
expect(revised.relativeUrl).not.toBe(result.relativeUrl);
201+
for (const itemId of [screenshotItem.id, otherHtmlItem.id, TurnItemId.make("missing")]) {
202+
const error = yield* issueAssetUrl({ resource: { ...resource, itemId } }).pipe(Effect.flip);
203+
expect(error._tag).toBe("AssetWorkspaceAssetNotFoundError");
204+
}
205+
}).pipe(Effect.provide(layerTest)),
206+
);
125207
describe("readMcpAppHtml", () => {
126208
it.effect("accepts snapshots at the stored HTML byte limit", () =>
127209
Effect.gen(function* () {

‎apps/server/src/assets/AssetAccess.ts‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import {
3232
} from "@t3tools/shared/imageDimensions";
3333
import { githubMediaFetchUrl, githubMediaFileName } from "@t3tools/shared/githubMedia";
3434
import { MCP_APP_MAX_HTML_BYTES } from "@t3tools/shared/mcpApp";
35+
import { HTML_RENDER_MAX_SOURCE_LENGTH, isHtmlRenderTool } from "@t3tools/shared/htmlRender";
3536
import {
3637
isProjectFaviconPath,
3738
PROJECT_FAVICON_FALLBACK_MARKER,
@@ -58,6 +59,7 @@ import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
5859
import { parseAttachmentFileExtension, resolveAttachmentPathById } from "../attachmentStore.ts";
5960
import * as ServerConfig from "../config.ts";
6061
import * as Orchestrator from "../orchestration-v2/Orchestrator.ts";
62+
import * as HtmlRender from "../htmlRender/HtmlRender.ts";
6163
import { expandHomePath } from "@t3tools/provider-core/server/pathExpansion";
6264
import * as ProjectFaviconResolver from "../project/ProjectFaviconResolver.ts";
6365
import * as WorkspacePaths from "../workspace/WorkspacePaths.ts";
@@ -156,6 +158,14 @@ const AssetClaimsSchema = Schema.Union([
156158
index: Schema.Number,
157159
expiresAt: Schema.Number,
158160
}),
161+
Schema.Struct({
162+
version: Schema.Literal(1),
163+
kind: Schema.Literal("tool-input-html"),
164+
threadId: ThreadId,
165+
itemId: TurnItemId,
166+
revision: Schema.String,
167+
expiresAt: Schema.Number,
168+
}),
159169
Schema.Struct({
160170
version: Schema.Literal(1),
161171
kind: Schema.Literal("github-media"),
@@ -251,6 +261,24 @@ const readToolOutputImage = Effect.fn("AssetAccess.readToolOutputImage")(functio
251261
: { mimeType: image.mimeType, bytes: Buffer.from(image.data, "base64") };
252262
});
253263

264+
const readToolInputHtml = Effect.fn("AssetAccess.readToolInputHtml")(function* (input: {
265+
readonly threadId: ThreadId;
266+
readonly itemId: TurnItemId;
267+
}) {
268+
const orchestrator = yield* Orchestrator.OrchestratorV2;
269+
const item = yield* orchestrator.getTurnItem(input);
270+
if (item?.type !== "dynamic_tool" || !isHtmlRenderTool(item.toolName)) return null;
271+
const args = item.input;
272+
return typeof args === "object" &&
273+
args !== null &&
274+
"html" in args &&
275+
typeof args.html === "string" &&
276+
args.html.length <= HTML_RENDER_MAX_SOURCE_LENGTH &&
277+
args.html.trim()
278+
? args.html
279+
: null;
280+
});
281+
254282
/** Serves an `.icns` favicon as its embedded PNG, since browsers cannot draw it. */
255283
const projectFaviconAsset = Effect.fn("AssetAccess.projectFaviconAsset")(function* (
256284
filePath: string,
@@ -747,6 +775,25 @@ export const issueAssetUrl = Effect.fn("AssetAccess.issueAssetUrl")(function* (i
747775
}
748776
break;
749777
}
778+
case "tool-input-html": {
779+
const html = yield* readToolInputHtml(input.resource).pipe(
780+
Effect.mapError(
781+
(cause) => new AssetWorkspaceAssetInspectionError({ resource: input.resource, cause }),
782+
),
783+
);
784+
if (html === null)
785+
return yield* new AssetWorkspaceAssetNotFoundError({ resource: input.resource });
786+
claims = {
787+
version: 1,
788+
kind: "tool-input-html",
789+
threadId: input.resource.threadId,
790+
itemId: input.resource.itemId,
791+
revision: input.resource.revision,
792+
expiresAt,
793+
};
794+
fileName = "preview.html";
795+
break;
796+
}
750797
case "tool-output-image": {
751798
const image = yield* readToolOutputImage(input.resource).pipe(
752799
Effect.mapError(
@@ -923,6 +970,35 @@ export const resolveAsset = Effect.fn("AssetAccess.resolveAsset")(function* (
923970
return image ? ({ kind: "bytes", ...image } satisfies ResolvedAsset) : null;
924971
}
925972

973+
if (claims.kind === "tool-input-html") {
974+
const html = yield* readToolInputHtml(claims).pipe(
975+
Effect.tapError((cause) =>
976+
Effect.logError("Failed to read tool input HTML.", {
977+
threadId: claims.threadId,
978+
itemId: claims.itemId,
979+
cause,
980+
}),
981+
),
982+
Effect.orElseSucceed(() => null),
983+
);
984+
const document =
985+
html === null
986+
? null
987+
: yield* HtmlRender.prepareDocument(html).pipe(
988+
Effect.tapError((cause) =>
989+
Effect.logWarning("Failed to prepare tool input HTML.", { cause }),
990+
),
991+
Effect.orElseSucceed(() => null),
992+
);
993+
return document === null
994+
? null
995+
: ({
996+
kind: "bytes",
997+
mimeType: "text/html",
998+
bytes: new TextEncoder().encode(document.html),
999+
} satisfies ResolvedAsset);
1000+
}
1001+
9261002
if (claims.kind === "native-app-icon") {
9271003
const nativeAppIconResolver = yield* NativeAppIconResolver.NativeAppIconResolver;
9281004
const iconPath = yield* nativeAppIconResolver.resolve(claims.app);

‎apps/server/src/htmlRender/HtmlRender.ts‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -327,6 +327,12 @@ const inlineLocalImages = Effect.fn("HtmlRender.inlineLocalImages")(function* (h
327327
};
328328
});
329329

330+
export const prepareDocument = Effect.fn("HtmlRender.prepareDocument")(function* (html: string) {
331+
yield* rejectViewportHeight(html);
332+
// Scan the head before local images expand into data URLs.
333+
return yield* inlineLocalImages(injectHtmlRenderBootstrap(html));
334+
});
335+
330336
const MIN_PREVIEW_WIDTH = 240;
331337
const MAX_PREVIEW_WIDTH = 1_600;
332338
const MAX_CONCURRENT_BROWSERS = 2;
@@ -404,13 +410,8 @@ const make = Effect.gen(function* () {
404410
),
405411
);
406412

407-
// The bootstrap goes in first so its head scan never runs over inlined image data.
408-
const inline = (html: string) =>
409-
inlineLocalImages(injectHtmlRenderBootstrap(html)).pipe(Effect.provideContext(services));
410-
411413
const prepare = Effect.fn("HtmlRender.prepare")(function* (html: string) {
412-
yield* rejectViewportHeight(html);
413-
const inlined = yield* inline(html);
414+
const inlined = yield* prepareDocument(html).pipe(Effect.provideContext(services));
414415
if (inlined.missing.length > 0) {
415416
return yield* new HtmlRenderImagesNotFoundError({ paths: inlined.missing });
416417
}
@@ -477,8 +478,7 @@ const make = Effect.gen(function* () {
477478
Math.max(MIN_PREVIEW_WIDTH, Math.round(input.width ?? HTML_RENDER_COLUMN_WIDTH)),
478479
);
479480
const appearance = input.appearance ?? "dark";
480-
yield* rejectViewportHeight(input.html);
481-
const inlined = yield* inline(input.html);
481+
const inlined = yield* prepareDocument(input.html).pipe(Effect.provideContext(services));
482482
const executable = yield* previewBrowser.executable;
483483
const theme = htmlRenderTheme(
484484
appearance === "light" ? T3_CODE_LIGHT_THEME_COLORS : T3_CODE_DARK_THEME_COLORS,

‎apps/server/src/http.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -429,7 +429,7 @@ export const layerAssetRoute = HttpRouter.add(
429429
if (asset.kind === "bytes") {
430430
return HttpServerResponse.uint8Array(asset.bytes, {
431431
contentType: asset.mimeType,
432-
headers: { "cache-control": "private, max-age=3600", "x-content-type-options": "nosniff" },
432+
headers: assetResponseHeaders("", { mimeType: asset.mimeType }),
433433
});
434434
}
435435
if (asset.kind === "github-media") {

0 commit comments

Comments
 (0)