Skip to content

Commit 6c69534

Browse files
authored
fix(preview): render website favicons for browser tool activity (pingdotgg#11032)
1 parent 02297e3 commit 6c69534

9 files changed

Lines changed: 320 additions & 78 deletions

File tree

‎apps/server/src/mcp/McpHttpServer.test.ts‎

Lines changed: 33 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -250,7 +250,10 @@ it.effect.each([
250250
const { accessibilityTree: _tree, ...boundedMetadata } = metadata;
251251
expect(snapshot.isError).toBe(false);
252252
expect(snapshot.structuredContent).toEqual(metadata);
253-
const [text, ...rest] = snapshot.content;
253+
const [identity, text, ...rest] = snapshot.content;
254+
expect(identity?.type === "text" ? decodeJsonText(identity.text) : null).toEqual({
255+
url: page.url,
256+
});
254257
expect(text?.type === "text" ? decodeJsonText(text.text) : null).toEqual(boundedMetadata);
255258
expect(rest).toEqual([
256259
{
@@ -279,7 +282,12 @@ it.effect.each([
279282
Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),
280283
Effect.provideService(McpSchema.McpServerClient, client),
281284
);
282-
expect(nextDefault.content.map((content) => content.type)).toEqual(["text", "text", "image"]);
285+
expect(nextDefault.content.map((content) => content.type)).toEqual([
286+
"text",
287+
"text",
288+
"text",
289+
"image",
290+
]);
283291
expect(nextDefault.structuredContent).toEqual({ ...page, title: "Snapshot 7", screenshot });
284292
expect(requests).toBe(7);
285293
}),
@@ -329,7 +337,7 @@ it.effect("saves the snapshot PNG on request and reports its path", () =>
329337
/^browser-screenshot-example-test-[0-9a-z]+-[0-9a-f]{8}\.png$/,
330338
);
331339
expect(Buffer.from(yield* fileSystem.readFile(screenshotPath!)).toString()).toBe("png");
332-
const text = snapshot.content.find((content) => content.type === "text");
340+
const [, text] = snapshot.content;
333341
expect(text?.type === "text" ? text.text : "").toContain(screenshotPath);
334342

335343
const unsaved = yield* callSnapshot({});
@@ -429,7 +437,10 @@ it.effect("keeps the snapshot text under the agent's output ceiling", () =>
429437
const snapshot = yield* callSnapshot({ includeImage: false });
430438

431439
expect(snapshot.isError).toBe(false);
432-
const [text, notice] = snapshot.content;
440+
const [identity, text, notice] = snapshot.content;
441+
expect(identity?.type === "text" ? decodeJsonText(identity.text) : null).toEqual({
442+
url: oversized.url,
443+
});
433444
expect(text?.type).toBe("text");
434445
const body = text?.type === "text" ? text.text : "";
435446
expect(Buffer.byteLength(body, "utf8")).toBeLessThanOrEqual(
@@ -471,7 +482,7 @@ it.effect("bounds the snapshot text even when nothing but logs and the title are
471482

472483
const snapshot = yield* callSnapshot({ includeImage: false });
473484

474-
const [text] = snapshot.content;
485+
const [, text] = snapshot.content;
475486
const body = text?.type === "text" ? text.text : "";
476487
expect(Buffer.byteLength(body, "utf8")).toBeLessThanOrEqual(
477488
McpHttpServer.MAX_SNAPSHOT_TEXT_BYTES,
@@ -482,7 +493,7 @@ it.effect("bounds the snapshot text even when nothing but logs and the title are
482493
};
483494
expect(parsed.title.length).toBe(2_049);
484495
expect(parsed.consoleEntries[0]?.text.length).toBe(501);
485-
const notice = snapshot.content[1];
496+
const notice = snapshot.content[2];
486497
const noticeText = notice?.type === "text" ? notice.text : "";
487498
expect(noticeText).toContain("url or title after 2048 characters");
488499
expect(noticeText).toContain("console entries text after 500 characters");
@@ -533,7 +544,7 @@ it.effect("sheds log entries before locators when every list is full", () =>
533544

534545
const snapshot = yield* callSnapshot({ includeImage: false });
535546

536-
const [text, notice] = snapshot.content;
547+
const [, text, notice] = snapshot.content;
537548
const body = text?.type === "text" ? text.text : "";
538549
expect(Buffer.byteLength(body, "utf8")).toBeLessThanOrEqual(
539550
McpHttpServer.MAX_SNAPSHOT_TEXT_BYTES,
@@ -615,6 +626,10 @@ it.effect("registers annotated tools and preserves authenticated request context
615626
Effect.gen(function* () {
616627
const server = yield* McpServer.McpServer;
617628
const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker;
629+
const toolIcon = {
630+
_tag: "website" as const,
631+
pageUrl: "http://example.test/",
632+
};
618633
const routedRequests: Array<{
619634
readonly operation: string;
620635
readonly tabId?: string | undefined;
@@ -664,7 +679,7 @@ it.effect("registers annotated tools and preserves authenticated request context
664679
expect(clickTool?.tool.annotations?.readOnlyHint).toBe(false);
665680
expect(clickTool?.tool.annotations?.destructiveHint).toBe(true);
666681
expect(clickTool?.tool.annotations?.openWorldHint).toBe(true);
667-
expect(clickTool?.tool.outputSchema).toEqual({
682+
expect(clickTool?.tool.outputSchema).toMatchObject({
668683
type: "object",
669684
additionalProperties: false,
670685
description: "The preview action completed successfully.",
@@ -721,10 +736,12 @@ it.effect("registers annotated tools and preserves authenticated request context
721736
Effect.provideService(McpSchema.McpServerClient, client),
722737
);
723738
expect(evaluated.isError).toBe(false);
724-
expect(evaluated.structuredContent).toEqual({ value: ["Connect", "Continue"] });
725-
expect(evaluated.content).toEqual([
726-
{ type: "text", text: '{"value":["Connect","Continue"]}' },
727-
]);
739+
expect(evaluated.structuredContent).toEqual({ value: ["Connect", "Continue"], toolIcon });
740+
const evaluatedText = evaluated.content[0];
741+
expect(evaluatedText?.type === "text" ? decodeJsonText(evaluatedText.text) : null).toEqual({
742+
toolIcon,
743+
value: ["Connect", "Continue"],
744+
});
728745

729746
const actionRequests = [
730747
{ name: "preview_click", arguments: { x: 10, y: 10 } },
@@ -741,8 +758,10 @@ it.effect("registers annotated tools and preserves authenticated request context
741758
Effect.provideService(McpSchema.McpServerClient, client),
742759
);
743760
expect(result.isError).toBe(false);
744-
expect(result.structuredContent).toEqual({});
745-
expect(result.content).toEqual([{ type: "text", text: "{}" }]);
761+
expect(result.structuredContent).toEqual({ toolIcon });
762+
expect(routedRequests.at(-1)?.operation).toBe("status");
763+
const text = result.content[0];
764+
expect(text?.type === "text" ? decodeJsonText(text.text) : null).toEqual({ toolIcon });
746765
}
747766
}),
748767
).pipe(Effect.provide(TestLayer)),

‎apps/server/src/mcp/McpHttpServer.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -410,6 +410,13 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
410410
isError: false,
411411
structuredContent: metadata,
412412
content: [
413+
// Keep the page identity readable even if a provider truncates the snapshot.
414+
{
415+
type: "text",
416+
text: encodeJsonText({
417+
url: cutText(snapshot.url, MAX_SNAPSHOT_IDENTIFIER_CHARS),
418+
}),
419+
},
413420
{ type: "text", text: bounded.text },
414421
...(bounded.omitted.length === 0
415422
? []

‎apps/server/src/mcp/PreviewAutomationBroker.test.ts‎

Lines changed: 61 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -127,50 +127,70 @@ it.effect("targets multiple tabs explicitly while retaining a default tab", () =
127127
),
128128
);
129129

130-
it.effect("does not let an older response replace a newer explicit tab target", () =>
131-
Effect.scoped(
132-
Effect.gen(function* () {
133-
const broker = yield* makeBroker;
134-
const olderTabId = PreviewTabId.make("tab-older-request");
135-
const newerTabId = PreviewTabId.make("tab-newer-request");
136-
const releaseOlderResponse = yield* Deferred.make<void>();
137-
const routedRequests: RoutedRequest[] = [];
138-
const requests = requestsFrom(yield* broker.connect(makeHost()));
139-
yield* Stream.runForEach(requests, (request) => {
140-
routedRequests.push(request);
141-
const response = Effect.gen(function* () {
142-
if (request.tabId === olderTabId) {
143-
yield* Deferred.await(releaseOlderResponse);
144-
}
145-
yield* broker.respond({
146-
clientId: "client-1",
147-
connectionId: request.connectionId,
148-
requestId: request.requestId,
149-
ok: true,
150-
result: { url: "http://localhost:3200" },
130+
it.effect.each([true, false])(
131+
"keeps an older target stable while a newer explicit tab responds (implicit: %s)",
132+
(implicit) =>
133+
Effect.scoped(
134+
Effect.gen(function* () {
135+
const broker = yield* makeBroker;
136+
const olderTabId = PreviewTabId.make("tab-older-request");
137+
const newerTabId = PreviewTabId.make("tab-newer-request");
138+
const releaseOlderResponse = yield* Deferred.make<void>();
139+
const routedRequests: RoutedRequest[] = [];
140+
const requests = requestsFrom(yield* broker.connect(makeHost()));
141+
yield* Stream.runForEach(requests, (request) => {
142+
routedRequests.push(request);
143+
const response = Effect.gen(function* () {
144+
if (request.tabId === olderTabId && request.operation === "snapshot") {
145+
yield* Deferred.await(releaseOlderResponse);
146+
}
147+
yield* broker.respond({
148+
clientId: "client-1",
149+
connectionId: request.connectionId,
150+
requestId: request.requestId,
151+
ok: true,
152+
result: { url: "http://localhost:3200" },
153+
});
154+
if (request.tabId === newerTabId) {
155+
yield* Deferred.succeed(releaseOlderResponse, undefined);
156+
}
151157
});
152-
if (request.tabId === newerTabId) {
153-
yield* Deferred.succeed(releaseOlderResponse, undefined);
154-
}
158+
return response.pipe(Effect.forkScoped, Effect.asVoid);
159+
}).pipe(Effect.forkScoped);
160+
yield* Effect.yieldNow;
161+
162+
yield* broker.invoke({ scope, operation: "status", input: {}, tabId: olderTabId });
163+
let capturedTabId: PreviewTabId | undefined;
164+
const older = yield* broker
165+
.invoke({
166+
scope,
167+
operation: "snapshot",
168+
input: {},
169+
...(implicit ? {} : { tabId: olderTabId }),
170+
onTargetTab: (tabId) => {
171+
capturedTabId = tabId;
172+
},
173+
})
174+
.pipe(Effect.forkScoped);
175+
yield* Effect.yieldNow;
176+
const newer = yield* broker
177+
.invoke({ scope, operation: "snapshot", input: {}, tabId: newerTabId })
178+
.pipe(Effect.forkScoped);
179+
yield* Fiber.join(newer);
180+
yield* Fiber.join(older);
181+
yield* broker.invoke({
182+
scope,
183+
operation: "status",
184+
input: {},
185+
tabId: olderTabId,
186+
updateCurrentTab: false,
155187
});
156-
return response.pipe(Effect.forkScoped, Effect.asVoid);
157-
}).pipe(Effect.forkScoped);
158-
yield* Effect.yieldNow;
159-
160-
const older = yield* broker
161-
.invoke({ scope, operation: "snapshot", input: {}, tabId: olderTabId })
162-
.pipe(Effect.forkScoped);
163-
yield* Effect.yieldNow;
164-
const newer = yield* broker
165-
.invoke({ scope, operation: "snapshot", input: {}, tabId: newerTabId })
166-
.pipe(Effect.forkScoped);
167-
yield* Fiber.join(newer);
168-
yield* Fiber.join(older);
169-
yield* broker.invoke({ scope, operation: "snapshot", input: {} });
188+
yield* broker.invoke({ scope, operation: "snapshot", input: {} });
170189

171-
expect(routedRequests.at(-1)?.tabId).toBe(newerTabId);
172-
}),
173-
),
190+
expect(routedRequests.at(-1)?.tabId).toBe(newerTabId);
191+
expect(capturedTabId).toBe(olderTabId);
192+
}),
193+
),
174194
);
175195

176196
it.effect("tracks the tab returned by a targeted recording stop", () =>

‎apps/server/src/mcp/PreviewAutomationBroker.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,10 @@ export interface PreviewAutomationInvokeInput {
4444
readonly input: unknown;
4545
readonly tabId?: PreviewTabId;
4646
readonly timeoutMs?: number;
47+
/** Background metadata reads must not change the agent's current tab. */
48+
readonly updateCurrentTab?: boolean;
49+
/** Capture the routed tab before another request changes the current assignment. */
50+
readonly onTargetTab?: (tabId: PreviewTabId | undefined) => void;
4751
}
4852

4953
export class PreviewAutomationBroker extends Context.Service<
@@ -541,6 +545,7 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() {
541545
});
542546
}
543547
const { connection, requestId, requestContext, requestSequence } = route;
548+
input.onTargetTab?.(requestContext.tabId);
544549
const removePending = SynchronizedRef.update(state, (next) => {
545550
if (!next.pending.has(requestId)) return next;
546551
const pending = new Map(next.pending);
@@ -575,6 +580,7 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() {
575580
});
576581
});
577582
const result = yield* awaitResponse().pipe(Effect.ensuring(removePending));
583+
if (input.updateCurrentTab === false) return result;
578584
const responseTabId = readResultTabId(result);
579585
const resultTabId = responseTabId === undefined ? input.tabId : responseTabId;
580586
if (resultTabId === undefined) return result;

‎apps/server/src/mcp/toolkits/preview/handlers.ts‎

Lines changed: 52 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import {
77
PreviewAutomationRecordingTransferError,
88
PreviewAutomationRecordingDesktopUpdateRequiredError,
99
PreviewAutomationRecordingArtifact,
10+
type ToolActivityIcon,
1011
type ThreadId,
1112
type PreviewAutomationOperation,
1213
type PreviewAutomationOpenInput,
@@ -55,22 +56,47 @@ const invoke = Effect.fn("PreviewToolkit.invoke")(function* <A>(
5556
timeoutMs?: number,
5657
tabId?: PreviewTabId,
5758
): Effect.fn.Return<
58-
A,
59+
{ result: A; toolIcon?: ToolActivityIcon },
5960
import("@t3tools/contracts").PreviewAutomationError,
6061
McpInvocationContext.McpInvocationContext | PreviewAutomationBroker.PreviewAutomationBroker
6162
> {
6263
const scope = yield* McpInvocationContext.requireMcpCapability("preview");
6364
const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker;
64-
return yield* broker.invoke<A>({
65+
let targetTabId = tabId;
66+
const result = yield* broker.invoke<A>({
67+
onTargetTab: (resolvedTabId) => {
68+
targetTabId = resolvedTabId;
69+
},
6570
scope,
6671
operation,
6772
input,
6873
...(timeoutMs === undefined ? {} : { timeoutMs }),
6974
...(tabId === undefined ? {} : { tabId }),
7075
});
76+
if (["status", "open", "navigate", "snapshot"].includes(operation)) return { result };
77+
const statusTabId =
78+
(operation !== "evaluate" && typeof result === "object" && result !== null
79+
? (result as { tabId?: PreviewTabId }).tabId
80+
: undefined) ?? targetTabId;
81+
const page = yield* broker
82+
.invoke<PreviewAutomationStatus>({
83+
scope,
84+
operation: "status",
85+
input: {},
86+
timeoutMs: 500,
87+
updateCurrentTab: false,
88+
...(statusTabId === undefined ? {} : { tabId: statusTabId }),
89+
})
90+
.pipe(Effect.catch(() => Effect.succeed(null)));
91+
return {
92+
result,
93+
...(page?.url && /^https?:\/\//i.test(page.url) && page.url.length <= 4096
94+
? { toolIcon: { _tag: "website" as const, pageUrl: page.url } }
95+
: {}),
96+
};
7197
});
7298

73-
const invokeTargeted = <A>(
99+
const invokeTargeted = <A extends object>(
74100
operation: PreviewAutomationOperation,
75101
input: {
76102
readonly tabId?: PreviewTabId | undefined;
@@ -79,7 +105,12 @@ const invokeTargeted = <A>(
79105
timeoutMs?: number,
80106
) => {
81107
const { tabId, ...operationInput } = input;
82-
return invoke<A>(operation, operationInput, timeoutMs, tabId);
108+
return invoke<A>(operation, operationInput, timeoutMs, tabId).pipe(
109+
Effect.map(({ result, toolIcon }) => ({
110+
...result,
111+
...(toolIcon ? { toolIcon } : {}),
112+
})),
113+
);
83114
};
84115

85116
const UploadedRecordingArtifact = Schema.Struct({
@@ -170,28 +201,32 @@ const handlers = {
170201
const { includeImage: _includeImage, save: _save, ...operationInput } = input ?? {};
171202
return invokeTargeted<PreviewAutomationSnapshot>("snapshot", operationInput);
172203
},
173-
preview_click: (input) =>
174-
invokeTargeted<void>("click", input, input.timeoutMs).pipe(Effect.as({})),
175-
preview_type: (input) => invokeTargeted<void>("type", input, input.timeoutMs).pipe(Effect.as({})),
176-
preview_press: (input) => invokeTargeted<void>("press", input).pipe(Effect.as({})),
177-
preview_scroll: (input) => invokeTargeted<void>("scroll", input).pipe(Effect.as({})),
178-
preview_evaluate: (input) =>
179-
invokeTargeted<unknown>("evaluate", input).pipe(
180-
Effect.map((result) => ({ value: result ?? null })),
204+
preview_click: (input) => invokeTargeted<object>("click", input, input.timeoutMs),
205+
preview_type: (input) => invokeTargeted<object>("type", input, input.timeoutMs),
206+
preview_press: (input) => invokeTargeted<object>("press", input),
207+
preview_scroll: (input) => invokeTargeted<object>("scroll", input),
208+
preview_evaluate: ({ tabId, ...input }) =>
209+
invoke<unknown>("evaluate", input, undefined, tabId).pipe(
210+
Effect.map(({ result, toolIcon }) => ({
211+
value: result ?? null,
212+
...(toolIcon ? { toolIcon } : {}),
213+
})),
181214
),
182-
preview_wait_for: (input) =>
183-
invokeTargeted<void>("waitFor", input, input.timeoutMs).pipe(Effect.as({})),
215+
preview_wait_for: (input) => invokeTargeted<object>("waitFor", input, input.timeoutMs),
184216
preview_recording_start: (input) =>
185217
invokeTargeted<PreviewAutomationRecordingStatus>("recordingStart", input ?? {}),
186218
preview_recording_stop: (input) =>
187219
Effect.gen(function* () {
188220
const scope = yield* McpInvocationContext.requireMcpCapability("preview");
189-
const response = yield* invokeTargeted<unknown>(
221+
const { tabId, ...operationInput } = input;
222+
const response = yield* invoke<unknown>(
190223
"recordingStop",
191-
{ ...input, transferToEnvironment: true },
224+
{ ...operationInput, transferToEnvironment: true },
192225
PREVIEW_RECORDING_STOP_TIMEOUT_MS,
226+
tabId,
193227
);
194-
return yield* claimPreviewRecording(scope.threadId, response);
228+
const artifact = yield* claimPreviewRecording(scope.threadId, response.result);
229+
return { ...artifact, ...(response.toolIcon ? { toolIcon: response.toolIcon } : {}) };
195230
}),
196231
} satisfies Parameters<typeof PreviewToolkit.toLayer>[0];
197232

0 commit comments

Comments
 (0)