Skip to content

Commit 6a33362

Browse files
authored
Merge branch 'main' into t3code/preview-fallback-messages
2 parents 1e52a34 + 4f27a84 commit 6a33362

23 files changed

Lines changed: 1432 additions & 204 deletions

‎apps/server/src/device/DeviceHubProxy.test.ts‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,32 @@ describe("device hub proxy", () => {
111111
await response.text();
112112
});
113113

114+
it("reads Android fold state but requires operate scope to change it", async () => {
115+
const path = "http://t3.test/api/device-hub/vendor/serve-emu/api/fold?device=emulator-5554";
116+
const reader = fixture([AuthOrchestrationReadScope]);
117+
const read = await reader.handler(new Request(path));
118+
expect(read.status).toBe(200);
119+
await read.text();
120+
expect(reader.requests).toEqual([
121+
"http://hub.test/vendor/serve-emu/api/fold?device=emulator-5554",
122+
]);
123+
const denied = await reader.handler(
124+
new Request(path, { method: "POST", body: '{"posture":"closed"}' }),
125+
);
126+
expect(denied.status).toBe(403);
127+
expect(reader.requests).toHaveLength(1);
128+
129+
const operator = fixture([AuthOrchestrationOperateScope]);
130+
const changed = await operator.handler(
131+
new Request(path, { method: "POST", body: '{"posture":"closed"}' }),
132+
);
133+
expect(changed.status).toBe(200);
134+
await changed.text();
135+
expect(operator.requests).toEqual([
136+
"http://hub.test/vendor/serve-emu/api/fold?device=emulator-5554",
137+
]);
138+
});
139+
114140
it("never forwards the vendor shell endpoint", async () => {
115141
const { handler, requests } = fixture([AuthOrchestrationOperateScope]);
116142
expect(

‎apps/server/src/device/DeviceHubProxy.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,14 +44,15 @@ const ALLOWED_PATHS: ReadonlyArray<RegExp> = [
4444
/^\/vendor\/serve-sim\/helper\/[^/]+\/(stream\.mjpeg|stream\.avcc|config|health|ax|foreground)$/,
4545
/^\/vendor\/serve-sim\/helper\/[^/]+\/panel\/(1|3)\/stream\.avcc$/,
4646
/^\/vendor\/serve-sim\/appstate$/,
47-
/^\/vendor\/serve-emu\/api\/(devices|screenshot|stream-mode|stream-settings|accessibility)$/,
47+
/^\/vendor\/serve-emu\/api\/(devices|screenshot|stream-mode|stream-settings|accessibility|fold)$/,
4848
/^\/vendor\/serve-emu\/health$/,
4949
];
5050

5151
/** Read paths are GET-only; only these accept other methods (screenshot captures, stream tuning). */
5252
const MUTABLE_PATHS: ReadonlyArray<RegExp> = [
5353
/^\/vendor\/serve-sim\/api\/screenshot$/,
5454
/^\/vendor\/serve-emu\/api\/(screenshot|stream-mode|stream-settings)$/,
55+
/^\/vendor\/serve-emu\/api\/fold$/,
5556
];
5657

5758
const ALLOWED_WS_PATHS: ReadonlyArray<RegExp> = [
@@ -204,7 +205,7 @@ const handler = Effect.gen(function* () {
204205
}
205206
const controlsDevice =
206207
(upgrade && hubPath !== "/api/devices/ws") ||
207-
(!readOnly && /\/api\/stream-(mode|settings)$/.test(hubPath));
208+
(!readOnly && /\/api\/(stream-(mode|settings)|fold)$/.test(hubPath));
208209
yield* authenticate(controlsDevice ? AuthOrchestrationOperateScope : AuthOrchestrationReadScope);
209210
const devices = yield* DeviceService.DeviceService;
210211
const ready = yield* devices.currentReadiness(url.value.searchParams.get("hostId") ?? undefined);

‎apps/server/src/mcp/McpHttpServer.test.ts‎

Lines changed: 58 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -265,7 +265,10 @@ it.effect.each([
265265
const metadata = { ...page, title: `Snapshot ${call}`, screenshot };
266266
const { accessibilityTree: _tree, ...boundedMetadata } = metadata;
267267
expect(snapshot.isError).toBe(false);
268-
expect(snapshot.structuredContent).toEqual(metadata);
268+
expect(snapshot.structuredContent).toEqual({
269+
...boundedMetadata,
270+
omitted: ["accessibilityTree (use interactiveElements locators or preview_evaluate)"],
271+
});
269272
const [identity, text, ...rest] = snapshot.content;
270273
expect(identity?.type === "text" ? decodeJsonText(identity.text) : null).toEqual({
271274
url: page.url,
@@ -304,7 +307,8 @@ it.effect.each([
304307
"text",
305308
"image",
306309
]);
307-
expect(nextDefault.structuredContent).toEqual({ ...page, title: "Snapshot 7", screenshot });
310+
expect(nextDefault.structuredContent).toMatchObject({ title: "Snapshot 7", screenshot });
311+
expect(nextDefault.structuredContent).not.toHaveProperty("accessibilityTree");
308312
expect(requests).toBe(7);
309313
}),
310314
).pipe(Effect.provide(TestLayer)),
@@ -358,6 +362,15 @@ it.effect("saves the snapshot PNG on request and reports its path", () =>
358362

359363
const unsaved = yield* callSnapshot({});
360364
expect(unsaved.structuredContent).not.toHaveProperty("screenshotPath");
365+
366+
// A save without the image skips the page dump.
367+
const pathOnly = yield* callSnapshot({ save: true, includeImage: false });
368+
const saved = pathOnly.structuredContent as { readonly screenshotPath: string };
369+
expect(saved).toEqual({ url: snapshotResult.url, screenshotPath: expect.any(String) });
370+
expect(Buffer.from(yield* fileSystem.readFile(saved.screenshotPath)).toString()).toBe("png");
371+
const [only, ...others] = pathOnly.content;
372+
expect(others).toEqual([]);
373+
expect(only?.type === "text" ? decodeJsonText(only.text) : null).toEqual(saved);
361374
}),
362375
).pipe(Effect.provide(TestLayer)),
363376
);
@@ -477,9 +490,10 @@ it.effect("keeps the snapshot text under the agent's output ceiling", () =>
477490
expect(parsed.consoleEntries[0]?.text).toBe("entry 60");
478491
expect(notice?.type === "text" ? notice.text : "").toContain("accessibilityTree");
479492
expect(notice?.type === "text" ? notice.text : "").toContain("60 older console entries");
480-
// The structured result is untouched; only the text the agent reads is bounded.
481-
expect(snapshot.structuredContent).toMatchObject({
482-
accessibilityTree: oversized.accessibilityTree,
493+
// Claude Code shows the model structuredContent instead of the text, so it is bounded too.
494+
expect(snapshot.structuredContent).toEqual({
495+
...parsed,
496+
omitted: expect.arrayContaining(["60 older console entries"]),
483497
});
484498
}),
485499
).pipe(Effect.provide(TestLayer)),
@@ -517,6 +531,45 @@ it.effect("bounds the snapshot text even when nothing but logs and the title are
517531
).pipe(Effect.provide(TestLayer)),
518532
);
519533

534+
it.effect("bounds page text made of wide characters before dropping locators", () =>
535+
Effect.scoped(
536+
Effect.gen(function* () {
537+
// The character caps alone leave 8,000 three-byte characters, about 24 KB.
538+
yield* serveSnapshots("mcp-wide-text-client", {
539+
...snapshotResult,
540+
visibleText: "界".repeat(9_000),
541+
interactiveElements: Array.from({ length: 20 }, (_, i) => ({
542+
tag: "button",
543+
role: "button",
544+
name: `Button ${i}`,
545+
selector: `#button-${i}`,
546+
x: 0,
547+
y: 0,
548+
width: 10,
549+
height: 10,
550+
})),
551+
});
552+
553+
const snapshot = yield* callSnapshot({ includeImage: false });
554+
555+
const [, text, notice] = snapshot.content;
556+
const body = text?.type === "text" ? text.text : "";
557+
expect(Buffer.byteLength(body, "utf8")).toBeLessThanOrEqual(
558+
McpHttpServer.MAX_SNAPSHOT_TEXT_BYTES,
559+
);
560+
const parsed = decodeJsonText(body) as {
561+
readonly visibleText: string;
562+
readonly interactiveElements: ReadonlyArray<unknown>;
563+
};
564+
expect(parsed.visibleText).toMatch(/^界+…$/);
565+
expect(parsed.interactiveElements).toHaveLength(20);
566+
expect(notice?.type === "text" ? notice.text : "").toContain(
567+
"visibleText after 4000 characters",
568+
);
569+
}),
570+
).pipe(Effect.provide(TestLayer)),
571+
);
572+
520573
it.effect("sheds log entries before locators when every list is full", () =>
521574
Effect.scoped(
522575
Effect.gen(function* () {

‎apps/server/src/mcp/McpHttpServer.ts‎

Lines changed: 65 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -115,12 +115,15 @@ const McpAuthMiddlewareLive = HttpRouter.middleware<{
115115
}>()(makeMcpAuthMiddleware).layer;
116116

117117
/**
118-
* Claude Code drops every MCP result above 25k tokens (~100 KB of text) and
119-
* hands the agent a truncation notice instead, so a snapshot that carries the
120-
* full accessibility tree and 20 KB of page text loses its locators too. Keep
121-
* the text under that ceiling and tell the agent what was cut.
118+
* Claude Code moves an MCP result above its output limit to a file and hands
119+
* the agent a notice instead, so a snapshot that carries the full
120+
* accessibility tree and page text loses its locators too. Claude Code also
121+
* shows the model `structuredContent` in place of the text blocks when a
122+
* result has both, so both carry the same bounded snapshot. Keep it near
123+
* 20 KB and tell the agent what was cut. The short `omitted` notes may go a
124+
* little over; the provider limit is far above this.
122125
*/
123-
export const MAX_SNAPSHOT_TEXT_BYTES = 60_000;
126+
export const MAX_SNAPSHOT_TEXT_BYTES = 20_000;
124127
const MAX_SNAPSHOT_VISIBLE_TEXT_CHARS = 8_000;
125128
const MAX_SNAPSHOT_ELEMENT_NAME_CHARS = 200;
126129
const MAX_SNAPSHOT_LOG_ENTRIES = 40;
@@ -165,12 +168,11 @@ type SnapshotMetadata = {
165168
/**
166169
* Drops the accessibility tree, shortens page text, element names, identifiers,
167170
* and log strings, keeps only the newest log entries, and finally sheds
168-
* interactive elements until the JSON fits. Returns the text plus notes on
169-
* what is missing so the agent can reach for preview_evaluate.
171+
* interactive elements until the JSON fits. Returns the bounded value, its
172+
* text, and notes on what is missing so the agent can reach for
173+
* preview_evaluate.
170174
*/
171-
const boundSnapshotMetadata = (
172-
metadata: SnapshotMetadata,
173-
): { readonly text: string; readonly omitted: ReadonlyArray<string> } => {
175+
const boundSnapshotMetadata = (metadata: SnapshotMetadata) => {
174176
const omitted: Array<string> = [];
175177
const { accessibilityTree, ...withoutTree } = metadata;
176178
if (accessibilityTree !== undefined) {
@@ -199,16 +201,10 @@ const boundSnapshotMetadata = (
199201
) {
200202
omitted.push(`element names longer than ${MAX_SNAPSHOT_ELEMENT_NAME_CHARS} characters`);
201203
}
202-
if (metadata.visibleText.length > MAX_SNAPSHOT_VISIBLE_TEXT_CHARS) {
203-
omitted.push(
204-
`visibleText after ${MAX_SNAPSHOT_VISIBLE_TEXT_CHARS} characters (use preview_evaluate for more)`,
205-
);
206-
}
207204
const bounded = {
208205
...withoutTree,
209206
url: cutText(metadata.url, MAX_SNAPSHOT_IDENTIFIER_CHARS),
210207
title: cutText(metadata.title, MAX_SNAPSHOT_IDENTIFIER_CHARS),
211-
visibleText: cutText(metadata.visibleText, MAX_SNAPSHOT_VISIBLE_TEXT_CHARS),
212208
interactiveElements: metadata.interactiveElements.map((element) => ({
213209
...element,
214210
name: cutText(element.name, MAX_SNAPSHOT_ELEMENT_NAME_CHARS),
@@ -219,9 +215,10 @@ const boundSnapshotMetadata = (
219215
};
220216

221217
// Per-field caps do not sum below the ceiling: three log arrays of 40 capped
222-
// entries alone can pass 60 KB. Shed the least useful lists first, halving
223-
// one list per round, until the JSON fits. With every list empty the rest
224-
// is bounded by the identifier and visibleText caps, so this terminates.
218+
// entries alone can pass 60 KB, and the caps count characters, not bytes.
219+
// Halve one thing per round until the JSON fits: logs first, then page
220+
// text, then the locators. The identifier caps bound the rest, so this
221+
// terminates.
225222
const shedOrder = [
226223
"actionTimeline",
227224
"networkEntries",
@@ -240,31 +237,51 @@ const boundSnapshotMetadata = (
240237
networkEntries: 0,
241238
actionTimeline: 0,
242239
};
243-
let text = encodeJsonText({ ...bounded, ...lists });
240+
let visibleTextChars = Math.min(metadata.visibleText.length, MAX_SNAPSHOT_VISIBLE_TEXT_CHARS);
241+
const value = () => ({
242+
...bounded,
243+
visibleText: cutText(metadata.visibleText, visibleTextChars),
244+
...lists,
245+
});
246+
let text = encodeJsonText(value());
244247
while (utf8Length(text) > MAX_SNAPSHOT_TEXT_BYTES) {
245248
// Elements carry the locators, so they go last; logs shed newest-last.
246249
const key =
247250
shedOrder.find(
248251
(candidate) => candidate !== "interactiveElements" && lists[candidate].length > 0,
249-
) ?? (lists.interactiveElements.length > 0 ? "interactiveElements" : undefined);
252+
) ??
253+
(visibleTextChars > 0
254+
? "visibleText"
255+
: lists.interactiveElements.length > 0
256+
? "interactiveElements"
257+
: undefined);
250258
if (key === undefined) break;
251-
const keep = Math.floor(lists[key].length / 2);
252-
dropped[key] += lists[key].length - keep;
253-
// slice(-0) keeps everything, so spell out the empty case.
254-
lists[key] =
255-
keep === 0
256-
? []
257-
: key === "interactiveElements"
258-
? lists[key].slice(0, keep)
259-
: lists[key].slice(-keep);
260-
text = encodeJsonText({ ...bounded, ...lists });
259+
if (key === "visibleText") {
260+
visibleTextChars = Math.floor(visibleTextChars / 2);
261+
} else {
262+
const keep = Math.floor(lists[key].length / 2);
263+
dropped[key] += lists[key].length - keep;
264+
// slice(-0) keeps everything, so spell out the empty case.
265+
lists[key] =
266+
keep === 0
267+
? []
268+
: key === "interactiveElements"
269+
? lists[key].slice(0, keep)
270+
: lists[key].slice(-keep);
271+
}
272+
text = encodeJsonText(value());
273+
}
274+
if (visibleTextChars < metadata.visibleText.length) {
275+
omitted.push(
276+
`visibleText after ${visibleTextChars} characters (use preview_evaluate for more)`,
277+
);
261278
}
262279
for (const key of shedOrder) {
263280
if (dropped[key] > 0) {
264281
omitted.push(`${dropped[key]} of ${bounded[key].length} ${key}`);
265282
}
266283
}
267-
return { text, omitted };
284+
return { value: value(), text, omitted };
268285
};
269286

270287
export class PreviewScreenshotSaveError extends Schema.TaggedError<PreviewScreenshotSaveError>()(
@@ -403,6 +420,18 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
403420
const png = new Uint8Array(Buffer.from(screenshot.data, "base64"));
404421
const screenshotPath =
405422
payload?.save === true ? yield* saveScreenshot(snapshot.url, png) : undefined;
423+
if (screenshotPath !== undefined && payload?.includeImage === false) {
424+
// The agent only wants a file to show the user. The url keeps the site icon on the tool row.
425+
const saved = {
426+
url: cutText(snapshot.url, MAX_SNAPSHOT_IDENTIFIER_CHARS),
427+
screenshotPath,
428+
};
429+
return new McpSchema.CallToolResult({
430+
isError: false,
431+
structuredContent: saved,
432+
content: [{ type: "text", text: encodeJsonText(saved) }],
433+
});
434+
}
406435
const metadata = {
407436
...page,
408437
screenshot: {
@@ -415,7 +444,10 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
415444
const bounded = boundSnapshotMetadata(metadata);
416445
return new McpSchema.CallToolResult({
417446
isError: false,
418-
structuredContent: metadata,
447+
structuredContent:
448+
bounded.omitted.length === 0
449+
? bounded.value
450+
: { ...bounded.value, omitted: bounded.omitted },
419451
content: [
420452
// Keep the page identity readable even if a provider truncates the snapshot.
421453
{

‎apps/server/src/mcp/toolkits/preview/tools.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ const PreviewSetAppearanceTool = safeBrowserTool(
119119
export const PreviewSnapshotTool = readonlyBrowserTool(
120120
Tool.make("preview_snapshot", {
121121
description:
122-
"Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot. Set includeImage=false for text-only output with the same page metadata. Set save=true to also write the PNG to disk and get screenshotPath back; embed that path in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.",
122+
"Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot. The text is capped near 20 KB and lists what it omitted; use preview_evaluate to read more. Set includeImage=false for text-only output with the same page metadata. Set save=true to also write the PNG to disk and get screenshotPath back; with includeImage=false, save=true returns only the url and screenshotPath. Embed that path in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.",
123123
parameters: Schema.Struct({
124124
...PreviewAutomationTabTargetInput.fields,
125125
includeImage: Schema.optional(
@@ -131,7 +131,7 @@ export const PreviewSnapshotTool = readonlyBrowserTool(
131131
save: Schema.optional(
132132
Schema.Boolean.annotate({
133133
description:
134-
"Write the screenshot PNG to disk and return its absolute path as screenshotPath. Defaults to false.",
134+
"Write the screenshot PNG to disk and return its absolute path as screenshotPath. With includeImage=false, return only the url and screenshotPath. Defaults to false.",
135135
}),
136136
),
137137
}),

0 commit comments

Comments
 (0)