Skip to content

Commit 38c187b

Browse files
authored
fix(pi): preserve tool images and structured results (#17851)
1 parent 4f63275 commit 38c187b

6 files changed

Lines changed: 659 additions & 141 deletions

File tree

‎packages/provider-pi/src/server/adapter.test.ts‎

Lines changed: 294 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -39,13 +39,16 @@ import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
3939
import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
4040
import { handoffBudget } from "@t3tools/provider-core/server/handoffBudget";
4141
import * as HostProcess from "@t3tools/shared/HostProcess";
42+
import { toolOutputImages, compactDynamicToolOutput } from "@t3tools/shared/toolOutput";
4243
import {
4344
makePiAdapterV2,
4445
PiAdapterV2Driver,
4546
PI_PROVIDER,
4647
type PiAdapterV2Options,
4748
} from "./adapter.ts";
4849
import { makePiRpcConnection, type PiRpcRecord } from "./rpc.ts";
50+
import { loadMcpBridge } from "./mcpBridge.testkit.ts";
51+
import { turnItemOutputText } from "../../../client-runtime/src/work-log/itemDetail.ts";
4952

5053
const layerTest = Layer.mergeAll(
5154
NodeServices.layer,
@@ -77,7 +80,7 @@ const modelSelection = (model: string): ModelSelection => ({
7780

7881
interface FakePi {
7982
readonly spawner: ChildProcessSpawner.ChildProcessSpawner["Service"];
80-
readonly emit: (record: PiRpcRecord) => Effect.Effect<void>;
83+
readonly emit: (record: PiRpcRecord, chunkBytes?: number) => Effect.Effect<void>;
8184
readonly takeRequest: (type: string) => Effect.Effect<PiRpcRecord>;
8285
/** Data returned by the next `get_entries` acks, consumed in order. */
8386
readonly queueEntries: (data: unknown) => void;
@@ -153,10 +156,14 @@ const makeFakePi: Effect.Effect<FakePi> = Effect.gen(function* () {
153156
let models: ReadonlyArray<unknown> = [];
154157
let stdinBuffer = "";
155158

156-
const emit = (record: PiRpcRecord) =>
157-
Queue.offer(stdout, new TextEncoder().encode(`${encodeJsonLine(record)}\n`)).pipe(
158-
Effect.asVoid,
159-
);
159+
const emit = (record: PiRpcRecord, chunkBytes?: number) =>
160+
Effect.gen(function* () {
161+
const bytes = new TextEncoder().encode(`${encodeJsonLine(record)}\n`);
162+
const size = chunkBytes ?? bytes.byteLength;
163+
for (let offset = 0; offset < bytes.byteLength; offset += size) {
164+
yield* Queue.offer(stdout, bytes.subarray(offset, offset + size));
165+
}
166+
});
160167

161168
const respondTo = (record: PiRpcRecord): PiRpcRecord | null => {
162169
if (typeof record["id"] !== "string") return null;
@@ -1852,6 +1859,284 @@ describe("PiAdapterV2", () => {
18521859
}).pipe(Effect.scoped, Effect.provide(layerTest)),
18531860
);
18541861

1862+
it.effect(
1863+
"keeps native and MCP images available to the shared asset reader with typed results",
1864+
() =>
1865+
Effect.gen(function* () {
1866+
const fake = yield* makeFakePi;
1867+
const { runtime, takeEvent } = yield* openRuntime(fake);
1868+
const providerThread = yield* runtime.ensureThread({
1869+
threadId: THREAD_ID,
1870+
modelSelection: modelSelection("default"),
1871+
runtimePolicy,
1872+
});
1873+
yield* startTurn(runtime, providerThread);
1874+
yield* fake.takeRequest("prompt");
1875+
yield* fake.emit({ type: "agent_start" });
1876+
const image = {
1877+
type: "image",
1878+
mimeType: "image/png",
1879+
data: Buffer.alloc(64 * 1024, 0x61).toString("base64"),
1880+
};
1881+
const note = "Read image file [image/png]";
1882+
const content = [{ type: "text", text: note }, image];
1883+
const structured = { count: 2, ready: true, paths: ["one.png", "two.png"] };
1884+
const scriptResult = { content, structuredContent: { threadId: "child-thread" } };
1885+
const cases = [
1886+
{
1887+
toolName: "read",
1888+
result: { content, structuredContent: { ...image, note } },
1889+
expected: { content },
1890+
},
1891+
{
1892+
toolName: "image_generate",
1893+
result: { content, structuredContent: structured },
1894+
expected: { content, structuredContent: structured },
1895+
},
1896+
{
1897+
toolName: "mcp__t3-code__preview_snapshot",
1898+
result: {
1899+
content,
1900+
structuredContent: scriptResult,
1901+
details: { server: "t3-code", tool: "preview_snapshot" },
1902+
},
1903+
expected: { content, structuredContent: { threadId: "child-thread" } },
1904+
},
1905+
{
1906+
toolName: "structured_tool",
1907+
result: { content: [], structuredContent: [false, 2, null, { ready: true }] },
1908+
expected: { content: [], structuredContent: [false, 2, null, { ready: true }] },
1909+
},
1910+
{
1911+
toolName: "mcp__custom__extension",
1912+
result: { content, structuredContent: { content: ["domain content"], ready: true } },
1913+
expected: { content, structuredContent: { content: ["domain content"], ready: true } },
1914+
},
1915+
{
1916+
toolName: "mcp__t3-code__preview_snapshot",
1917+
result: {
1918+
content: [
1919+
{ type: "text", text: "first" },
1920+
{ type: "text", text: "second" },
1921+
],
1922+
structuredContent: {
1923+
content: [
1924+
{ type: "text", text: "first" },
1925+
{ type: "text", text: "second" },
1926+
],
1927+
},
1928+
details: { server: "t3-code", tool: "preview_snapshot" },
1929+
},
1930+
expected: {
1931+
content: [
1932+
{ type: "text", text: "first" },
1933+
{ type: "text", text: "second" },
1934+
],
1935+
},
1936+
},
1937+
];
1938+
for (const [index, test] of cases.entries()) {
1939+
for (const phase of ["update", "end"] as const) {
1940+
yield* fake.emit({
1941+
type: `tool_execution_${phase}`,
1942+
toolCallId: `image-${index}`,
1943+
toolName: test.toolName,
1944+
...(phase === "end" ? { result: test.result } : { partialResult: test.result }),
1945+
isError: false,
1946+
});
1947+
const event = yield* takeEvent(
1948+
(event) =>
1949+
event.type === "turn_item.updated" && event.turnItem.type === "dynamic_tool",
1950+
);
1951+
if (event.type !== "turn_item.updated" || event.turnItem.type !== "dynamic_tool")
1952+
return yield* Effect.die("Expected an image tool item");
1953+
assert.equal(event.turnItem.status, phase === "end" ? "completed" : "running");
1954+
assert.deepEqual(event.turnItem.output, test.expected);
1955+
assert.deepEqual(
1956+
toolOutputImages(event.turnItem.output),
1957+
test.expected.content.some((block) => block.type === "image")
1958+
? [{ mimeType: "image/png", data: image.data }]
1959+
: [],
1960+
);
1961+
if (index === 0) assert.equal(turnItemOutputText(event.turnItem), note);
1962+
if (index === 5) assert.equal(turnItemOutputText(event.turnItem), "first\nsecond");
1963+
if (index === 2) {
1964+
assert.deepEqual(compactDynamicToolOutput(event.turnItem.output), {
1965+
threadId: "child-thread",
1966+
});
1967+
assert.equal(JSON.stringify(event.turnItem.output).split(image.data).length - 1, 1);
1968+
}
1969+
}
1970+
}
1971+
}).pipe(Effect.scoped, Effect.provide(layerTest)),
1972+
);
1973+
1974+
it.effect(
1975+
"keeps large tool text readable, drops identical read mirrors, and bounds structured values",
1976+
() =>
1977+
Effect.gen(function* () {
1978+
const fake = yield* makeFakePi;
1979+
const { runtime, takeEvent } = yield* openRuntime(fake);
1980+
const providerThread = yield* runtime.ensureThread({
1981+
threadId: THREAD_ID,
1982+
modelSelection: modelSelection("default"),
1983+
runtimePolicy,
1984+
});
1985+
yield* startTurn(runtime, providerThread);
1986+
yield* fake.takeRequest("prompt");
1987+
yield* fake.emit({ type: "agent_start" });
1988+
const text = "hello\nworld\n" + "🙂".repeat(40_000);
1989+
const content = [{ type: "text", text }];
1990+
const cases = [
1991+
{ content, structuredContent: text },
1992+
{ content, structuredContent: { oversized: "x".repeat(128 * 1024) } },
1993+
{ content, structuredContent: Array.from({ length: 10_000 }, () => 1) },
1994+
{
1995+
content,
1996+
structuredContent: Array.from({ length: 40 }).reduce<unknown>(
1997+
(nested) => ({ nested }),
1998+
null,
1999+
),
2000+
},
2001+
];
2002+
for (const [index, result] of cases.entries()) {
2003+
yield* fake.emit({
2004+
type: "tool_execution_end",
2005+
toolCallId: `structured-${index}`,
2006+
toolName: "read",
2007+
result,
2008+
isError: true,
2009+
});
2010+
const event = yield* takeEvent(
2011+
(event) => event.type === "turn_item.updated" && event.turnItem.type === "dynamic_tool",
2012+
);
2013+
if (event.type !== "turn_item.updated" || event.turnItem.type !== "dynamic_tool")
2014+
return yield* Effect.die("Expected a tool item");
2015+
assert.equal(event.turnItem.status, "failed");
2016+
assert.notProperty(event.turnItem.output, "structuredContent");
2017+
const displayed = turnItemOutputText(event.turnItem);
2018+
if (index === 0) {
2019+
assert.equal(event.turnItem.output, text);
2020+
assert.equal(displayed, text);
2021+
} else {
2022+
assert.equal(
2023+
displayed,
2024+
text + "\nStructured output omitted because it exceeds the stored result limit.",
2025+
);
2026+
assert.deepEqual(Object.keys(event.turnItem.output as object), ["content"]);
2027+
}
2028+
}
2029+
// Distinct typed metadata remains stored even when readable text exists.
2030+
yield* fake.emit({
2031+
type: "tool_execution_end",
2032+
toolCallId: "distinct-metadata",
2033+
toolName: "read",
2034+
result: { content, structuredContent: { path: "read.ts", ready: true } },
2035+
});
2036+
const event = yield* takeEvent(
2037+
(event) => event.type === "turn_item.updated" && event.turnItem.type === "dynamic_tool",
2038+
);
2039+
if (event.type !== "turn_item.updated" || event.turnItem.type !== "dynamic_tool")
2040+
return yield* Effect.die("Expected a tool item");
2041+
assert.deepEqual(event.turnItem.output, {
2042+
content,
2043+
structuredContent: { path: "read.ts", ready: true },
2044+
});
2045+
assert.equal(turnItemOutputText(event.turnItem), text);
2046+
const note = "Read image file [image/png]";
2047+
const image = { type: "image", data: "AAAA", mimeType: "image/png" };
2048+
const imageContent = [{ type: "text", text: note }, image];
2049+
// An extension can attach distinct typed metadata to a read result.
2050+
for (const structuredContent of [
2051+
{ ...image, note, width: 1200 },
2052+
{ ...image, note: "A distinct note" },
2053+
{ ...image, note, data: "AQID" },
2054+
]) {
2055+
yield* fake.emit({
2056+
type: "tool_execution_end",
2057+
toolCallId: "distinct-image-metadata",
2058+
toolName: "read",
2059+
result: { content: imageContent, structuredContent },
2060+
});
2061+
const event = yield* takeEvent(
2062+
(event) => event.type === "turn_item.updated" && event.turnItem.type === "dynamic_tool",
2063+
);
2064+
if (event.type !== "turn_item.updated" || event.turnItem.type !== "dynamic_tool")
2065+
return yield* Effect.die("Expected a read image tool item");
2066+
assert.deepEqual(event.turnItem.output, { content: imageContent, structuredContent });
2067+
assert.equal(turnItemOutputText(event.turnItem), note);
2068+
}
2069+
}).pipe(Effect.scoped, Effect.provide(layerTest)),
2070+
);
2071+
2072+
it.effect(
2073+
"delivers realistic MCP screenshots through the bridge, JSONL transport, and adapter",
2074+
() =>
2075+
Effect.gen(function* () {
2076+
const fake = yield* makeFakePi;
2077+
const { runtime, takeEvent } = yield* openRuntime(fake);
2078+
const providerThread = yield* runtime.ensureThread({
2079+
threadId: THREAD_ID,
2080+
modelSelection: modelSelection("default"),
2081+
runtimePolicy,
2082+
});
2083+
yield* startTurn(runtime, providerThread);
2084+
yield* fake.takeRequest("prompt");
2085+
yield* fake.emit({ type: "agent_start" });
2086+
for (const size of [3.25 * 1024 * 1024, 10 * 1024 * 1024]) {
2087+
const pixels = Buffer.alloc(size, 0x61);
2088+
const data = pixels.toString("base64");
2089+
const content = [
2090+
{ type: "text", text: "Screenshot captured" },
2091+
{ type: "image", data, mimeType: "image/png" },
2092+
];
2093+
const metadata = { screenshot: { width: 1200, height: 800 }, ready: true };
2094+
const bridge = yield* Effect.promise(() =>
2095+
loadMcpBridge({ modern: true, result: { content, structuredContent: metadata } }),
2096+
);
2097+
const tool = bridge.tools.find((tool) => tool.name === "mcp__t3-code__preview_snapshot")!;
2098+
const result = yield* Effect.promise(() => tool.execute("screenshot", {}));
2099+
assert.deepEqual(result.structuredContent, { content, structuredContent: metadata });
2100+
yield* fake.emit(
2101+
{
2102+
type: "tool_execution_end",
2103+
toolCallId: "screenshot",
2104+
toolName: tool.name,
2105+
result,
2106+
isError: false,
2107+
},
2108+
64 * 1024,
2109+
);
2110+
// A following small completion proves a dropped screenshot immediately,
2111+
// without waiting for a timeout when framing loses the large event.
2112+
yield* fake.emit({
2113+
type: "tool_execution_end",
2114+
toolCallId: "after-screenshot",
2115+
toolName: "marker",
2116+
result: { content: [{ type: "text", text: "after" }] },
2117+
});
2118+
const event = yield* takeEvent(
2119+
(event) => event.type === "turn_item.updated" && event.turnItem.type === "dynamic_tool",
2120+
);
2121+
if (event.type !== "turn_item.updated" || event.turnItem.type !== "dynamic_tool")
2122+
return yield* Effect.die("Expected a screenshot tool item");
2123+
assert.equal(event.turnItem.toolName, tool.name);
2124+
assert.equal(event.turnItem.status, "completed");
2125+
assert.deepEqual(event.turnItem.output, { content, structuredContent: metadata });
2126+
assert.equal(turnItemOutputText(event.turnItem), "Screenshot captured");
2127+
const image = toolOutputImages(event.turnItem.output)[0];
2128+
assert.equal(image?.mimeType, "image/png");
2129+
assert.deepEqual(Buffer.from(image!.data!, "base64"), pixels);
2130+
yield* takeEvent(
2131+
(event) =>
2132+
event.type === "turn_item.updated" &&
2133+
event.turnItem.type === "dynamic_tool" &&
2134+
event.turnItem.toolName === "marker",
2135+
);
2136+
}
2137+
}).pipe(Effect.scoped, Effect.provide(layerTest)),
2138+
);
2139+
18552140
it.effect("observes official subagent results without inventing child threads", () =>
18562141
Effect.gen(function* () {
18572142
const fake = yield* makeFakePi;
@@ -3512,8 +3797,10 @@ describe("PiRpc framing", () => {
35123797
yield* push('{"type":"agent_');
35133798
yield* push('start"}\r\n{"type":"agent_settled"}\nnot json\n{"type":"queue_update"}\n');
35143799

3515-
yield* push("x".repeat(8 * 1024 * 1024));
3516-
yield* push('x{"type":"must_not_emit"}\n{"type":"after_oversized"}\n');
3800+
yield* push('{"type":"must_not_emit","text":"');
3801+
const chunk = new TextEncoder().encode("x".repeat(8 * 1024 * 1024));
3802+
for (let index = 0; index < 32; index++) yield* Queue.offer(stdout, chunk);
3803+
yield* push('"}\n{"type":"after_oversized"}\n');
35173804

35183805
const first = yield* Queue.take(connection.events);
35193806
assert.equal(first["type"], "agent_start");

0 commit comments

Comments
 (0)