Skip to content

Commit b63e06f

Browse files
snowzlmbotsnowzlmbot
authored
fix(llm): preserve structured tool result replay
Preserve structured tool-result replay text across provider transports while keeping provider-facing redaction and Anthropic media ordering intact. Thanks @snowzlmbot!
1 parent b2355ef commit b63e06f

30 files changed

Lines changed: 1514 additions & 120 deletions

‎docs/.generated/plugin-sdk-api-baseline.sha256‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
4f41a4ade7eadd84724ef2b951734c3c7573ac23f621a79b88976a15cb114f4c plugin-sdk-api-baseline.json
2-
14991eb87ea7b402c11c85bd2c2d817f2dfe2a45f695922cb7232327323e5803 plugin-sdk-api-baseline.jsonl
1+
c52e9007a94f19e63663495fb7e54824f9c29c4ebe403ea3b1a4e75f4ce8cedf plugin-sdk-api-baseline.json
2+
abbfc139068a2f98ecb7759d82597293cbcdba77b5036e8a2e9f07040834eb6d plugin-sdk-api-baseline.jsonl

‎docs/plugins/sdk-migration.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -621,7 +621,7 @@ releases.
621621
| `plugin-sdk/provider-tools` | Provider tool/schema compat helpers | `ProviderToolCompatFamily`, `buildProviderToolCompatFamilyHooks`, and DeepSeek/Gemini/OpenAI schema cleanup + diagnostics |
622622
| `plugin-sdk/provider-usage` | Provider usage helpers | `fetchClaudeUsage`, `fetchGeminiUsage`, `fetchGithubCopilotUsage`, and other provider usage helpers |
623623
| `plugin-sdk/provider-stream` | Provider stream wrapper helpers | `ProviderStreamFamily`, `buildProviderStreamFamilyHooks`, `composeProviderStreamWrappers`, stream wrapper types, and shared Anthropic/Bedrock/DeepSeek V4/Google/Kilocode/Moonshot/OpenAI/OpenRouter/Z.A.I/MiniMax/Copilot wrapper helpers |
624-
| `plugin-sdk/provider-transport-runtime` | Provider transport helpers | Native provider transport helpers such as guarded fetch, transport message transforms, and writable transport event streams |
624+
| `plugin-sdk/provider-transport-runtime` | Provider transport helpers | Native provider transport helpers such as guarded fetch, tool-result text extraction, transport message transforms, and writable transport event streams |
625625
| `plugin-sdk/keyed-async-queue` | Ordered async queue | `KeyedAsyncQueue` |
626626
| `plugin-sdk/media-runtime` | Shared media helpers | Media fetch/transform/store helpers, ffprobe-backed video dimension probing, and media payload builders |
627627
| `plugin-sdk/media-generation-runtime` | Shared media-generation helpers | Shared failover helpers, candidate selection, and missing-model messaging for image/video/music generation |

‎docs/plugins/sdk-subpaths.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,7 @@ and pairing-path families.
165165
| `plugin-sdk/provider-usage` | Provider usage snapshot types, shared usage fetch helpers, and provider fetchers such as `fetchClaudeUsage` |
166166
| `plugin-sdk/provider-stream` | `ProviderStreamFamily`, `buildProviderStreamFamilyHooks`, `composeProviderStreamWrappers`, stream wrapper types, plain-text tool-call compat, and shared Anthropic/Bedrock/DeepSeek V4/Google/Kilocode/Moonshot/OpenAI/OpenRouter/Z.A.I/MiniMax/Copilot wrapper helpers |
167167
| `plugin-sdk/provider-stream-shared` | Public shared provider stream wrapper helpers including `composeProviderStreamWrappers`, `createOpenAICompatibleCompletionsThinkingOffWrapper`, `createPlainTextToolCallCompatWrapper`, `createPayloadPatchStreamWrapper`, `createToolStreamWrapper`, `normalizeOpenAICompatibleReasoningPayload`, `setQwenChatTemplateThinking`, and Anthropic/DeepSeek/OpenAI-compatible stream utilities |
168-
| `plugin-sdk/provider-transport-runtime` | Native provider transport helpers such as guarded fetch, transport message transforms, and writable transport event streams |
168+
| `plugin-sdk/provider-transport-runtime` | Native provider transport helpers such as guarded fetch, tool-result text extraction, transport message transforms, and writable transport event streams |
169169
| `plugin-sdk/provider-onboard` | Onboarding config patch helpers |
170170
| `plugin-sdk/global-singleton` | Process-local singleton/map/cache helpers |
171171
| `plugin-sdk/group-activation` | Narrow group activation mode and command parsing helpers |

‎extensions/google/transport-stream.test.ts‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2166,6 +2166,173 @@ describe("google transport stream", () => {
21662166
expect(params.contents).toEqual([{ role: "user", parts: [{ text: " " }] }]);
21672167
});
21682168

2169+
it("serializes structured-only Google tool results before fallback", () => {
2170+
const params = buildGoogleGenerativeAiParams(buildGeminiModel(), {
2171+
messages: [
2172+
googleToolCallAssistantTurn(),
2173+
{
2174+
role: "toolResult",
2175+
toolCallId: "call_1",
2176+
toolName: "lookup",
2177+
content: [
2178+
{
2179+
type: "json",
2180+
value: { city: "Paris", temperatureC: 21 },
2181+
apiToken: "secret-token-123",
2182+
},
2183+
],
2184+
isError: false,
2185+
timestamp: 1,
2186+
},
2187+
],
2188+
} as never);
2189+
2190+
const functionResponse = (params.contents[1] as GoogleTestContentTurn).parts[0]
2191+
.functionResponse as { response: { output: string } };
2192+
2193+
expect(functionResponse).toMatchObject({ name: "lookup" });
2194+
expect(functionResponse.response.output).toContain('"city":"Paris"');
2195+
expect(functionResponse.response.output).toContain('"temperatureC":21');
2196+
expect(functionResponse.response.output).toContain('"apiToken":"');
2197+
expect(functionResponse.response.output).not.toContain("secret-token-123");
2198+
});
2199+
2200+
it("keeps explicit Google tool-result text before structured fallback", () => {
2201+
const params = buildGoogleGenerativeAiParams(buildGeminiModel(), {
2202+
messages: [
2203+
googleToolCallAssistantTurn(),
2204+
{
2205+
role: "toolResult",
2206+
toolCallId: "call_1",
2207+
toolName: "lookup",
2208+
content: [
2209+
{ type: "json", value: { ignored: true } },
2210+
{ type: "text", text: "explicit result" },
2211+
],
2212+
isError: false,
2213+
timestamp: 1,
2214+
},
2215+
],
2216+
} as never);
2217+
2218+
expect(params.contents[1]).toMatchObject({
2219+
parts: [{ functionResponse: { response: { output: "explicit result" } } }],
2220+
});
2221+
});
2222+
2223+
it("redacts opaque and binary structured Google tool-result fields", () => {
2224+
const params = buildGoogleGenerativeAiParams(buildGeminiModel(), {
2225+
messages: [
2226+
googleToolCallAssistantTurn(),
2227+
{
2228+
role: "toolResult",
2229+
toolCallId: "call_1",
2230+
toolName: "lookup",
2231+
content: [
2232+
{
2233+
type: "resource",
2234+
mimeType: "image/png",
2235+
data: "abcdef",
2236+
encrypted_content: "opaque",
2237+
text: "data:image/png;base64,abcdef",
2238+
},
2239+
],
2240+
isError: false,
2241+
timestamp: 1,
2242+
},
2243+
],
2244+
} as never);
2245+
2246+
const functionResponse = (params.contents[1] as GoogleTestContentTurn).parts[0]
2247+
.functionResponse as { response: { output: string } };
2248+
2249+
expect(functionResponse.response.output).toContain('"data":"[binary data omitted: 6 chars]"');
2250+
expect(functionResponse.response.output).toContain(
2251+
'"encrypted_content":"[omitted encrypted_content]"',
2252+
);
2253+
expect(functionResponse.response.output).toContain('"text":"[inline data URI: 23 chars]"');
2254+
});
2255+
2256+
it("uses shared structured redaction for Google tool-result fields", () => {
2257+
const params = buildGoogleGenerativeAiParams(buildGeminiModel(), {
2258+
messages: [
2259+
googleToolCallAssistantTurn(),
2260+
{
2261+
role: "toolResult",
2262+
toolCallId: "call_1",
2263+
toolName: "lookup",
2264+
content: [
2265+
{
2266+
type: "json",
2267+
privateKey: "leaked-private-key-value-12345",
2268+
private_key: "leaked-private-key-snake-12345",
2269+
key: "leaked-generic-key-value-12345",
2270+
keyMaterial: "leaked-key-material-value-12345",
2271+
jwt: "leaked-jwt-value-1234567890",
2272+
session: "leaked-session-value-123456",
2273+
code: "code-value-1234567890",
2274+
error: { code: "ERR_VISIBLE_GOOGLE_CODE" },
2275+
oauth: { code: "OPAQUEGOOGLECODE1234567890" },
2276+
providerError: { error: { code: "ERR_VISIBLE_PROVIDER_GOOGLE_CODE" } },
2277+
signature: "leaked-signature-value-12345",
2278+
cookie: "leaked-cookie-value-123456",
2279+
"set-cookie": "leaked-set-cookie-value-12345",
2280+
paymentCredential: "leaked-payment-credential-12345",
2281+
cardNumber: "41111111111111112222",
2282+
visible: "safe-value",
2283+
},
2284+
],
2285+
isError: false,
2286+
timestamp: 1,
2287+
},
2288+
],
2289+
} as never);
2290+
2291+
const functionResponse = (params.contents[1] as GoogleTestContentTurn).parts[0]
2292+
.functionResponse as { response: { output: string } };
2293+
2294+
expect(functionResponse.response.output).toContain('"visible":"safe-value"');
2295+
expect(functionResponse.response.output).toContain('"code":"ERR_VISIBLE_GOOGLE_CODE"');
2296+
expect(functionResponse.response.output).toContain('"code":"ERR_VISIBLE_PROVIDER_GOOGLE_CODE"');
2297+
for (const leakedValue of [
2298+
"leaked-private-key-value-12345",
2299+
"leaked-private-key-snake-12345",
2300+
"leaked-generic-key-value-12345",
2301+
"leaked-key-material-value-12345",
2302+
"leaked-jwt-value-1234567890",
2303+
"leaked-session-value-123456",
2304+
"code-value-1234567890",
2305+
"OPAQUEGOOGLECODE1234567890",
2306+
"leaked-signature-value-12345",
2307+
"leaked-cookie-value-123456",
2308+
"leaked-set-cookie-value-12345",
2309+
"leaked-payment-credential-12345",
2310+
"41111111111111112222",
2311+
]) {
2312+
expect(functionResponse.response.output).not.toContain(leakedValue);
2313+
}
2314+
});
2315+
2316+
it("keeps Google media-only tool results on media placeholders", () => {
2317+
const params = buildGoogleGenerativeAiParams(buildGeminiModel(), {
2318+
messages: [
2319+
googleToolCallAssistantTurn(),
2320+
{
2321+
role: "toolResult",
2322+
toolCallId: "call_1",
2323+
toolName: "lookup",
2324+
content: [{ type: "audio", mimeType: "audio/wav", data: "wav-bytes" }],
2325+
isError: false,
2326+
timestamp: 1,
2327+
},
2328+
],
2329+
} as never);
2330+
2331+
expect(params.contents[1]).toMatchObject({
2332+
parts: [{ functionResponse: { response: { output: "(see attached audio)" } } }],
2333+
});
2334+
});
2335+
21692336
it.each([
21702337
["image first", ["screenshot", "weather"]],
21712338
["image last", ["weather", "screenshot"]],

‎extensions/google/transport-stream.ts‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ import {
1515
coerceTransportToolCallArguments,
1616
createEmptyTransportUsage,
1717
createWritableTransportEventStream,
18+
describeToolResultMediaPlaceholder,
19+
extractToolResultText,
1820
failTransportStream,
1921
finalizeTransportStream,
2022
mergeTransportHeaders,
@@ -647,24 +649,17 @@ function convertGoogleMessages(model: GoogleTransportModel, context: Context) {
647649
}
648650

649651
if (msg.role === "toolResult") {
650-
const textResult = msg.content
651-
.filter(
652-
(item): item is Extract<(typeof msg.content)[number], { type: "text" }> =>
653-
item.type === "text",
654-
)
655-
.map((item) => item.text)
656-
.join("\n");
652+
const textResult = extractToolResultText(msg.content);
657653
const imageContent = model.input.includes("image")
658654
? msg.content.filter(
659655
(item): item is Extract<(typeof msg.content)[number], { type: "image" }> =>
660656
item.type === "image",
661657
)
662658
: [];
659+
const mediaPlaceholder = describeToolResultMediaPlaceholder(msg.content);
663660
const responseValue = textResult
664661
? sanitizeTransportPayloadText(textResult)
665-
: imageContent.length > 0
666-
? "(see attached image)"
667-
: "";
662+
: (mediaPlaceholder ?? "");
668663
const imageParts = imageContent.map((imageBlock) => ({
669664
inlineData: {
670665
mimeType: imageBlock.mimeType,

‎extensions/xai/stream.test.ts‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -625,4 +625,31 @@ describe("xai stream wrappers", () => {
625625
},
626626
]);
627627
});
628+
629+
it("uses audio fallback text for audio-only tool outputs", () => {
630+
const payload: Record<string, unknown> = {
631+
input: [
632+
{
633+
type: "function_call_output",
634+
call_id: "call_audio",
635+
output: [
636+
{
637+
type: "input_audio",
638+
mimeType: "audio/wav",
639+
data: "QUJDRA==",
640+
},
641+
],
642+
},
643+
],
644+
};
645+
runXaiToolPayloadWrapper({ payload, input: ["text"] });
646+
647+
expect(payload.input).toEqual([
648+
{
649+
type: "function_call_output",
650+
call_id: "call_audio",
651+
output: "(see attached audio)",
652+
},
653+
]);
654+
});
628655
});

‎extensions/xai/stream.ts‎

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,47 @@ function isReplayableInputImagePart(
115115
);
116116
}
117117

118+
function describeXaiFunctionOutputMediaPlaceholder(
119+
parts: Array<Record<string, unknown>>,
120+
): string | undefined {
121+
let hasImage = false;
122+
let hasAudio = false;
123+
let hasOtherMedia = false;
124+
125+
for (const part of parts) {
126+
const type = typeof part.type === "string" ? part.type : "";
127+
const mimeType =
128+
typeof part.mimeType === "string"
129+
? part.mimeType
130+
: typeof part.mime_type === "string"
131+
? part.mime_type
132+
: typeof part.mediaType === "string"
133+
? part.mediaType
134+
: typeof part.contentType === "string"
135+
? part.contentType
136+
: "";
137+
const normalizedMime = mimeType.toLowerCase();
138+
if (type.includes("image") || normalizedMime.startsWith("image/")) {
139+
hasImage = true;
140+
} else if (type.includes("audio") || normalizedMime.startsWith("audio/")) {
141+
hasAudio = true;
142+
} else if (type !== "input_text") {
143+
hasOtherMedia = true;
144+
}
145+
}
146+
147+
if ((hasImage && hasAudio) || hasOtherMedia) {
148+
return "(see attached media)";
149+
}
150+
if (hasAudio) {
151+
return "(see attached audio)";
152+
}
153+
if (hasImage) {
154+
return "(see attached image)";
155+
}
156+
return undefined;
157+
}
158+
118159
function normalizeXaiResponsesFunctionCallOutput(
119160
item: unknown,
120161
includeImages: boolean,
@@ -143,11 +184,12 @@ function normalizeXaiResponsesFunctionCallOutput(
143184
)
144185
: [];
145186
const hadNonTextParts = outputParts.some((part) => part.type !== "input_text");
187+
const mediaPlaceholder = describeXaiFunctionOutputMediaPlaceholder(outputParts);
146188

147189
return {
148190
normalizedItem: {
149191
...itemObj,
150-
output: textOutput || (hadNonTextParts ? "(see attached image)" : ""),
192+
output: textOutput || mediaPlaceholder || (hadNonTextParts ? "(see attached media)" : ""),
151193
},
152194
imageParts,
153195
};

‎scripts/plugin-sdk-surface-report.mjs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -202,8 +202,8 @@ let publicDeprecatedExportsByEntrypointBudget;
202202
try {
203203
budgets = {
204204
publicEntrypoints: readBudgetEnv("OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_ENTRYPOINTS", 322),
205-
publicExports: readBudgetEnv("OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_EXPORTS", 10402),
206-
publicFunctionExports: readBudgetEnv("OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_FUNCTION_EXPORTS", 5220),
205+
publicExports: readBudgetEnv("OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_EXPORTS", 10404),
206+
publicFunctionExports: readBudgetEnv("OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_FUNCTION_EXPORTS", 5222),
207207
publicDeprecatedExports: readBudgetEnv(
208208
"OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_DEPRECATED_EXPORTS",
209209
3261,

0 commit comments

Comments
 (0)