Skip to content

Commit eb7a082

Browse files
vincentkocvincentkoc
committed
fix(providers): harden image response schemas
1 parent 14142a9 commit eb7a082

13 files changed

Lines changed: 477 additions & 100 deletions

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ Docs: https://docs.openclaw.ai
2424
- Media/files: sniff `input_file` bytes before trusting declared MIME headers, rejecting spoofed image or zip payloads before they become agent-visible text.
2525
- Config persistence: ignore malformed array/scalar auth profile, cron job state, and session store entries instead of hydrating them into numeric profile ids, crashed cron rows, or invalid session records.
2626
- Providers: reject malformed successful Runway, BytePlus, and Ollama embedding responses with provider-owned errors instead of raw parser/type failures, silent bad vectors, or long bogus polling.
27+
- Providers/images: reject malformed successful OpenAI-compatible, OpenAI, Google, fal, and OpenRouter image responses with provider-owned errors instead of raw shape failures, silent invalid base64 skips, or empty image results.
2728
- Trajectory export: skip and report malformed session/runtime JSONL rows in `manifest.json` instead of letting wrong-shaped session rows crash support bundle export.
2829
- Voice calls: persist rejected inbound-call replay keys so duplicate carrier webhook retries stay ignored after a Gateway restart.
2930
- Config/doctor: copy fallback-enabled channel `allowFrom` entries into explicit `groupAllowFrom` allowlists during `openclaw doctor --fix`, preserving current group access without adding runtime fallback-transition flags.

‎extensions/fal/image-generation-provider.test.ts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,35 @@ describe("fal image-generation provider", () => {
116116
});
117117
});
118118

119+
it("wraps wrong-shape successful fal image responses", async () => {
120+
vi.spyOn(providerAuth, "resolveApiKeyForProvider").mockResolvedValue({
121+
apiKey: "fal-test-key",
122+
source: "env",
123+
mode: "api-key",
124+
});
125+
_setFalFetchGuardForTesting(fetchWithSsrFGuardMock);
126+
fetchWithSsrFGuardMock.mockResolvedValueOnce({
127+
response: new Response(
128+
JSON.stringify({ images: { url: "https://example.test/image.png" } }),
129+
{
130+
status: 200,
131+
headers: { "Content-Type": "application/json" },
132+
},
133+
),
134+
release: vi.fn(async () => {}),
135+
});
136+
137+
const provider = buildFalImageGenerationProvider();
138+
await expect(
139+
provider.generateImage({
140+
provider: "fal",
141+
model: "fal-ai/flux/dev",
142+
prompt: "draw a cat",
143+
cfg: {},
144+
}),
145+
).rejects.toThrow("fal image generation response malformed");
146+
});
147+
119148
it("uses image-to-image endpoint and data-uri input for edits", async () => {
120149
vi.spyOn(providerAuth, "resolveApiKeyForProvider").mockResolvedValue({
121150
apiKey: "fal-test-key",

‎extensions/fal/image-generation-provider.ts‎

Lines changed: 38 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,10 @@ import {
2020
type SsrFPolicy,
2121
ssrfPolicyFromDangerouslyAllowPrivateNetwork,
2222
} from "openclaw/plugin-sdk/ssrf-runtime";
23-
import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/string-coerce-runtime";
23+
import {
24+
normalizeLowercaseStringOrEmpty,
25+
normalizeOptionalString,
26+
} from "openclaw/plugin-sdk/string-coerce-runtime";
2427

2528
const DEFAULT_FAL_BASE_URL = "https://fal.run";
2629
const DEFAULT_FAL_IMAGE_MODEL = "fal-ai/flux/dev";
@@ -38,15 +41,7 @@ const FAL_SUPPORTED_SIZES = [
3841
] as const;
3942
const FAL_SUPPORTED_ASPECT_RATIOS = ["1:1", "4:3", "3:4", "16:9", "9:16"] as const;
4043

41-
type FalGeneratedImage = {
42-
url?: string;
43-
content_type?: string;
44-
};
45-
46-
type FalImageGenerationResponse = {
47-
images?: FalGeneratedImage[];
48-
prompt?: string;
49-
};
44+
const FAL_IMAGE_MALFORMED_RESPONSE = "fal image generation response malformed";
5045

5146
type FalImageSize = string | { width: number; height: number };
5247
type FalNetworkPolicy = {
@@ -67,6 +62,34 @@ function matchesTrustedHostSuffix(hostname: string, trustedSuffix: string): bool
6762
return normalizedHost === normalizedSuffix || normalizedHost.endsWith(`.${normalizedSuffix}`);
6863
}
6964

65+
function isRecord(value: unknown): value is Record<string, unknown> {
66+
return Boolean(value && typeof value === "object" && !Array.isArray(value));
67+
}
68+
69+
function parseFalImageGenerationResponse(payload: unknown): {
70+
images: Record<string, unknown>[];
71+
prompt?: string;
72+
} {
73+
if (!isRecord(payload)) {
74+
throw new Error(FAL_IMAGE_MALFORMED_RESPONSE);
75+
}
76+
const rawImages = payload.images;
77+
if (rawImages === undefined || rawImages === null) {
78+
return { images: [], prompt: normalizeOptionalString(payload.prompt) };
79+
}
80+
if (!Array.isArray(rawImages)) {
81+
throw new Error(FAL_IMAGE_MALFORMED_RESPONSE);
82+
}
83+
const images: Record<string, unknown>[] = [];
84+
for (const entry of rawImages) {
85+
if (!isRecord(entry)) {
86+
throw new Error(FAL_IMAGE_MALFORMED_RESPONSE);
87+
}
88+
images.push(entry);
89+
}
90+
return { images, prompt: normalizeOptionalString(payload.prompt) };
91+
}
92+
7093
function resolveFalNetworkPolicy(params: {
7194
baseUrl: string;
7295
allowPrivateNetwork: boolean;
@@ -404,21 +427,21 @@ export function buildFalImageGenerationProvider(): ImageGenerationProvider {
404427
try {
405428
await assertOkOrThrowHttpError(response, "fal image generation failed");
406429

407-
const payload = (await response.json()) as FalImageGenerationResponse;
430+
const payload = parseFalImageGenerationResponse(await response.json());
408431
const images: GeneratedImageAsset[] = [];
409432
let imageIndex = 0;
410-
for (const entry of payload.images ?? []) {
411-
const url = entry.url?.trim();
433+
for (const entry of payload.images) {
434+
const url = normalizeOptionalString(entry.url);
412435
if (!url) {
413-
continue;
436+
throw new Error(FAL_IMAGE_MALFORMED_RESPONSE);
414437
}
415438
const downloaded = await fetchImageBuffer(url, networkPolicy);
416439
imageIndex += 1;
417440
images.push({
418441
buffer: downloaded.buffer,
419442
mimeType: downloaded.mimeType,
420443
fileName: `image-${imageIndex}.${imageFileExtensionForMimeType(
421-
downloaded.mimeType || entry.content_type,
444+
downloaded.mimeType || normalizeOptionalString(entry.content_type),
422445
)}`,
423446
});
424447
}

‎extensions/google/image-generation-provider.test.ts‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,56 @@ describe("Google image-generation provider", () => {
203203
});
204204
});
205205

206+
it("wraps wrong-shape successful Gemini image responses", async () => {
207+
mockGoogleApiKeyAuth();
208+
vi.stubGlobal(
209+
"fetch",
210+
vi.fn().mockResolvedValue({
211+
ok: true,
212+
json: async () => ({ candidates: { content: { parts: [] } } }),
213+
}),
214+
);
215+
216+
const provider = buildGoogleImageGenerationProvider();
217+
await expect(
218+
provider.generateImage({
219+
provider: "google",
220+
model: "gemini-3.1-flash-image-preview",
221+
prompt: "draw a cat",
222+
cfg: {},
223+
}),
224+
).rejects.toThrow("Google image generation response malformed");
225+
});
226+
227+
it("rejects invalid inline image data in successful Gemini responses", async () => {
228+
mockGoogleApiKeyAuth();
229+
vi.stubGlobal(
230+
"fetch",
231+
vi.fn().mockResolvedValue({
232+
ok: true,
233+
json: async () => ({
234+
candidates: [
235+
{
236+
content: {
237+
parts: [{ inlineData: { mimeType: "image/png", data: "not-base64!" } }],
238+
},
239+
},
240+
],
241+
}),
242+
}),
243+
);
244+
245+
const provider = buildGoogleImageGenerationProvider();
246+
await expect(
247+
provider.generateImage({
248+
provider: "google",
249+
model: "gemini-3.1-flash-image-preview",
250+
prompt: "draw a cat",
251+
cfg: {},
252+
}),
253+
).rejects.toThrow("Google image generation response malformed");
254+
});
255+
206256
it("accepts OAuth JSON auth and inline_data responses", async () => {
207257
vi.spyOn(providerAuthRuntime, "resolveApiKeyForProvider").mockResolvedValue({
208258
apiKey: JSON.stringify({ token: "oauth-token" }),

‎extensions/google/image-generation-provider.ts‎

Lines changed: 89 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,19 @@
1-
import type { ImageGenerationProvider } from "openclaw/plugin-sdk/image-generation";
2-
import { extensionForMime } from "openclaw/plugin-sdk/media-mime";
1+
import {
2+
generatedImageAssetFromBase64,
3+
type GeneratedImageAsset,
4+
type ImageGenerationProvider,
5+
} from "openclaw/plugin-sdk/image-generation";
36
import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth";
47
import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime";
58
import {
69
assertOkOrThrowHttpError,
710
postJsonRequest,
811
sanitizeConfiguredModelProviderRequest,
912
} from "openclaw/plugin-sdk/provider-http";
10-
import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/string-coerce-runtime";
13+
import {
14+
normalizeLowercaseStringOrEmpty,
15+
normalizeOptionalString,
16+
} from "openclaw/plugin-sdk/string-coerce-runtime";
1117
import { normalizeGoogleModelId, resolveGoogleGenerativeAiHttpRequestConfig } from "./api.js";
1218

1319
const DEFAULT_GOOGLE_IMAGE_MODEL = "gemini-3.1-flash-image-preview";
@@ -32,23 +38,11 @@ const GOOGLE_SUPPORTED_ASPECT_RATIOS = [
3238
"21:9",
3339
] as const;
3440

35-
type GoogleInlineDataPart = {
36-
mimeType?: string;
37-
mime_type?: string;
38-
data?: string;
39-
};
40-
41-
type GoogleGenerateImageResponse = {
42-
candidates?: Array<{
43-
content?: {
44-
parts?: Array<{
45-
text?: string;
46-
inlineData?: GoogleInlineDataPart;
47-
inline_data?: GoogleInlineDataPart;
48-
}>;
49-
};
50-
}>;
51-
};
41+
const GOOGLE_IMAGE_MALFORMED_RESPONSE = "Google image generation response malformed";
42+
43+
function isRecord(value: unknown): value is Record<string, unknown> {
44+
return Boolean(value && typeof value === "object" && !Array.isArray(value));
45+
}
5246

5347
function normalizeGoogleImageModel(model: string | undefined): string {
5448
const trimmed = model?.trim();
@@ -89,6 +83,56 @@ function mapSizeToImageConfig(
8983
};
9084
}
9185

86+
function googleResponseParts(payload: unknown): unknown[] {
87+
if (!isRecord(payload)) {
88+
throw new Error(GOOGLE_IMAGE_MALFORMED_RESPONSE);
89+
}
90+
const candidates = payload.candidates;
91+
if (candidates === undefined || candidates === null) {
92+
return [];
93+
}
94+
if (!Array.isArray(candidates)) {
95+
throw new Error(GOOGLE_IMAGE_MALFORMED_RESPONSE);
96+
}
97+
98+
const parts: unknown[] = [];
99+
for (const candidate of candidates) {
100+
if (!isRecord(candidate)) {
101+
throw new Error(GOOGLE_IMAGE_MALFORMED_RESPONSE);
102+
}
103+
const content = candidate.content;
104+
if (content === undefined || content === null) {
105+
continue;
106+
}
107+
if (!isRecord(content)) {
108+
throw new Error(GOOGLE_IMAGE_MALFORMED_RESPONSE);
109+
}
110+
const candidateParts = content.parts;
111+
if (candidateParts === undefined || candidateParts === null) {
112+
continue;
113+
}
114+
if (!Array.isArray(candidateParts)) {
115+
throw new Error(GOOGLE_IMAGE_MALFORMED_RESPONSE);
116+
}
117+
parts.push(...candidateParts);
118+
}
119+
return parts;
120+
}
121+
122+
function googleInlineDataFromPart(part: unknown): Record<string, unknown> | undefined {
123+
if (!isRecord(part)) {
124+
throw new Error(GOOGLE_IMAGE_MALFORMED_RESPONSE);
125+
}
126+
const inline = part.inlineData ?? part.inline_data;
127+
if (inline === undefined || inline === null) {
128+
return undefined;
129+
}
130+
if (!isRecord(inline)) {
131+
throw new Error(GOOGLE_IMAGE_MALFORMED_RESPONSE);
132+
}
133+
return inline;
134+
}
135+
92136
export function buildGoogleImageGenerationProvider(): ImageGenerationProvider {
93137
return {
94138
id: "google",
@@ -184,26 +228,32 @@ export function buildGoogleImageGenerationProvider(): ImageGenerationProvider {
184228
try {
185229
await assertOkOrThrowHttpError(res, "Google image generation failed");
186230

187-
const payload = (await res.json()) as GoogleGenerateImageResponse;
231+
const payload = await res.json();
188232
let imageIndex = 0;
189-
const images = (payload.candidates ?? [])
190-
.flatMap((candidate) => candidate.content?.parts ?? [])
191-
.map((part) => {
192-
const inline = part.inlineData ?? part.inline_data;
193-
const data = inline?.data?.trim();
194-
if (!data) {
195-
return null;
196-
}
197-
const mimeType = inline?.mimeType ?? inline?.mime_type ?? DEFAULT_OUTPUT_MIME;
198-
const extension = extensionForMime(mimeType)?.slice(1) ?? "png";
199-
imageIndex += 1;
200-
return {
201-
buffer: Buffer.from(data, "base64"),
202-
mimeType,
203-
fileName: `image-${imageIndex}.${extension}`,
204-
};
205-
})
206-
.filter((entry): entry is NonNullable<typeof entry> => entry !== null);
233+
const images: GeneratedImageAsset[] = [];
234+
for (const part of googleResponseParts(payload)) {
235+
const inline = googleInlineDataFromPart(part);
236+
if (!inline) {
237+
continue;
238+
}
239+
const data = normalizeOptionalString(inline.data);
240+
if (!data) {
241+
throw new Error(GOOGLE_IMAGE_MALFORMED_RESPONSE);
242+
}
243+
const image = generatedImageAssetFromBase64({
244+
base64: data,
245+
index: imageIndex,
246+
mimeType:
247+
normalizeOptionalString(inline.mimeType) ??
248+
normalizeOptionalString(inline.mime_type) ??
249+
DEFAULT_OUTPUT_MIME,
250+
});
251+
if (!image) {
252+
throw new Error(GOOGLE_IMAGE_MALFORMED_RESPONSE);
253+
}
254+
imageIndex += 1;
255+
images.push(image);
256+
}
207257

208258
if (images.length === 0) {
209259
throw new Error("Google image generation response missing image data");

‎extensions/openai/image-generation-provider.test.ts‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -437,6 +437,23 @@ describe("openai image generation provider", () => {
437437
expect(jsonRequestCall().ssrfPolicy).toEqual({ allowRfc2544BenchmarkRange: true });
438438
});
439439

440+
it("wraps malformed successful OpenAI image responses", async () => {
441+
postJsonRequestMock.mockResolvedValue({
442+
response: { json: async () => ({ data: { b64_json: "not-an-array" } }) },
443+
release: vi.fn(async () => {}),
444+
});
445+
446+
const provider = buildOpenAIImageGenerationProvider();
447+
await expect(
448+
provider.generateImage({
449+
provider: "openai",
450+
model: "gpt-image-2",
451+
prompt: "bad shape",
452+
cfg: {},
453+
}),
454+
).rejects.toThrow("OpenAI image generation response malformed");
455+
});
456+
440457
it("forwards generation count and custom size overrides", async () => {
441458
mockGeneratedPngResponse();
442459

0 commit comments

Comments
 (0)