Skip to content

Commit 506ac20

Browse files
authored
fix(ai): request inline image bytes by default (#59947)
1 parent 71d3556 commit 506ac20

9 files changed

Lines changed: 77 additions & 15 deletions

File tree

‎packages/ai/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -791,7 +791,7 @@ const xai = Image.generate({
791791
aspect: "16:9",
792792
resolution: "1K",
793793
providerOptions: {
794-
responseFormat: "b64_json",
794+
responseFormat: "url", // defaults to "b64_json" (inline bytes)
795795
future_option: true,
796796
},
797797
http,

‎packages/ai/src/protocols/fal-images.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ export type FalImageOptions = {
2525
/** Inpainting mask, sent as `mask_url`. */
2626
readonly mask?: Media.Asset
2727
readonly enable_safety_checker?: boolean
28+
/** Defaults to `true`, which returns images inline as data URIs; `false` returns hosted URLs. */
29+
readonly sync_mode?: boolean
2830
} & Record<string, unknown>
2931

3032
export type Request = ImageRequestFor<FalImageOptions>
@@ -96,6 +98,7 @@ const fromRequest = Effect.fn("FalImages.fromRequest")(function* (request: Reque
9698
image_urls: list && images.length > 0 ? images : undefined,
9799
image_url: list ? undefined : images[0],
98100
mask_url: mask === undefined ? undefined : yield* FalQueue.mediaUrl(mask, route.name),
101+
sync_mode: true,
99102
},
100103
native,
101104
request.http?.body,

‎packages/ai/src/protocols/openai-images.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ const Settings = {
6464
}
6565

6666
const OpenAIImageResponse = Schema.Struct({
67-
// `response_format` picks `b64_json` or `url`; DALL·E and many compatible hosts default to `url`.
67+
// GPT Image models always return `b64_json`; compatible hosts that default to `url` can return either.
6868
data: Schema.Array(Schema.Struct({ b64_json: optionalNull(Schema.String), url: optionalNull(Schema.String) })),
6969
...Settings,
7070
usage: Schema.optional(Usage),
@@ -141,7 +141,13 @@ const RESERVED_FORM_FIELDS = new Set(["model", "prompt", "image", "image[]", "im
141141
const nativeOptions = (options: OpenAIImageOptions | undefined) => {
142142
if (!options) return undefined
143143
const { outputCompression, outputFormat, partialImages: _, mask: __, ...native } = options
144-
return { output_compression: outputCompression, output_format: outputFormat, ...native }
144+
// `response_format: null` drops a provider's default, for hosts that reject the field.
145+
return {
146+
output_compression: outputCompression,
147+
output_format: outputFormat,
148+
...native,
149+
response_format: native.response_format ?? undefined,
150+
}
145151
}
146152

147153
/**

‎packages/ai/src/protocols/xai-images.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ export const EDIT_PATH = "/images/edits"
2020

2121
/** Provider-native options. Common fields (`n`, `aspect`, `resolution`, `images`) live on the request. */
2222
export type XAIImageOptions = {
23+
/** Defaults to `b64_json`; `url` returns a short-lived hosted URL instead. */
2324
readonly responseFormat?: OpenString<"url" | "b64_json">
2425
readonly response_format?: OpenString<"url" | "b64_json">
2526
} & Record<string, unknown>
@@ -47,9 +48,8 @@ const XAIImageResponse = Schema.Struct({
4748
// ---------------------------------------------------------------------------
4849

4950
const nativeOptions = (options: XAIImageOptions | undefined) => {
50-
if (!options) return undefined
51-
const { responseFormat, ...native } = options
52-
return { response_format: responseFormat, ...native }
51+
const { responseFormat, ...native } = options ?? {}
52+
return { response_format: responseFormat ?? "b64_json", ...native }
5353
}
5454

5555
/** xAI renders `1k` or `2k`; other tiers use the nearest one. */

‎packages/ai/src/providers/openai-compatible.ts‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { ProviderID, type ModelID, type ProviderOptions } from "../schema/index.js"
1+
import { mergeProviderOptions, ProviderID, type ModelID, type ProviderOptions } from "../schema/index.js"
22
import { OpenAICompatibleChat } from "../protocols/openai-compatible-chat.js"
33
import { OpenAIImages } from "../protocols/openai-images.js"
44
import { MediaRoute } from "../route/media.js"
@@ -39,8 +39,18 @@ export const configure = (input: Config) => {
3939
return {
4040
id: ProviderID.make(provider),
4141
model: (modelID: string | ModelID) => route.model<OpenAIProviderOptionsInput>({ id: modelID }),
42+
// Many compatible hosts default to hosted URLs. GPT Image models always return bytes and reject the field; a
43+
// proxy serving one under another name can drop the default with `response_format: null`.
4244
image: (modelID: string | ModelID, providerOptions?: ProviderOptions) =>
43-
OpenAIImages.model({ ...media, id: modelID, provider, providerOptions }),
45+
OpenAIImages.model({
46+
...media,
47+
id: modelID,
48+
provider,
49+
providerOptions: mergeProviderOptions(
50+
/gpt-image|chatgpt-image/.test(modelID) ? undefined : { response_format: "b64_json" },
51+
providerOptions,
52+
),
53+
}),
4454
configure,
4555
}
4656
}

‎packages/ai/test/fixtures/recordings/fal-images/generates-an-image.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
"headers": {
1919
"content-type": "application/json"
2020
},
21-
"body": "{\"prompt\":\"A simple flat black circle centered on a plain white background.\",\"image_size\":{\"width\":512,\"height\":512},\"output_format\":\"jpeg\"}"
21+
"body": "{\"prompt\":\"A simple flat black circle centered on a plain white background.\",\"image_size\":{\"width\":512,\"height\":512},\"sync_mode\":false,\"output_format\":\"jpeg\"}"
2222
},
2323
"response": {
2424
"status": 200,

‎packages/ai/test/image.test.ts‎

Lines changed: 48 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import {
77
Fal,
88
Google,
99
OpenAI,
10+
OpenAICompatible,
1011
Replicate,
1112
Stability,
1213
VercelAIGateway,
@@ -169,6 +170,39 @@ describe("Image", () => {
169170
}),
170171
)
171172

173+
it.effect("requests inline bytes by default where hosts would return URLs", () =>
174+
Effect.gen(function* () {
175+
const bodies: Array<unknown> = []
176+
const compatible = OpenAICompatible.configure({ apiKey: "test", baseURL: "https://compatible.test/v1" })
177+
const xai = XAI.configure({ apiKey: "test", baseURL: "https://api.xai.test/v1" })
178+
const generate = (model: Parameters<typeof Image.generate>[0]["model"], providerOptions?: Record<string, unknown>) =>
179+
Image.generate({ model, prompt: "A lighthouse", providerOptions }).pipe(
180+
Effect.provide(
181+
layer((input) => {
182+
bodies.push(JSON.parse(input.text))
183+
return Effect.succeed(json(input, { data: [{ b64_json: "AQID" }] }))
184+
}),
185+
),
186+
)
187+
188+
yield* generate(compatible.image("flux-schnell"))
189+
yield* generate(compatible.image("flux-schnell"), { response_format: "url" })
190+
yield* generate(compatible.image("openai/gpt-image-2"))
191+
yield* generate(compatible.image("image-alias"), { response_format: null })
192+
yield* generate(xai.image("grok-imagine-image"))
193+
yield* generate(xai.image("grok-imagine-image"), { responseFormat: "url" })
194+
195+
expect(bodies).toEqual([
196+
{ model: "flux-schnell", prompt: "A lighthouse", response_format: "b64_json" },
197+
{ model: "flux-schnell", prompt: "A lighthouse", response_format: "url" },
198+
{ model: "openai/gpt-image-2", prompt: "A lighthouse" },
199+
{ model: "image-alias", prompt: "A lighthouse" },
200+
{ model: "grok-imagine-image", prompt: "A lighthouse", response_format: "b64_json" },
201+
{ model: "grok-imagine-image", prompt: "A lighthouse", response_format: "url" },
202+
])
203+
}),
204+
)
205+
172206
describe("OpenAI-style url results", () => {
173207
const generate = (data: unknown) =>
174208
Image.generate({
@@ -457,6 +491,7 @@ describe("Image", () => {
457491
{ url: "https://example.test/source.jpg", type: "image_url" },
458492
{ file_id: "file_123" },
459493
],
494+
response_format: "b64_json",
460495
})
461496
return Effect.succeed(
462497
input.respond(JSON.stringify({ data: [{ b64_json: "AQID", mime_type: "image/png" }] }), {
@@ -484,6 +519,7 @@ describe("Image", () => {
484519
model: "future-model",
485520
prompt: "Edit this image",
486521
image: { file_id: "file_123" },
522+
response_format: "b64_json",
487523
})
488524
return Effect.succeed(
489525
input.respond(JSON.stringify({ data: [{ b64_json: "AQID", mime_type: "image/png" }] }), {
@@ -945,12 +981,21 @@ describe("Image", () => {
945981
images: images.slice(0, 1),
946982
aspect: "16:9",
947983
})
948-
yield* Image.start({ model: fal.image("fal-ai/flux-pro/kontext/max/multi"), prompt: "A lighthouse", images })
984+
yield* Image.start({
985+
model: fal.image("fal-ai/flux-pro/kontext/max/multi"),
986+
prompt: "A lighthouse",
987+
images,
988+
providerOptions: { sync_mode: false },
989+
})
949990

950991
expect(rejected.reason).toMatchObject({ _tag: "UnsupportedOperation", operation: "media.resolution" })
951992
expect(bodies).toEqual([
952-
{ prompt: "A lighthouse", aspect_ratio: "16:9", image_url: "https://example.test/a.png" },
953-
{ prompt: "A lighthouse", image_urls: ["https://example.test/a.png", "https://example.test/b.png"] },
993+
{ prompt: "A lighthouse", aspect_ratio: "16:9", image_url: "https://example.test/a.png", sync_mode: true },
994+
{
995+
prompt: "A lighthouse",
996+
image_urls: ["https://example.test/a.png", "https://example.test/b.png"],
997+
sync_mode: false,
998+
},
954999
])
9551000
}).pipe(
9561001
Effect.provide(

‎packages/ai/test/provider/fal-images.recorded.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ describe("fal Images recorded", () => {
2828
prompt: "A simple flat black circle centered on a plain white background.",
2929
aspect: "1:1",
3030
resolution: "0.5K",
31-
providerOptions: { output_format: "jpeg" },
31+
providerOptions: { output_format: "jpeg", sync_mode: false },
3232
},
3333
{ poll: queuedPoll },
3434
)

‎packages/ai/test/provider/xai-images.recorded.test.ts‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ describe("xAI Images recorded", () => {
2424
prompt: "A simple flat black diamond centered on a plain white background.",
2525
aspect: "1:1",
2626
resolution: "1K",
27-
providerOptions: { responseFormat: "b64_json" },
2827
})
2928

3029
expect(response.images).toHaveLength(1)
@@ -46,7 +45,6 @@ describe("xAI Images recorded", () => {
4645
],
4746
aspect: "1:1",
4847
resolution: "1K",
49-
providerOptions: { responseFormat: "b64_json" },
5048
})
5149

5250
expect(response.image.mediaType).toMatch(/^image\/(jpeg|png)$/)

0 commit comments

Comments
 (0)