Skip to content

Commit 02acb0b

Browse files
yetvalvincentkoc
yetval
and
vincentkoc
authored
fix(gateway): accept file-only input on /v1/responses (parity with image-only) (#93011)
* fix(gateway): accept file-only input on /v1/responses (parity with image-only) * fix(gateway): preserve file-only model context --------- Co-authored-by: Vincent Koc <[email protected]>
1 parent a1f18ef commit 02acb0b

5 files changed

Lines changed: 241 additions & 7 deletions

‎src/gateway/openresponses-http.server.file-only.test.ts‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
2+
3+
const extractFileContentFromSourceMock = vi.fn();
4+
5+
vi.mock("../media/input-files.js", async () => {
6+
const actual =
7+
await vi.importActual<typeof import("../media/input-files.js")>("../media/input-files.js");
8+
return {
9+
...actual,
10+
extractFileContentFromSource: (...args: unknown[]) => extractFileContentFromSourceMock(...args),
11+
};
12+
});
13+
14+
import {
15+
agentCommand,
16+
getFreePort,
17+
installGatewayTestHooks,
18+
startGatewayServerWithRetries,
19+
} from "./test-helpers.js";
20+
21+
installGatewayTestHooks({ scope: "suite" });
22+
23+
let server: Awaited<ReturnType<typeof startGatewayServerWithRetries>>["server"];
24+
let port: number;
25+
26+
beforeAll(async () => {
27+
const started = await startGatewayServerWithRetries({
28+
port: await getFreePort(),
29+
opts: {
30+
host: "127.0.0.1",
31+
auth: { mode: "none" },
32+
controlUiEnabled: false,
33+
openResponsesEnabled: true,
34+
},
35+
});
36+
port = started.port;
37+
server = started.server;
38+
});
39+
40+
afterAll(async () => {
41+
await server?.close({ reason: "openresponses file-only suite done" });
42+
});
43+
44+
beforeEach(() => {
45+
vi.clearAllMocks();
46+
});
47+
48+
async function postResponses(body: unknown) {
49+
return await fetch(`http://127.0.0.1:${port}/v1/responses`, {
50+
method: "POST",
51+
headers: {
52+
"content-type": "application/json",
53+
"x-openclaw-scopes": "operator.write",
54+
},
55+
body: JSON.stringify(body),
56+
});
57+
}
58+
59+
describe("OpenResponses file-only input that renders to images", () => {
60+
it("accepts a file-only turn whose file renders to images and forwards them", async () => {
61+
extractFileContentFromSourceMock.mockResolvedValueOnce({
62+
filename: "scan.pdf",
63+
text: "",
64+
images: [
65+
{ type: "image", data: Buffer.alloc(8, 1).toString("base64"), mimeType: "image/png" },
66+
],
67+
});
68+
agentCommand.mockResolvedValueOnce({ payloads: [{ text: "ok" }] } as never);
69+
70+
const res = await postResponses({
71+
model: "openclaw",
72+
instructions: "Describe the attached scan.",
73+
input: [
74+
{
75+
type: "message",
76+
role: "user",
77+
content: [
78+
{
79+
type: "input_file",
80+
source: {
81+
type: "base64",
82+
media_type: "application/pdf",
83+
data: Buffer.from("%PDF-1.4 scanned").toString("base64"),
84+
filename: "scan.pdf",
85+
},
86+
},
87+
],
88+
},
89+
],
90+
});
91+
92+
expect(res.status).toBe(200);
93+
expect(agentCommand).toHaveBeenCalledTimes(1);
94+
const opts = agentCommand.mock.calls[0]?.[0] as { message?: string; images?: unknown[] };
95+
expect(opts.message ?? "").not.toBe("");
96+
expect(opts.images?.length).toBe(1);
97+
await res.text();
98+
});
99+
100+
it("keeps an empty extracted file visible to the model", async () => {
101+
extractFileContentFromSourceMock.mockResolvedValueOnce({
102+
filename: "empty.txt",
103+
text: "",
104+
images: [],
105+
});
106+
agentCommand.mockResolvedValueOnce({ payloads: [{ text: "ok" }] } as never);
107+
108+
const res = await postResponses({
109+
model: "openclaw",
110+
input: [
111+
{
112+
type: "message",
113+
role: "user",
114+
content: [
115+
{
116+
type: "input_file",
117+
source: {
118+
type: "base64",
119+
media_type: "text/plain",
120+
data: Buffer.from("binary-only file").toString("base64"),
121+
filename: "empty.txt",
122+
},
123+
},
124+
],
125+
},
126+
],
127+
});
128+
129+
const body = await res.text();
130+
expect(res.status, body).toBe(200);
131+
expect(agentCommand).toHaveBeenCalledTimes(1);
132+
const opts = agentCommand.mock.calls[0]?.[0] as { extraSystemPrompt?: string };
133+
expect(opts.extraSystemPrompt).toContain('<file name="empty.txt">');
134+
expect(opts.extraSystemPrompt).toContain("[No extractable text]");
135+
});
136+
});

‎src/gateway/openresponses-http.test.ts‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1746,6 +1746,43 @@ describe("OpenResponses HTTP API (e2e)", () => {
17461746
await ensureResponseConsumed(res);
17471747
});
17481748

1749+
it("accepts file-only input without text, matching image-only", async () => {
1750+
const port = enabledPort;
1751+
agentCommand.mockClear();
1752+
agentCommand.mockResolvedValueOnce({ payloads: [{ text: "ok" }] } as never);
1753+
1754+
const res = await postResponses(port, {
1755+
model: "openclaw",
1756+
instructions: "Summarize the attached document.",
1757+
input: [
1758+
{
1759+
type: "message",
1760+
role: "user",
1761+
content: [
1762+
{
1763+
type: "input_file",
1764+
source: {
1765+
type: "base64",
1766+
media_type: "text/plain",
1767+
data: Buffer.from("the quick brown fox").toString("base64"),
1768+
filename: "doc.txt",
1769+
},
1770+
},
1771+
],
1772+
},
1773+
],
1774+
});
1775+
1776+
expect(res.status).toBe(200);
1777+
expect(agentCommand).toHaveBeenCalledTimes(1);
1778+
const opts = firstAgentOpts();
1779+
expect((opts as { message?: string }).message ?? "").not.toBe("");
1780+
const extraSystemPrompt = (opts as { extraSystemPrompt?: string }).extraSystemPrompt ?? "";
1781+
expect(extraSystemPrompt).toContain('<file name="doc.txt">');
1782+
expect(extraSystemPrompt).toContain("the quick brown fox");
1783+
await ensureResponseConsumed(res);
1784+
});
1785+
17491786
it("still rejects input with neither text nor image", async () => {
17501787
const port = enabledPort;
17511788
agentCommand.mockClear();

‎src/gateway/openresponses-http.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -612,6 +612,14 @@ export async function handleOpenResponsesHttpRequest(
612612
surroundContentWithNewlines: false,
613613
}),
614614
);
615+
} else {
616+
fileContexts.push(
617+
renderFileContextBlock({
618+
filename: file.filename,
619+
content: "[No extractable text]",
620+
surroundContentWithNewlines: false,
621+
}),
622+
);
615623
}
616624
if (file.images && file.images.length > 0) {
617625
images = images.concat(file.images);

‎src/gateway/openresponses-parity.test.ts‎

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -393,7 +393,43 @@ describe("OpenResponses Feature Parity", () => {
393393
expect(result.message).toBe(IMAGE_ONLY_USER_MESSAGE);
394394
});
395395

396-
it("keeps an empty message when the active turn has neither text nor image", () => {
396+
it("substitutes a placeholder for a file-only active user turn", () => {
397+
const result = buildAgentPrompt([
398+
{
399+
type: "message" as const,
400+
role: "user" as const,
401+
content: [
402+
{
403+
type: "input_file" as const,
404+
source: { type: "url" as const, url: "https://example.com/report.pdf" },
405+
},
406+
],
407+
},
408+
]);
409+
410+
expect(result.message).not.toBe("");
411+
expect(result.message.toLowerCase()).toContain("file");
412+
});
413+
414+
it("keeps the user text when a file-only turn also carries text", () => {
415+
const result = buildAgentPrompt([
416+
{
417+
type: "message" as const,
418+
role: "user" as const,
419+
content: [
420+
{ type: "input_text" as const, text: "summarize this" },
421+
{
422+
type: "input_file" as const,
423+
source: { type: "url" as const, url: "https://example.com/report.pdf" },
424+
},
425+
],
426+
},
427+
]);
428+
429+
expect(result.message).toBe("summarize this");
430+
});
431+
432+
it("keeps an empty message when the active turn has neither text, image, nor file", () => {
397433
const result = buildAgentPrompt([
398434
{ type: "message" as const, role: "user" as const, content: [] },
399435
]);

‎src/gateway/openresponses-prompt.ts‎

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ import {
66
} from "./agent-prompt.js";
77
import type { ContentPart, ItemParam } from "./open-responses.schema.js";
88

9+
const FILE_ONLY_USER_MESSAGE = "User sent file(s) with no text.";
10+
911
function extractTextContent(content: string | ContentPart[]): string {
1012
if (typeof content === "string") {
1113
return content;
@@ -28,6 +30,20 @@ function hasImageContent(content: string | ContentPart[]): boolean {
2830
return typeof content !== "string" && content.some((part) => part.type === "input_image");
2931
}
3032

33+
function hasFileContent(content: string | ContentPart[]): boolean {
34+
return typeof content !== "string" && content.some((part) => part.type === "input_file");
35+
}
36+
37+
function placeholderForActiveTurn(content: string | ContentPart[]): string {
38+
if (hasImageContent(content)) {
39+
return IMAGE_ONLY_USER_MESSAGE;
40+
}
41+
if (hasFileContent(content)) {
42+
return FILE_ONLY_USER_MESSAGE;
43+
}
44+
return "";
45+
}
46+
3147
/** Index of the last user message item, or -1 when there is none. */
3248
function findActiveUserMessageIndex(input: ItemParam[]): number {
3349
for (let i = input.length - 1; i >= 0; i -= 1) {
@@ -55,14 +71,15 @@ export function buildAgentPrompt(input: string | ItemParam[]): {
5571
for (const [i, item] of input.entries()) {
5672
if (item.type === "message") {
5773
const content = extractTextContent(item.content).trim();
58-
// Substitute a placeholder for an image-only active user turn so the turn
59-
// is not dropped and the downstream agent command (which requires non-empty
60-
// message text) still runs with the attached image, matching /v1/chat/completions.
61-
// Historical image-only turns stay skipped because their bytes are not replayed.
74+
// Substitute a placeholder for an image-only or file-only active user turn
75+
// so the turn is not dropped and the downstream agent command (which requires
76+
// non-empty message text) still runs with the attached image or file context,
77+
// matching /v1/chat/completions. Historical media-only turns stay skipped
78+
// because their bytes are not replayed.
6279
const body =
6380
content ||
64-
(item.role === "user" && i === activeUserMessageIndex && hasImageContent(item.content)
65-
? IMAGE_ONLY_USER_MESSAGE
81+
(item.role === "user" && i === activeUserMessageIndex
82+
? placeholderForActiveTurn(item.content)
6683
: "");
6784
if (!body) {
6885
continue;

0 commit comments

Comments
 (0)