fix(gateway): pass image-only /v1/responses turns to the agent

songwendong · cursoragent · vincentkoc · commit fea5e51de020 · 2026-06-14T07:55:34.000+08:00
The one-line guard alone was insufficient: even after letting image-only
input past the `Missing user message` check, the downstream agent command
(`prepareAgentCommandExecution`) throws `Message (--message) is required`
for an empty message, so image-only `/v1/responses` returned 500.

Mirror the /v1/chat/completions prompt builder: substitute the shared
IMAGE_ONLY_USER_MESSAGE placeholder for the active image-only user turn so
the turn is not dropped and the real image is still attached via `images`.
Promote the placeholder constant to the shared gateway agent-prompt module
so both endpoints stay in sync, and revert the responses guard back to the
original `!prompt.message` check (responses images are not scoped to the
active turn, so the placeholder is the correct, single source of truth).

Co-authored-by: Cursor &lt;cursoragent@cursor.com&gt;
diff --git a/src/gateway/agent-prompt.ts b/src/gateway/agent-prompt.ts
@@ -10,6 +10,12 @@ export type ConversationEntry = {
   internalStreamError?: boolean;
 };
 
+// Placeholder user text for an image-only turn. The agent command requires a
+// non-empty message even when the real payload is the attached image, so both
+// the /v1/chat/completions and /v1/responses prompt builders substitute this
+// for the active user turn. Keep it shared so the two endpoints stay in sync.
+export const IMAGE_ONLY_USER_MESSAGE = "User sent image(s) with no text.";
+
 /**
  * Coerce body to string. Handles cases where body is a content array
  * (e.g. [{type:"text", text:"hello"}]) that would serialize as
diff --git a/src/gateway/openai-http.ts b/src/gateway/openai-http.ts
@@ -39,6 +39,7 @@ import { resolveAssistantStreamDeltaText } from "./agent-event-assistant-text.js
 import {
   buildAgentMessageFromConversationEntries,
   type ConversationEntry,
+  IMAGE_ONLY_USER_MESSAGE,
 } from "./agent-prompt.js";
 import type { AuthRateLimiter } from "./auth-rate-limit.js";
 import type { ResolvedGatewayAuth } from "./auth.js";
@@ -105,7 +106,6 @@ type OpenAiChatCompletionRequest = {
 };
 
 const DEFAULT_OPENAI_CHAT_COMPLETIONS_BODY_BYTES = 20 * 1024 * 1024;
-const IMAGE_ONLY_USER_MESSAGE = "User sent image(s) with no text.";
 const DEFAULT_OPENAI_MAX_IMAGE_PARTS = 8;
 const DEFAULT_OPENAI_MAX_TOTAL_IMAGE_BYTES = 20 * 1024 * 1024;
 const DEFAULT_OPENAI_IMAGE_LIMITS: InputImageLimits = {
diff --git a/src/gateway/openresponses-http.test.ts b/src/gateway/openresponses-http.test.ts
@@ -10,6 +10,7 @@ import { HISTORY_CONTEXT_MARKER } from "../auto-reply/reply/history.js";
 import { CURRENT_MESSAGE_MARKER } from "../auto-reply/reply/mentions.js";
 import { resetConfigRuntimeState } from "../config/config.js";
 import { emitAgentEvent } from "../infra/agent-events.js";
+import { IMAGE_ONLY_USER_MESSAGE } from "./agent-prompt.js";
 import { buildAssistantDeltaResult } from "./test-helpers.agent-results.js";
 import {
   agentCommand,
@@ -1738,7 +1739,9 @@ describe("OpenResponses HTTP API (e2e)", () => {
     expect(res.status).toBe(200);
     expect(agentCommand).toHaveBeenCalledTimes(1);
     const opts = firstAgentOpts();
-    expect((opts as { message?: string }).message ?? "").toBe("");
+    // Image-only turn carries a non-empty placeholder so the agent command runs,
+    // with the real image attached via `images` (parity with /v1/chat/completions).
+    expect((opts as { message?: string }).message ?? "").toBe(IMAGE_ONLY_USER_MESSAGE);
     expect((opts as { images?: unknown[] }).images?.length).toBe(1);
     await ensureResponseConsumed(res);
   });
diff --git a/src/gateway/openresponses-http.ts b/src/gateway/openresponses-http.ts
@@ -697,9 +697,7 @@ export async function handleOpenResponsesHttpRequest(
     .filter(Boolean)
     .join("\n\n");
 
-  // Image-only input is a valid user turn; require text only when no image accompanies it,
-  // matching the /v1/chat/completions guard (openai-http.ts).
-  if (!prompt.message && images.length === 0) {
+  if (!prompt.message) {
     sendJson(res, 400, {
       error: {
         message: "Missing user message in `input`.",
diff --git a/src/gateway/openresponses-parity.test.ts b/src/gateway/openresponses-parity.test.ts
@@ -6,6 +6,7 @@
  */
 
 import { beforeAll, describe, it, expect } from "vitest";
+import { IMAGE_ONLY_USER_MESSAGE } from "./agent-prompt.js";
 import { wrapUntrustedFileContent } from "./openresponses-file-content.js";
 
 let InputImageContentPartSchema: typeof import("./open-responses.schema.js").InputImageContentPartSchema;
@@ -374,6 +375,31 @@ describe("OpenResponses Feature Parity", () => {
       expect(result.message).toContain("72°F");
       expect(result.message).toContain("Thanks");
     });
+
+    it("substitutes a placeholder for an image-only active user turn", () => {
+      const result = buildAgentPrompt([
+        {
+          type: "message" as const,
+          role: "user" as const,
+          content: [
+            {
+              type: "input_image" as const,
+              source: { type: "url" as const, url: "https://example.com/cat.png" },
+            },
+          ],
+        },
+      ]);
+
+      expect(result.message).toBe(IMAGE_ONLY_USER_MESSAGE);
+    });
+
+    it("keeps an empty message when the active turn has neither text nor image", () => {
+      const result = buildAgentPrompt([
+        { type: "message" as const, role: "user" as const, content: [] },
+      ]);
+
+      expect(result.message).toBe("");
+    });
   });
 
   describe("input_file hardening", () => {
diff --git a/src/gateway/openresponses-prompt.ts b/src/gateway/openresponses-prompt.ts
@@ -2,6 +2,7 @@
 import {
   buildAgentMessageFromConversationEntries,
   type ConversationEntry,
+  IMAGE_ONLY_USER_MESSAGE,
 } from "./agent-prompt.js";
 import type { ContentPart, ItemParam } from "./open-responses.schema.js";
 
@@ -23,6 +24,21 @@ function extractTextContent(content: string | ContentPart[]): string {
     .join("\n");
 }
 
+function hasImageContent(content: string | ContentPart[]): boolean {
+  return typeof content !== "string" && content.some((part) => part.type === "input_image");
+}
+
+/** Index of the last user message item, or -1 when there is none. */
+function findActiveUserMessageIndex(input: ItemParam[]): number {
+  for (let i = input.length - 1; i >= 0; i -= 1) {
+    const item = input[i];
+    if (item?.type === "message" && item.role === "user") {
+      return i;
+    }
+  }
+  return -1;
+}
+
 /** Build the user message and optional system prompt from Responses API input. */
 export function buildAgentPrompt(input: string | ItemParam[]): {
   message: string;
@@ -34,16 +50,26 @@ export function buildAgentPrompt(input: string | ItemParam[]): {
 
   const systemParts: string[] = [];
   const conversationEntries: ConversationEntry[] = [];
+  const activeUserMessageIndex = findActiveUserMessageIndex(input);
 
-  for (const item of input) {
+  for (const [i, item] of input.entries()) {
     if (item.type === "message") {
       const content = extractTextContent(item.content).trim();
-      if (!content) {
+      // Substitute a placeholder for an image-only active user turn so the turn
+      // is not dropped and the downstream agent command (which requires non-empty
+      // message text) still runs with the attached image, matching /v1/chat/completions.
+      // Historical image-only turns stay skipped because their bytes are not replayed.
+      const body =
+        content ||
+        (item.role === "user" && i === activeUserMessageIndex && hasImageContent(item.content)
+          ? IMAGE_ONLY_USER_MESSAGE
+          : "");
+      if (!body) {
         continue;
       }
 
       if (item.role === "system" || item.role === "developer") {
-        systemParts.push(content);
+        systemParts.push(body);
         continue;
       }
 
@@ -52,7 +78,7 @@ export function buildAgentPrompt(input: string | ItemParam[]): {
 
       conversationEntries.push({
         role: normalizedRole,
-        entry: { sender, body: content },
+        entry: { sender, body },
       });
     } else if (item.type === "function_call_output") {
       conversationEntries.push({
