fix(agents): preserve mixed image attachment order (#99902)

vincentkoc · web-flow · commit a0e604862470 · 2026-07-04T02:32:03.000-07:00
diff --git a/src/auto-reply/reply/current-turn-images.test.ts b/src/auto-reply/reply/current-turn-images.test.ts
@@ -61,6 +61,59 @@ describe("resolveCurrentTurnImages", () => {
     });
   });
 
+  it("preserves the full order when only inline image payloads are present", async () => {
+    const inlineImage = {
+      type: "image" as const,
+      data: Buffer.from("inline").toString("base64"),
+      mimeType: "image/png",
+    };
+
+    const result = await resolveCurrentTurnImages({
+      ctx: { Body: "compare these" } satisfies MsgContext,
+      cfg: {} as OpenClawConfig,
+      images: [inlineImage],
+      imageOrder: ["offloaded", "inline", "offloaded"],
+    });
+
+    expect(result).toEqual({
+      images: [inlineImage],
+      imageOrder: ["offloaded", "inline", "offloaded"],
+    });
+  });
+
+  it("preserves all-offloaded image order without inline payloads", async () => {
+    const result = await resolveCurrentTurnImages({
+      ctx: { Body: "compare these" } satisfies MsgContext,
+      cfg: {} as OpenClawConfig,
+      images: [],
+      imageOrder: ["offloaded", "offloaded"],
+    });
+
+    expect(result).toEqual({
+      imageOrder: ["offloaded", "offloaded"],
+    });
+  });
+
+  it("preserves interleaved offloaded slots around inline image payloads", async () => {
+    const inlineImages = ["first", "second"].map((data) => ({
+      type: "image" as const,
+      data: Buffer.from(data).toString("base64"),
+      mimeType: "image/png",
+    }));
+
+    const result = await resolveCurrentTurnImages({
+      ctx: { Body: "compare these" } satisfies MsgContext,
+      cfg: {} as OpenClawConfig,
+      images: inlineImages,
+      imageOrder: ["inline", "offloaded", "inline"],
+    });
+
+    expect(result).toEqual({
+      images: inlineImages,
+      imageOrder: ["inline", "offloaded", "inline"],
+    });
+  });
+
   it("appends extracted PDF page images without dropping current image attachments", async () => {
     await withTempDir({ prefix: "openclaw-current-turn-pdf-images-" }, async (base) => {
       const imagePath = path.join(base, "photo.png");
diff --git a/src/auto-reply/reply/current-turn-images.ts b/src/auto-reply/reply/current-turn-images.ts
@@ -20,7 +20,7 @@ type CurrentImageAttachment = {
 };
 
 type OrderedTurnImage = {
-  image: ImageContent;
+  image?: ImageContent;
   imageOrder: PromptImageOrderEntry;
   sourceIndex?: number;
   sequence: number;
@@ -105,13 +105,32 @@ function appendOrderedImages(params: {
   imageOrder?: PromptImageOrderEntry[];
   sourceIndex?: number;
 }) {
-  if (!params.images || params.images.length === 0) {
+  const images = params.images ?? [];
+  if (!params.imageOrder || params.imageOrder.length === 0) {
+    for (const image of images) {
+      params.entries.push({
+        image,
+        imageOrder: "inline",
+        sourceIndex: params.sourceIndex,
+        sequence: params.entries.length,
+      });
+    }
     return;
   }
-  for (const [index, image] of params.images.entries()) {
+
+  let inlineIndex = 0;
+  for (const imageOrder of params.imageOrder) {
+    params.entries.push({
+      image: imageOrder === "inline" ? images[inlineIndex++] : undefined,
+      imageOrder,
+      sourceIndex: params.sourceIndex,
+      sequence: params.entries.length,
+    });
+  }
+  while (inlineIndex < images.length) {
     params.entries.push({
-      image,
-      imageOrder: params.imageOrder?.[index] ?? "inline",
+      image: images[inlineIndex++],
+      imageOrder: "inline",
       sourceIndex: params.sourceIndex,
       sequence: params.entries.length,
     });
@@ -134,8 +153,9 @@ function resolveMergedTurnImages(entries: OrderedTurnImage[]): {
     }
     return left.sequence - right.sequence;
   });
+  const images = merged.flatMap((entry) => (entry.image ? [entry.image] : []));
   return {
-    images: merged.map((entry) => entry.image),
+    ...(images.length > 0 ? { images } : {}),
     imageOrder: merged.map((entry) => entry.imageOrder),
   };
 }
