Skip to content

Commit 0690583

Browse files
HonorVanErHonorVanEr
committed
fix(media-understanding): forward scanned PDF page images to vision models on chat channels
1 parent 1a04b8e commit 0690583

6 files changed

Lines changed: 210 additions & 33 deletions

File tree

‎src/auto-reply/reply/current-turn-images.test.ts‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,4 +59,65 @@ describe("resolveCurrentTurnImages", () => {
5959
});
6060
});
6161
});
62+
63+
it("merges current-turn extracted images with existing inline images", async () => {
64+
const existingImage = {
65+
type: "image" as const,
66+
data: Buffer.from("existing-image").toString("base64"),
67+
mimeType: "image/jpeg",
68+
};
69+
const extractedPdfPage = {
70+
type: "image" as const,
71+
data: Buffer.from("pdf-page").toString("base64"),
72+
mimeType: "image/png",
73+
};
74+
75+
const result = await resolveCurrentTurnImages({
76+
ctx: {
77+
Body: "<media:document>",
78+
CurrentTurnImages: [extractedPdfPage],
79+
CurrentTurnImageOrder: ["inline"],
80+
} satisfies MsgContext,
81+
cfg: {} as OpenClawConfig,
82+
images: [existingImage],
83+
imageOrder: ["inline"],
84+
});
85+
86+
expect(result).toStrictEqual({
87+
images: [existingImage, extractedPdfPage],
88+
imageOrder: ["inline", "inline"],
89+
});
90+
});
91+
92+
it("does not merge extracted images twice across the prepare and run passes", async () => {
93+
const extractedPdfPage = {
94+
type: "image" as const,
95+
data: Buffer.from("pdf-page").toString("base64"),
96+
mimeType: "image/png",
97+
};
98+
const ctx = {
99+
Body: "<media:document>",
100+
CurrentTurnImages: [extractedPdfPage],
101+
CurrentTurnImageOrder: ["inline"],
102+
} satisfies MsgContext;
103+
104+
const prepared = await resolveCurrentTurnImages({
105+
ctx,
106+
cfg: {} as OpenClawConfig,
107+
});
108+
const runPass = await resolveCurrentTurnImages({
109+
ctx,
110+
cfg: {} as OpenClawConfig,
111+
images: prepared.images,
112+
imageOrder: prepared.imageOrder,
113+
});
114+
115+
expect(prepared).toStrictEqual({
116+
images: [extractedPdfPage],
117+
imageOrder: ["inline"],
118+
});
119+
expect(runPass).toStrictEqual(prepared);
120+
expect(ctx.CurrentTurnImages).toBeUndefined();
121+
expect(ctx.CurrentTurnImageOrder).toBeUndefined();
122+
});
62123
});

‎src/auto-reply/reply/current-turn-images.ts‎

Lines changed: 59 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,41 @@ function createUndescribedImageContext(
8989
}
9090

9191
/** Resolves current-turn image attachments that were not already described by media understanding. */
92+
export function takeCurrentTurnImages(ctx: MsgContext): {
93+
images?: ImageContent[];
94+
imageOrder?: PromptImageOrderEntry[];
95+
} {
96+
const images = ctx.CurrentTurnImages;
97+
if (images && images.length > 0) {
98+
const imageOrder = ctx.CurrentTurnImageOrder;
99+
delete ctx.CurrentTurnImages;
100+
delete ctx.CurrentTurnImageOrder;
101+
return { images, imageOrder };
102+
}
103+
return {};
104+
}
105+
106+
function mergeImagePayloads(params: {
107+
baseImages?: ImageContent[];
108+
baseImageOrder?: PromptImageOrderEntry[];
109+
nextImages?: ImageContent[];
110+
nextImageOrder?: PromptImageOrderEntry[];
111+
}): { images?: ImageContent[]; imageOrder?: PromptImageOrderEntry[] } {
112+
const nextImages = params.nextImages ?? [];
113+
if (nextImages.length === 0) {
114+
return { images: params.baseImages, imageOrder: params.baseImageOrder };
115+
}
116+
const images = [...(params.baseImages ?? []), ...nextImages];
117+
const nextOrder =
118+
params.nextImageOrder && params.nextImageOrder.length === nextImages.length
119+
? params.nextImageOrder
120+
: nextImages.map((): PromptImageOrderEntry => "inline");
121+
return {
122+
images,
123+
imageOrder: [...(params.baseImageOrder ?? []), ...nextOrder],
124+
};
125+
}
126+
92127
export async function resolveCurrentTurnImages(params: {
93128
ctx: MsgContext;
94129
cfg: OpenClawConfig;
@@ -98,30 +133,37 @@ export async function resolveCurrentTurnImages(params: {
98133
images?: ImageContent[];
99134
imageOrder?: PromptImageOrderEntry[];
100135
}> {
136+
const extracted = takeCurrentTurnImages(params.ctx);
137+
let merged = mergeImagePayloads({
138+
baseImages: params.images,
139+
baseImageOrder: params.imageOrder,
140+
nextImages: extracted.images,
141+
nextImageOrder: extracted.imageOrder,
142+
});
101143
if (Array.isArray(params.images) && params.images.length > 0) {
102-
return { images: params.images, imageOrder: params.imageOrder };
144+
return merged;
103145
}
104146

105147
const currentImageAttachments = collectCurrentImageAttachments(params.ctx);
106148
if (currentImageAttachments.length === 0) {
107-
return { images: params.images, imageOrder: params.imageOrder };
149+
return merged;
108150
}
109151
const describedImageIndexes = collectDescribedImageAttachmentIndexes(params.ctx);
110152
const undescribedImageAttachments = currentImageAttachments.filter(
111153
(attachment) => !describedImageIndexes.has(attachment.index),
112154
);
113155
if (undescribedImageAttachments.length === 0) {
114-
return { images: params.images, imageOrder: params.imageOrder };
156+
return merged;
115157
}
116158

117159
try {
118160
// Only send undescribed current images natively; described images already exist as text context.
119-
const resolved = await resolveAgentTurnAttachments({
161+
const attachmentResult = await resolveAgentTurnAttachments({
120162
ctx: createUndescribedImageContext(params.ctx, undescribedImageAttachments),
121163
cfg: params.cfg,
122164
includeRecentHistoryImages: false,
123165
});
124-
const images = resolved.attachments.map(
166+
const images = attachmentResult.attachments.map(
125167
(attachment): ImageContent => ({
126168
type: "image",
127169
data: attachment.data,
@@ -132,15 +174,22 @@ export async function resolveCurrentTurnImages(params: {
132174
logVerbose(
133175
`agent-runner: native OpenClaw media resolution produced ${images.length}/${undescribedImageAttachments.length} current image attachment(s); falling back to prompt image refs`,
134176
);
135-
return { images: params.images, imageOrder: params.imageOrder };
177+
return merged;
178+
}
179+
if (images.length === 0) {
180+
return merged;
136181
}
137-
return images.length > 0
138-
? { images, imageOrder: images.map(() => "inline" as const) }
139-
: { images: params.images, imageOrder: params.imageOrder };
182+
merged = mergeImagePayloads({
183+
baseImages: merged.images,
184+
baseImageOrder: merged.imageOrder,
185+
nextImages: images,
186+
nextImageOrder: images.map((): PromptImageOrderEntry => "inline"),
187+
});
188+
return merged;
140189
} catch (error) {
141190
logVerbose(
142191
`agent-runner: media attachment image resolution failed, proceeding without native images: ${formatErrorMessage(error)}`,
143192
);
144-
return { images: params.images, imageOrder: params.imageOrder };
193+
return merged;
145194
}
146195
}

‎src/auto-reply/reply/dispatch-acp.ts‎

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ import {
3737
resolveInlineAgentImageAttachments,
3838
} from "./agent-turn-attachments.js";
3939
import { resolveFirstContextText } from "./context-text.js";
40+
import { takeCurrentTurnImages } from "./current-turn-images.js";
4041
import {
4142
createAcpDispatchDeliveryCoordinator,
4243
type AcpDispatchDeliveryCoordinator,
@@ -563,22 +564,26 @@ export async function tryDispatchAcpReply(params: {
563564
cfg: params.cfg,
564565
});
565566
const mediaAttachments = resolvedTurnAttachments.attachments;
566-
const inlineAttachments = resolveInlineAgentImageAttachments(params.images);
567+
const inputInlineAttachments = resolveInlineAgentImageAttachments(params.images);
568+
const extractedInlineAttachments = resolveInlineAgentImageAttachments(
569+
takeCurrentTurnImages(params.ctx).images,
570+
);
571+
const inlineAttachments = [...inputInlineAttachments, ...extractedInlineAttachments];
567572
const mediaAttachmentsAreOnlyRecentHistory =
568573
mediaAttachments.length > 0 &&
569574
mediaAttachments.length === resolvedTurnAttachments.recentHistoryImages.length;
570-
const attachments =
575+
const shouldUseMediaAttachments =
571576
mediaAttachments.length > 0 &&
572-
!(mediaAttachmentsAreOnlyRecentHistory && inlineAttachments.length > 0)
573-
? mediaAttachments
574-
: inlineAttachments;
575-
const turnPromptText =
576-
attachments === mediaAttachments
577-
? appendRecentHistoryImageContext({
578-
promptText,
579-
images: resolvedTurnAttachments.recentHistoryImages,
580-
})
581-
: promptText;
577+
!(mediaAttachmentsAreOnlyRecentHistory && inlineAttachments.length > 0);
578+
const attachments = shouldUseMediaAttachments
579+
? [...mediaAttachments, ...extractedInlineAttachments]
580+
: inlineAttachments;
581+
const turnPromptText = shouldUseMediaAttachments
582+
? appendRecentHistoryImageContext({
583+
promptText,
584+
images: resolvedTurnAttachments.recentHistoryImages,
585+
})
586+
: promptText;
582587
if (!turnPromptText && attachments.length === 0) {
583588
const counts = params.dispatcher.getQueuedCounts();
584589
delivery.applyRoutedCounts(counts);

‎src/auto-reply/templating.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
/** Shared inbound message context types used by prompt templating and reply dispatch. */
22
import type { InboundEventKind } from "../channels/inbound-event/kind.js";
3+
import type { ImageContent } from "../llm/types.js";
4+
import type { PromptImageOrderEntry } from "../media/prompt-image-order.js";
35
import type {
46
MediaUnderstandingDecision,
57
MediaUnderstandingOutput,
@@ -219,6 +221,9 @@ export type MsgContext = {
219221
Transcript?: string;
220222
MediaUnderstanding?: MediaUnderstandingOutput[];
221223
MediaUnderstandingDecisions?: MediaUnderstandingDecision[];
224+
/** Page images extracted from scanned/image-only PDFs during media understanding, forwarded to vision models. */
225+
CurrentTurnImages?: ImageContent[];
226+
CurrentTurnImageOrder?: PromptImageOrderEntry[];
222227
LinkUnderstanding?: string[];
223228
Prompt?: string;
224229
MaxChars?: number;

‎src/media-understanding/apply.test.ts‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ const readRemoteMediaBufferMock = vi.hoisted(() => vi.fn());
3131
const runFfmpegMock = vi.hoisted(() => vi.fn());
3232
const convertHeicToJpegMock = vi.hoisted(() => vi.fn());
3333
const runExecMock = vi.hoisted(() => vi.fn());
34+
const extractDocumentContentMock = vi.hoisted(() => vi.fn());
3435

3536
let applyMediaUnderstanding: typeof import("./apply.js").applyMediaUnderstanding;
3637
let clearMediaUnderstandingBinaryCacheForTests: typeof import("./runner.js").clearMediaUnderstandingBinaryCacheForTests;
@@ -39,6 +40,7 @@ const mockedReadRemoteMediaBuffer = readRemoteMediaBufferMock;
3940
const mockedRunFfmpeg = runFfmpegMock;
4041
const mockedConvertHeicToJpeg = convertHeicToJpegMock;
4142
const mockedRunExec = runExecMock;
43+
const mockedExtractDocumentContent = extractDocumentContentMock;
4244

4345
const TEMP_MEDIA_PREFIX = "openclaw-media-";
4446
let suiteTempMediaRootDir = "";
@@ -301,6 +303,9 @@ describe("applyMediaUnderstanding", () => {
301303
vi.doMock("../process/exec.js", () => ({
302304
runExec: runExecMock,
303305
}));
306+
vi.doMock("../media/document-extractors.runtime.js", () => ({
307+
extractDocumentContent: extractDocumentContentMock,
308+
}));
304309
vi.doMock("./provider-registry.js", async () => {
305310
const actual =
306311
await vi.importActual<typeof import("./provider-registry.js")>("./provider-registry.js");
@@ -352,6 +357,8 @@ describe("applyMediaUnderstanding", () => {
352357
mockedConvertHeicToJpeg.mockReset();
353358
mockedConvertHeicToJpeg.mockResolvedValue(Buffer.from("jpeg-normalized"));
354359
mockedRunExec.mockReset();
360+
mockedExtractDocumentContent.mockReset();
361+
mockedExtractDocumentContent.mockResolvedValue({ text: "", images: [] });
355362
mockedReadRemoteMediaBuffer.mockResolvedValue({
356363
buffer: createSafeAudioFixtureBuffer(2048),
357364
contentType: "audio/ogg",
@@ -1759,6 +1766,37 @@ describe("applyMediaUnderstanding", () => {
17591766
expectFileNotApplied({ ctx, result, body: "<media:file>" });
17601767
});
17611768

1769+
it("carries rendered PDF page images into current-turn image payloads", async () => {
1770+
const extractedImage = {
1771+
type: "image" as const,
1772+
data: Buffer.from("scanned-page").toString("base64"),
1773+
mimeType: "image/png",
1774+
};
1775+
mockedExtractDocumentContent.mockResolvedValueOnce({
1776+
text: "",
1777+
images: [extractedImage],
1778+
});
1779+
const pseudoPdf = Buffer.from("%PDF-1.7\n1 0 obj\n<< /Type /Catalog >>\nendobj\n", "utf8");
1780+
const filePath = await createTempMediaFile({
1781+
fileName: "receipt-scan.pdf",
1782+
content: pseudoPdf,
1783+
});
1784+
1785+
const { ctx, result } = await applyWithDisabledMedia({
1786+
body: "<media:document>",
1787+
mediaPath: filePath,
1788+
mediaType: "application/pdf",
1789+
});
1790+
1791+
expect(result.appliedFile).toBe(true);
1792+
expect(ctx.Body).toContain("[PDF content rendered to images]");
1793+
expect(ctx.Body).not.toContain("images not forwarded");
1794+
expect(ctx.CurrentTurnImages).toEqual([extractedImage]);
1795+
expect(ctx.CurrentTurnImageOrder).toEqual(["inline"]);
1796+
expect((ctx as unknown as { BodyForAgent?: string }).BodyForAgent).toBe(ctx.Body);
1797+
expect((ctx as unknown as { BodyForCommands?: string }).BodyForCommands).toBe(ctx.Body);
1798+
});
1799+
17621800
it("escapes XML special characters in filenames to prevent injection", async () => {
17631801
// Use & in filename — valid on all platforms (including Windows, which
17641802
// forbids < and > in NTFS filenames) and still requires XML escaping.

0 commit comments

Comments
 (0)