Skip to content

Commit 2cd82bd

Browse files
harjothkharaharjothkhara
committed
fix(media): forward scanned PDF page images in chat turns
1 parent 170bf72 commit 2cd82bd

7 files changed

Lines changed: 255 additions & 39 deletions

File tree

‎src/auto-reply/reply/current-turn-images.test.ts‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,4 +59,65 @@ describe("resolveCurrentTurnImages", () => {
5959
});
6060
});
6161
});
62+
63+
it("merges current-turn extracted images with existing inline images", async () => {
64+
const existingImage = {
65+
type: "image" as const,
66+
data: Buffer.from("existing-image").toString("base64"),
67+
mimeType: "image/jpeg",
68+
};
69+
const extractedPdfPage = {
70+
type: "image" as const,
71+
data: Buffer.from("pdf-page").toString("base64"),
72+
mimeType: "image/png",
73+
};
74+
75+
const result = await resolveCurrentTurnImages({
76+
ctx: {
77+
Body: "<media:document>",
78+
CurrentTurnImages: [extractedPdfPage],
79+
CurrentTurnImageOrder: ["inline"],
80+
} satisfies MsgContext,
81+
cfg: {} as OpenClawConfig,
82+
images: [existingImage],
83+
imageOrder: ["inline"],
84+
});
85+
86+
expect(result).toStrictEqual({
87+
images: [existingImage, extractedPdfPage],
88+
imageOrder: ["inline", "inline"],
89+
});
90+
});
91+
92+
it("does not merge extracted images twice across the prepare and run passes", async () => {
93+
const extractedPdfPage = {
94+
type: "image" as const,
95+
data: Buffer.from("pdf-page").toString("base64"),
96+
mimeType: "image/png",
97+
};
98+
const ctx = {
99+
Body: "<media:document>",
100+
CurrentTurnImages: [extractedPdfPage],
101+
CurrentTurnImageOrder: ["inline"],
102+
} satisfies MsgContext;
103+
104+
const prepared = await resolveCurrentTurnImages({
105+
ctx,
106+
cfg: {} as OpenClawConfig,
107+
});
108+
const runPass = await resolveCurrentTurnImages({
109+
ctx,
110+
cfg: {} as OpenClawConfig,
111+
images: prepared.images,
112+
imageOrder: prepared.imageOrder,
113+
});
114+
115+
expect(prepared).toStrictEqual({
116+
images: [extractedPdfPage],
117+
imageOrder: ["inline"],
118+
});
119+
expect(runPass).toStrictEqual(prepared);
120+
expect(ctx.CurrentTurnImages).toBeUndefined();
121+
expect(ctx.CurrentTurnImageOrder).toBeUndefined();
122+
});
62123
});

‎src/auto-reply/reply/current-turn-images.ts‎

Lines changed: 62 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,44 @@ function createUndescribedImageContext(
8888
};
8989
}
9090

91+
function mergeImagePayloads(params: {
92+
baseImages?: ImageContent[];
93+
baseImageOrder?: PromptImageOrderEntry[];
94+
nextImages?: ImageContent[];
95+
nextImageOrder?: PromptImageOrderEntry[];
96+
}): {
97+
images?: ImageContent[];
98+
imageOrder?: PromptImageOrderEntry[];
99+
} {
100+
const nextImages = params.nextImages ?? [];
101+
if (nextImages.length === 0) {
102+
return { images: params.baseImages, imageOrder: params.baseImageOrder };
103+
}
104+
const images = [...(params.baseImages ?? []), ...nextImages];
105+
const nextOrder =
106+
params.nextImageOrder && params.nextImageOrder.length === nextImages.length
107+
? params.nextImageOrder
108+
: nextImages.map(() => "inline" as const);
109+
return {
110+
images,
111+
imageOrder: [...(params.baseImageOrder ?? []), ...nextOrder],
112+
};
113+
}
114+
115+
export function takeCurrentTurnImages(ctx: MsgContext): {
116+
images?: ImageContent[];
117+
imageOrder?: PromptImageOrderEntry[];
118+
} {
119+
const images = ctx.CurrentTurnImages;
120+
const imageOrder = ctx.CurrentTurnImageOrder;
121+
if (images && images.length > 0) {
122+
delete ctx.CurrentTurnImages;
123+
delete ctx.CurrentTurnImageOrder;
124+
return { images, imageOrder };
125+
}
126+
return {};
127+
}
128+
91129
/** Resolves current-turn image attachments that were not already described by media understanding. */
92130
export async function resolveCurrentTurnImages(params: {
93131
ctx: MsgContext;
@@ -98,30 +136,37 @@ export async function resolveCurrentTurnImages(params: {
98136
images?: ImageContent[];
99137
imageOrder?: PromptImageOrderEntry[];
100138
}> {
139+
const extractedImages = takeCurrentTurnImages(params.ctx);
140+
let resolved = mergeImagePayloads({
141+
baseImages: params.images,
142+
baseImageOrder: params.imageOrder,
143+
nextImages: extractedImages.images,
144+
nextImageOrder: extractedImages.imageOrder,
145+
});
101146
if (Array.isArray(params.images) && params.images.length > 0) {
102-
return { images: params.images, imageOrder: params.imageOrder };
147+
return resolved;
103148
}
104149

105150
const currentImageAttachments = collectCurrentImageAttachments(params.ctx);
106151
if (currentImageAttachments.length === 0) {
107-
return { images: params.images, imageOrder: params.imageOrder };
152+
return resolved;
108153
}
109154
const describedImageIndexes = collectDescribedImageAttachmentIndexes(params.ctx);
110155
const undescribedImageAttachments = currentImageAttachments.filter(
111156
(attachment) => !describedImageIndexes.has(attachment.index),
112157
);
113158
if (undescribedImageAttachments.length === 0) {
114-
return { images: params.images, imageOrder: params.imageOrder };
159+
return resolved;
115160
}
116161

117162
try {
118163
// Only send undescribed current images natively; described images already exist as text context.
119-
const resolved = await resolveAgentTurnAttachments({
164+
const attachmentResult = await resolveAgentTurnAttachments({
120165
ctx: createUndescribedImageContext(params.ctx, undescribedImageAttachments),
121166
cfg: params.cfg,
122167
includeRecentHistoryImages: false,
123168
});
124-
const images = resolved.attachments.map(
169+
const images = attachmentResult.attachments.map(
125170
(attachment): ImageContent => ({
126171
type: "image",
127172
data: attachment.data,
@@ -132,15 +177,22 @@ export async function resolveCurrentTurnImages(params: {
132177
logVerbose(
133178
`agent-runner: native OpenClaw media resolution produced ${images.length}/${undescribedImageAttachments.length} current image attachment(s); falling back to prompt image refs`,
134179
);
135-
return { images: params.images, imageOrder: params.imageOrder };
180+
return resolved;
181+
}
182+
if (images.length === 0) {
183+
return resolved;
136184
}
137-
return images.length > 0
138-
? { images, imageOrder: images.map(() => "inline" as const) }
139-
: { images: params.images, imageOrder: params.imageOrder };
185+
resolved = mergeImagePayloads({
186+
baseImages: resolved.images,
187+
baseImageOrder: resolved.imageOrder,
188+
nextImages: images,
189+
nextImageOrder: images.map(() => "inline" as const),
190+
});
191+
return resolved;
140192
} catch (error) {
141193
logVerbose(
142194
`agent-runner: media attachment image resolution failed, proceeding without native images: ${formatErrorMessage(error)}`,
143195
);
144-
return { images: params.images, imageOrder: params.imageOrder };
196+
return resolved;
145197
}
146198
}

‎src/auto-reply/reply/dispatch-acp.test.ts‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1192,6 +1192,38 @@ describe("tryDispatchAcpReply", () => {
11921192
]);
11931193
});
11941194

1195+
it("forwards PDF page images extracted during media understanding into ACP runtime turns", async () => {
1196+
setReadyAcpResolution();
1197+
const extractedPdfPage = {
1198+
type: "image" as const,
1199+
mimeType: "image/png",
1200+
data: Buffer.from("pdf-page").toString("base64"),
1201+
};
1202+
mediaUnderstandingMocks.applyMediaUnderstanding.mockImplementationOnce(async (params) => {
1203+
const ctx = (
1204+
params as { ctx: { CurrentTurnImages?: unknown; CurrentTurnImageOrder?: unknown } }
1205+
).ctx;
1206+
ctx.CurrentTurnImages = [extractedPdfPage];
1207+
ctx.CurrentTurnImageOrder = ["inline"];
1208+
});
1209+
1210+
await runDispatch({
1211+
bodyForAgent: '<file name="scan.pdf">[PDF content rendered to images]</file>',
1212+
ctxOverrides: {
1213+
MediaPath: "/tmp/scan.pdf",
1214+
MediaType: "application/pdf",
1215+
},
1216+
});
1217+
1218+
expect(mediaUnderstandingMocks.applyMediaUnderstanding).toHaveBeenCalledOnce();
1219+
expect(runTurnCall().attachments).toEqual([
1220+
{
1221+
mediaType: "image/png",
1222+
data: extractedPdfPage.data,
1223+
},
1224+
]);
1225+
});
1226+
11951227
it("preserves chat.send inline image attachments over recent history images", async () => {
11961228
setReadyAcpResolution();
11971229
const image = {

‎src/auto-reply/reply/dispatch-acp.ts‎

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ import {
3737
resolveInlineAgentImageAttachments,
3838
} from "./agent-turn-attachments.js";
3939
import { resolveFirstContextText } from "./context-text.js";
40+
import { takeCurrentTurnImages } from "./current-turn-images.js";
4041
import {
4142
createAcpDispatchDeliveryCoordinator,
4243
type AcpDispatchDeliveryCoordinator,
@@ -563,22 +564,26 @@ export async function tryDispatchAcpReply(params: {
563564
cfg: params.cfg,
564565
});
565566
const mediaAttachments = resolvedTurnAttachments.attachments;
566-
const inlineAttachments = resolveInlineAgentImageAttachments(params.images);
567+
const inputInlineAttachments = resolveInlineAgentImageAttachments(params.images);
568+
const extractedInlineAttachments = resolveInlineAgentImageAttachments(
569+
takeCurrentTurnImages(params.ctx).images,
570+
);
571+
const inlineAttachments = [...inputInlineAttachments, ...extractedInlineAttachments];
567572
const mediaAttachmentsAreOnlyRecentHistory =
568573
mediaAttachments.length > 0 &&
569574
mediaAttachments.length === resolvedTurnAttachments.recentHistoryImages.length;
570-
const attachments =
575+
const shouldUseMediaAttachments =
571576
mediaAttachments.length > 0 &&
572-
!(mediaAttachmentsAreOnlyRecentHistory && inlineAttachments.length > 0)
573-
? mediaAttachments
574-
: inlineAttachments;
575-
const turnPromptText =
576-
attachments === mediaAttachments
577-
? appendRecentHistoryImageContext({
578-
promptText,
579-
images: resolvedTurnAttachments.recentHistoryImages,
580-
})
581-
: promptText;
577+
!(mediaAttachmentsAreOnlyRecentHistory && inlineAttachments.length > 0);
578+
const attachments = shouldUseMediaAttachments
579+
? [...mediaAttachments, ...extractedInlineAttachments]
580+
: inlineAttachments;
581+
const turnPromptText = shouldUseMediaAttachments
582+
? appendRecentHistoryImageContext({
583+
promptText,
584+
images: resolvedTurnAttachments.recentHistoryImages,
585+
})
586+
: promptText;
582587
if (!turnPromptText && attachments.length === 0) {
583588
const counts = params.dispatcher.getQueuedCounts();
584589
delivery.applyRoutedCounts(counts);

‎src/auto-reply/templating.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
/** Shared inbound message context types used by prompt templating and reply dispatch. */
22
import type { InboundEventKind } from "../channels/inbound-event/kind.js";
3+
import type { ImageContent } from "../llm/types.js";
34
import type {
45
MediaUnderstandingDecision,
56
MediaUnderstandingOutput,
67
} from "../media-understanding/types.js";
8+
import type { PromptImageOrderEntry } from "../media/prompt-image-order.js";
79
import type { PluginHookChannelContext } from "../plugins/hook-channel-context.types.js";
810
import type { InputProvenance } from "../sessions/input-provenance.js";
911
import type { CommandTurnContext } from "./command-turn-context.js";
@@ -219,6 +221,9 @@ export type MsgContext = {
219221
Transcript?: string;
220222
MediaUnderstanding?: MediaUnderstandingOutput[];
221223
MediaUnderstandingDecisions?: MediaUnderstandingDecision[];
224+
/** Inline images derived during current-turn preprocessing, such as PDF page renders. */
225+
CurrentTurnImages?: ImageContent[];
226+
CurrentTurnImageOrder?: PromptImageOrderEntry[];
222227
LinkUnderstanding?: string[];
223228
Prompt?: string;
224229
MaxChars?: number;

‎src/media-understanding/apply.test.ts‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ const readRemoteMediaBufferMock = vi.hoisted(() => vi.fn());
3131
const runFfmpegMock = vi.hoisted(() => vi.fn());
3232
const convertHeicToJpegMock = vi.hoisted(() => vi.fn());
3333
const runExecMock = vi.hoisted(() => vi.fn());
34+
const extractDocumentContentMock = vi.hoisted(() => vi.fn());
3435

3536
let applyMediaUnderstanding: typeof import("./apply.js").applyMediaUnderstanding;
3637
let clearMediaUnderstandingBinaryCacheForTests: typeof import("./runner.js").clearMediaUnderstandingBinaryCacheForTests;
@@ -39,6 +40,7 @@ const mockedReadRemoteMediaBuffer = readRemoteMediaBufferMock;
3940
const mockedRunFfmpeg = runFfmpegMock;
4041
const mockedConvertHeicToJpeg = convertHeicToJpegMock;
4142
const mockedRunExec = runExecMock;
43+
const mockedExtractDocumentContent = extractDocumentContentMock;
4244

4345
const TEMP_MEDIA_PREFIX = "openclaw-media-";
4446
let suiteTempMediaRootDir = "";
@@ -298,6 +300,9 @@ describe("applyMediaUnderstanding", () => {
298300
runFfmpeg: runFfmpegMock,
299301
convertHeicToJpeg: convertHeicToJpegMock,
300302
}));
303+
vi.doMock("../media/document-extractors.runtime.js", () => ({
304+
extractDocumentContent: extractDocumentContentMock,
305+
}));
301306
vi.doMock("../process/exec.js", () => ({
302307
runExec: runExecMock,
303308
}));
@@ -352,6 +357,8 @@ describe("applyMediaUnderstanding", () => {
352357
mockedConvertHeicToJpeg.mockReset();
353358
mockedConvertHeicToJpeg.mockResolvedValue(Buffer.from("jpeg-normalized"));
354359
mockedRunExec.mockReset();
360+
mockedExtractDocumentContent.mockReset();
361+
mockedExtractDocumentContent.mockResolvedValue({ text: "", images: [] });
355362
mockedReadRemoteMediaBuffer.mockResolvedValue({
356363
buffer: createSafeAudioFixtureBuffer(2048),
357364
contentType: "audio/ogg",
@@ -1742,6 +1749,37 @@ describe("applyMediaUnderstanding", () => {
17421749
expectFileNotApplied({ ctx, result, body: "<media:file>" });
17431750
});
17441751

1752+
it("carries rendered PDF page images into current-turn image payloads", async () => {
1753+
const extractedImage = {
1754+
type: "image" as const,
1755+
data: Buffer.from("scanned-page").toString("base64"),
1756+
mimeType: "image/png",
1757+
};
1758+
mockedExtractDocumentContent.mockResolvedValueOnce({
1759+
text: "",
1760+
images: [extractedImage],
1761+
});
1762+
const pseudoPdf = Buffer.from("%PDF-1.7\n1 0 obj\n<< /Type /Catalog >>\nendobj\n", "utf8");
1763+
const filePath = await createTempMediaFile({
1764+
fileName: "receipt-scan.pdf",
1765+
content: pseudoPdf,
1766+
});
1767+
1768+
const { ctx, result } = await applyWithDisabledMedia({
1769+
body: "<media:document>",
1770+
mediaPath: filePath,
1771+
mediaType: "application/pdf",
1772+
});
1773+
1774+
expect(result.appliedFile).toBe(true);
1775+
expect(ctx.Body).toContain("[PDF content rendered to images]");
1776+
expect(ctx.Body).not.toContain("images not forwarded");
1777+
expect(ctx.CurrentTurnImages).toEqual([extractedImage]);
1778+
expect(ctx.CurrentTurnImageOrder).toEqual(["inline"]);
1779+
expect(ctx.BodyForAgent).toBe(ctx.Body);
1780+
expect(ctx.BodyForCommands).toBe(ctx.Body);
1781+
});
1782+
17451783
it("respects configured allowedMimes for text-like attachments", async () => {
17461784
const tsvText = "a\tb\tc\n1\t2\t3";
17471785
const tsvPath = await createTempMediaFile({

0 commit comments

Comments
 (0)