Skip to content

Commit 248a417

Browse files
TurboTheTurtleTurboTheTurtle
committed
fix(media): forward scanned PDF page images
1 parent 31e941c commit 248a417

8 files changed

Lines changed: 190 additions & 25 deletions

File tree

‎docs/nodes/media-understanding.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -321,7 +321,7 @@ When `mode: "all"`, outputs are labeled `[Image 1/2]`, `[Audio 2/2]`, etc.
321321
- The injected block uses explicit boundary markers like `<<<EXTERNAL_UNTRUSTED_CONTENT id="...">>>` / `<<<END_EXTERNAL_UNTRUSTED_CONTENT id="...">>>` and includes a `Source: External` metadata line.
322322
- This attachment-extraction path intentionally omits the long `SECURITY NOTICE:` banner to avoid bloating the media prompt; the boundary markers and metadata still remain.
323323
- If a file has no extractable text, OpenClaw injects `[No extractable text]`.
324-
- If a PDF falls back to rendered page images in this path, the media prompt keeps the placeholder `[PDF content rendered to images; images not forwarded to model]` because this attachment-extraction step forwards text blocks, not the rendered PDF images.
324+
- If a PDF falls back to rendered page images in this path, the media prompt includes `[PDF content rendered to images]` and forwards those page images as current-turn image inputs on vision-capable model paths. If the model path cannot consume images, the placeholder still records that PDF raster fallback occurred.
325325

326326
</Accordion>
327327
</AccordionGroup>

‎src/auto-reply/reply/current-turn-images.test.ts‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,59 @@ describe("resolveCurrentTurnImages", () => {
2323
vi.restoreAllMocks();
2424
});
2525

26+
it("uses and consumes extracted current-turn images", async () => {
27+
const image = {
28+
type: "image" as const,
29+
data: Buffer.from("rendered-pdf-page").toString("base64"),
30+
mimeType: "image/png",
31+
};
32+
const ctx = {
33+
Body: "scan",
34+
CurrentTurnImages: [image],
35+
} satisfies MsgContext;
36+
37+
const result = await resolveCurrentTurnImages({
38+
ctx,
39+
cfg: {} as OpenClawConfig,
40+
});
41+
42+
expect(result).toStrictEqual({
43+
images: [image],
44+
imageOrder: ["inline"],
45+
});
46+
expect(ctx.CurrentTurnImages).toBeUndefined();
47+
});
48+
49+
it("appends extracted current-turn images to explicit inline images", async () => {
50+
const explicitImage = {
51+
type: "image" as const,
52+
data: Buffer.from("explicit-image").toString("base64"),
53+
mimeType: "image/jpeg",
54+
};
55+
const extractedImage = {
56+
type: "image" as const,
57+
data: Buffer.from("rendered-pdf-page").toString("base64"),
58+
mimeType: "image/png",
59+
};
60+
const ctx = {
61+
Body: "scan",
62+
CurrentTurnImages: [extractedImage],
63+
} satisfies MsgContext;
64+
65+
const result = await resolveCurrentTurnImages({
66+
ctx,
67+
cfg: {} as OpenClawConfig,
68+
images: [explicitImage],
69+
imageOrder: ["offloaded"],
70+
});
71+
72+
expect(result).toStrictEqual({
73+
images: [explicitImage, extractedImage],
74+
imageOrder: ["offloaded", "inline"],
75+
});
76+
expect(ctx.CurrentTurnImages).toBeUndefined();
77+
});
78+
2679
it("hydrates Telegram-style state-relative media into native prompt images", async () => {
2780
await withTempDir({ prefix: "openclaw-current-turn-images-" }, async (base) => {
2881
const stateDir = path.join(base, "state");

‎src/auto-reply/reply/current-turn-images.ts‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,23 @@ function createUndescribedImageContext(
8888
};
8989
}
9090

91+
function isValidCurrentTurnImage(image: ImageContent): boolean {
92+
return (
93+
image.type === "image" && image.mimeType.startsWith("image/") && image.data.trim().length > 0
94+
);
95+
}
96+
97+
/** Returns transient current-turn image payloads and removes them from the context. */
98+
export function takeCurrentTurnImages(ctx: MsgContext): ImageContent[] {
99+
const images = Array.isArray(ctx.CurrentTurnImages)
100+
? ctx.CurrentTurnImages.filter(isValidCurrentTurnImage)
101+
: [];
102+
if (Array.isArray(ctx.CurrentTurnImages)) {
103+
delete ctx.CurrentTurnImages;
104+
}
105+
return images;
106+
}
107+
91108
/** Resolves current-turn image attachments that were not already described by media understanding. */
92109
export async function resolveCurrentTurnImages(params: {
93110
ctx: MsgContext;
@@ -98,9 +115,19 @@ export async function resolveCurrentTurnImages(params: {
98115
images?: ImageContent[];
99116
imageOrder?: PromptImageOrderEntry[];
100117
}> {
101-
if (Array.isArray(params.images) && params.images.length > 0) {
118+
const explicitImages = Array.isArray(params.images) ? params.images : [];
119+
const currentTurnImages = takeCurrentTurnImages(params.ctx);
120+
if (explicitImages.length > 0 && currentTurnImages.length === 0) {
102121
return { images: params.images, imageOrder: params.imageOrder };
103122
}
123+
const combinedImages = [...explicitImages, ...currentTurnImages];
124+
if (combinedImages.length > 0) {
125+
const explicitImageOrder = params.imageOrder ?? explicitImages.map(() => "inline" as const);
126+
return {
127+
images: combinedImages,
128+
imageOrder: [...explicitImageOrder, ...currentTurnImages.map(() => "inline" as const)],
129+
};
130+
}
104131

105132
const currentImageAttachments = collectCurrentImageAttachments(params.ctx);
106133
if (currentImageAttachments.length === 0) {

‎src/auto-reply/reply/dispatch-acp.test.ts‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1192,6 +1192,30 @@ describe("tryDispatchAcpReply", () => {
11921192
]);
11931193
});
11941194

1195+
it("forwards media-understanding current-turn images into agent runtime turns", async () => {
1196+
setReadyAcpResolution();
1197+
const image = {
1198+
type: "image" as const,
1199+
mimeType: "image/png",
1200+
data: Buffer.from("rendered-pdf-page").toString("base64"),
1201+
};
1202+
1203+
await runDispatch({
1204+
bodyForAgent: "describe scanned pdf",
1205+
ctxOverrides: {
1206+
CurrentTurnImages: [image],
1207+
},
1208+
});
1209+
1210+
expect(runTurnCall().text).toBe("describe scanned pdf");
1211+
expect(runTurnCall().attachments).toEqual([
1212+
{
1213+
mediaType: "image/png",
1214+
data: image.data,
1215+
},
1216+
]);
1217+
});
1218+
11951219
it("preserves chat.send inline image attachments over recent history images", async () => {
11961220
setReadyAcpResolution();
11971221
const image = {

‎src/auto-reply/reply/dispatch-acp.ts‎

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ import {
3737
resolveInlineAgentImageAttachments,
3838
} from "./agent-turn-attachments.js";
3939
import { resolveFirstContextText } from "./context-text.js";
40+
import { takeCurrentTurnImages } from "./current-turn-images.js";
4041
import {
4142
createAcpDispatchDeliveryCoordinator,
4243
type AcpDispatchDeliveryCoordinator,
@@ -563,22 +564,26 @@ export async function tryDispatchAcpReply(params: {
563564
cfg: params.cfg,
564565
});
565566
const mediaAttachments = resolvedTurnAttachments.attachments;
566-
const inlineAttachments = resolveInlineAgentImageAttachments(params.images);
567+
const explicitInlineAttachments = resolveInlineAgentImageAttachments(params.images);
568+
const transientInlineAttachments = resolveInlineAgentImageAttachments(
569+
takeCurrentTurnImages(params.ctx),
570+
);
571+
const inlineAttachments = [...explicitInlineAttachments, ...transientInlineAttachments];
567572
const mediaAttachmentsAreOnlyRecentHistory =
568573
mediaAttachments.length > 0 &&
569574
mediaAttachments.length === resolvedTurnAttachments.recentHistoryImages.length;
570-
const attachments =
575+
const shouldUseMediaAttachments =
571576
mediaAttachments.length > 0 &&
572-
!(mediaAttachmentsAreOnlyRecentHistory && inlineAttachments.length > 0)
573-
? mediaAttachments
574-
: inlineAttachments;
575-
const turnPromptText =
576-
attachments === mediaAttachments
577-
? appendRecentHistoryImageContext({
578-
promptText,
579-
images: resolvedTurnAttachments.recentHistoryImages,
580-
})
581-
: promptText;
577+
!(mediaAttachmentsAreOnlyRecentHistory && inlineAttachments.length > 0);
578+
const attachments = shouldUseMediaAttachments
579+
? [...mediaAttachments, ...transientInlineAttachments]
580+
: inlineAttachments;
581+
const turnPromptText = shouldUseMediaAttachments
582+
? appendRecentHistoryImageContext({
583+
promptText,
584+
images: resolvedTurnAttachments.recentHistoryImages,
585+
})
586+
: promptText;
582587
if (!turnPromptText && attachments.length === 0) {
583588
const counts = params.dispatcher.getQueuedCounts();
584589
delivery.applyRoutedCounts(counts);

‎src/auto-reply/templating.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
/** Shared inbound message context types used by prompt templating and reply dispatch. */
22
import type { InboundEventKind } from "../channels/inbound-event/kind.js";
3+
import type { ImageContent } from "../llm/types.js";
34
import type {
45
MediaUnderstandingDecision,
56
MediaUnderstandingOutput,
@@ -200,6 +201,8 @@ export type MsgContext = {
200201
MediaWorkspaceDir?: string;
201202
/** Attachment indexes whose audio was already transcribed before media understanding runs. */
202203
MediaTranscribedIndexes?: number[];
204+
/** Extracted current-turn image payloads to forward once without persisting as text context. */
205+
CurrentTurnImages?: ImageContent[];
203206
/**
204207
* Marker: skip downstream stageSandboxMedia. chat.send RPC sets this so
205208
* staging runs synchronously before respond() and surfaces 5xx to the

‎src/media-understanding/apply.test.ts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ const readRemoteMediaBufferMock = vi.hoisted(() => vi.fn());
3131
const runFfmpegMock = vi.hoisted(() => vi.fn());
3232
const convertHeicToJpegMock = vi.hoisted(() => vi.fn());
3333
const runExecMock = vi.hoisted(() => vi.fn());
34+
const extractDocumentContentMock = vi.hoisted(() => vi.fn());
3435

3536
let applyMediaUnderstanding: typeof import("./apply.js").applyMediaUnderstanding;
3637
let clearMediaUnderstandingBinaryCacheForTests: typeof import("./runner.js").clearMediaUnderstandingBinaryCacheForTests;
@@ -39,6 +40,7 @@ const mockedReadRemoteMediaBuffer = readRemoteMediaBufferMock;
3940
const mockedRunFfmpeg = runFfmpegMock;
4041
const mockedConvertHeicToJpeg = convertHeicToJpegMock;
4142
const mockedRunExec = runExecMock;
43+
const mockedExtractDocumentContent = extractDocumentContentMock;
4244

4345
const TEMP_MEDIA_PREFIX = "openclaw-media-";
4446
let suiteTempMediaRootDir = "";
@@ -298,6 +300,9 @@ describe("applyMediaUnderstanding", () => {
298300
runFfmpeg: runFfmpegMock,
299301
convertHeicToJpeg: convertHeicToJpegMock,
300302
}));
303+
vi.doMock("../media/document-extractors.runtime.js", () => ({
304+
extractDocumentContent: extractDocumentContentMock,
305+
}));
301306
vi.doMock("../process/exec.js", () => ({
302307
runExec: runExecMock,
303308
}));
@@ -352,6 +357,8 @@ describe("applyMediaUnderstanding", () => {
352357
mockedConvertHeicToJpeg.mockReset();
353358
mockedConvertHeicToJpeg.mockResolvedValue(Buffer.from("jpeg-normalized"));
354359
mockedRunExec.mockReset();
360+
mockedExtractDocumentContent.mockReset();
361+
mockedExtractDocumentContent.mockResolvedValue(null);
355362
mockedReadRemoteMediaBuffer.mockResolvedValue({
356363
buffer: createSafeAudioFixtureBuffer(2048),
357364
contentType: "audio/ogg",
@@ -1742,6 +1749,38 @@ describe("applyMediaUnderstanding", () => {
17421749
expectFileNotApplied({ ctx, result, body: "<media:file>" });
17431750
});
17441751

1752+
it("forwards rendered PDF page images as current-turn images", async () => {
1753+
const pseudoPdf = Buffer.from("%PDF-1.7\n1 0 obj\n<< /Type /Catalog >>\nendobj\n", "utf8");
1754+
const filePath = await createTempMediaFile({
1755+
fileName: "scan.pdf",
1756+
content: pseudoPdf,
1757+
});
1758+
const renderedPageImage = {
1759+
type: "image" as const,
1760+
mimeType: "image/png",
1761+
data: Buffer.from("rendered-page").toString("base64"),
1762+
};
1763+
mockedExtractDocumentContent.mockResolvedValueOnce({
1764+
text: "",
1765+
images: [renderedPageImage],
1766+
});
1767+
1768+
const cfg = createMediaDisabledConfigWithAllowedMimes(["application/pdf"]);
1769+
1770+
const { ctx, result } = await applyWithDisabledMedia({
1771+
body: "<media:file>",
1772+
mediaPath: filePath,
1773+
mediaType: "application/pdf",
1774+
cfg,
1775+
});
1776+
1777+
expect(result.appliedFile).toBe(true);
1778+
expect(ctx.Body).toContain('<file name="scan.pdf" mime="application/pdf">');
1779+
expect(ctx.Body).toContain("[PDF content rendered to images]");
1780+
expect(ctx.Body).not.toContain("images not forwarded");
1781+
expect(ctx.CurrentTurnImages).toEqual([renderedPageImage]);
1782+
});
1783+
17451784
it("respects configured allowedMimes for text-like attachments", async () => {
17461785
const tsvText = "a\tb\tc\n1\t2\t3";
17471786
const tsvPath = await createTempMediaFile({

‎src/media-understanding/apply.ts‎

Lines changed: 25 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -5,19 +5,20 @@ import {
55
normalizeLowercaseStringOrEmpty,
66
normalizeOptionalString,
77
} from "@openclaw/normalization-core/string-coerce";
8+
import type { ActiveMediaModel } from "../../packages/media-understanding-common/src/active-model.js";
9+
import {
10+
extractMediaUserText,
11+
formatAudioTranscripts,
12+
formatMediaUnderstandingBody,
13+
} from "../../packages/media-understanding-common/src/format.js";
814
import { finalizeInboundContext } from "../auto-reply/reply/inbound-context.js";
915
import type { MsgContext } from "../auto-reply/templating.js";
1016
import type { OpenClawConfig } from "../config/types.js";
1117
import { logVerbose, shouldLogVerbose } from "../globals.js";
18+
import type { ImageContent } from "../llm/types.js";
1219
import { renderFileContextBlock } from "../media/file-context.js";
1320
import { extractFileContentFromSource, normalizeMimeType } from "../media/input-files.js";
1421
import { wrapExternalContent } from "../security/external-content.js";
15-
import type { ActiveMediaModel } from "../../packages/media-understanding-common/src/active-model.js";
16-
import {
17-
extractMediaUserText,
18-
formatAudioTranscripts,
19-
formatMediaUnderstandingBody,
20-
} from "../../packages/media-understanding-common/src/format.js";
2122
import { resolveAttachmentKind } from "./attachments.js";
2223
import { runWithConcurrency } from "./concurrency.js";
2324
import { DEFAULT_ECHO_TRANSCRIPT_FORMAT, sendTranscriptEcho } from "./echo-transcript.js";
@@ -49,6 +50,11 @@ type ApplyMediaUnderstandingResult = {
4950
appliedFile: boolean;
5051
};
5152

53+
type ExtractedFileBlocks = {
54+
blocks: string[];
55+
images: ImageContent[];
56+
};
57+
5258
const CAPABILITY_ORDER: MediaUnderstandingCapability[] = ["image", "audio", "video"];
5359
const EMPTY_VOICE_NOTE_PLACEHOLDER =
5460
"[Voice note could not be transcribed because the audio attachment was too small]";
@@ -383,12 +389,13 @@ async function extractFileBlocks(params: {
383389
cfg: OpenClawConfig;
384390
limits: FileExtractionLimits;
385391
skipAttachmentIndexes?: Set<number>;
386-
}): Promise<string[]> {
392+
}): Promise<ExtractedFileBlocks> {
387393
const { attachments, cache, cfg, limits, skipAttachmentIndexes } = params;
388394
if (!attachments || attachments.length === 0) {
389-
return [];
395+
return { blocks: [], images: [] };
390396
}
391397
const blocks: string[] = [];
398+
const images: ImageContent[] = [];
392399
for (const attachment of attachments) {
393400
if (!attachment) {
394401
continue;
@@ -494,10 +501,13 @@ async function extractFileBlocks(params: {
494501
continue;
495502
}
496503
const text = extracted?.text?.trim() ?? "";
504+
if (extracted?.images && extracted.images.length > 0) {
505+
images.push(...extracted.images);
506+
}
497507
let blockText = text ? wrapUntrustedAttachmentContent(text) : "";
498508
if (!blockText) {
499509
if (extracted?.images && extracted.images.length > 0) {
500-
blockText = "[PDF content rendered to images; images not forwarded to model]";
510+
blockText = "[PDF content rendered to images]";
501511
} else {
502512
blockText = "[No extractable text]";
503513
}
@@ -511,7 +521,7 @@ async function extractFileBlocks(params: {
511521
}),
512522
);
513523
}
514-
return blocks;
524+
return { blocks, images };
515525
}
516526

517527
export async function applyMediaUnderstanding(params: {
@@ -670,13 +680,17 @@ export async function applyMediaUnderstanding(params: {
670680
)
671681
.map((output) => output.attachmentIndex),
672682
);
673-
const fileBlocks = await extractFileBlocks({
683+
const extractedFiles = await extractFileBlocks({
674684
attachments,
675685
cache,
676686
cfg,
677687
limits: resolveFileExtractionLimits(cfg),
678688
skipAttachmentIndexes: audioAttachmentIndexes.size > 0 ? audioAttachmentIndexes : undefined,
679689
});
690+
const fileBlocks = extractedFiles.blocks;
691+
if (extractedFiles.images.length > 0) {
692+
ctx.CurrentTurnImages = [...(ctx.CurrentTurnImages ?? []), ...extractedFiles.images];
693+
}
680694
if (fileBlocks.length > 0) {
681695
ctx.Body = appendFileBlocks(ctx.Body, fileBlocks);
682696
}

0 commit comments

Comments
 (0)