Skip to content

Commit 07b9349

Browse files
joshavantjoshavant
authored
fix: scanned PDF pages reach chat vision models (#97354)
* fix: forward scanned PDF page images to chat models * fix: remove stale reply option cast
1 parent 1b07660 commit 07b9349

15 files changed

Lines changed: 508 additions & 81 deletions

‎docs/nodes/media-understanding.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -321,7 +321,7 @@ When `mode: "all"`, outputs are labeled `[Image 1/2]`, `[Audio 2/2]`, etc.
321321
- The injected block uses explicit boundary markers like `<<<EXTERNAL_UNTRUSTED_CONTENT id="...">>>` / `<<<END_EXTERNAL_UNTRUSTED_CONTENT id="...">>>` and includes a `Source: External` metadata line.
322322
- This attachment-extraction path intentionally omits the long `SECURITY NOTICE:` banner to avoid bloating the media prompt; the boundary markers and metadata still remain.
323323
- If a file has no extractable text, OpenClaw injects `[No extractable text]`.
324-
- If a PDF falls back to rendered page images in this path, the media prompt keeps the placeholder `[PDF content rendered to images; images not forwarded to model]` because this attachment-extraction step forwards text blocks, not the rendered PDF images.
324+
- If a PDF falls back to rendered page images in this path, OpenClaw forwards those page images to vision-capable reply models and keeps the placeholder `[PDF content rendered to images]` in the file block.
325325

326326
</Accordion>
327327
</AccordionGroup>

‎src/auto-reply/reply/agent-turn-attachments.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,10 @@ export async function resolveAgentTurnAttachments(params: {
5050
cfg: OpenClawConfig;
5151
runtime?: AgentTurnAttachmentRuntime;
5252
includeRecentHistoryImages?: boolean;
53+
includeAttachmentIndexes?: boolean;
5354
}): Promise<{
5455
attachments: AgentTurnAttachment[];
56+
attachmentIndexes?: number[];
5557
recentHistoryImages: RecentInboundHistoryImage[];
5658
}> {
5759
const includeRecentHistoryImages = params.includeRecentHistoryImages ?? true;
@@ -94,6 +96,7 @@ export async function resolveAgentTurnAttachments(params: {
9496
}),
9597
});
9698
const results: AgentTurnAttachment[] = [];
99+
const resultIndexes: number[] = [];
97100
const resolvedHistoryImages: RecentInboundHistoryImage[] = [];
98101
const resolveImageAttachment = async (attachment: MediaAttachment): Promise<boolean> => {
99102
const mediaType = attachment.mime ?? "application/octet-stream";
@@ -113,6 +116,7 @@ export async function resolveAgentTurnAttachments(params: {
113116
mediaType,
114117
data: buffer.toString("base64"),
115118
});
119+
resultIndexes.push(attachment.index);
116120
const historyImage = historyAttachmentByIndex.get(attachment.index);
117121
if (historyImage) {
118122
resolvedHistoryImages.push(historyImage);
@@ -149,7 +153,11 @@ export async function resolveAgentTurnAttachments(params: {
149153
await resolveImageAttachment(attachment);
150154
}
151155
}
152-
return { attachments: results, recentHistoryImages: resolvedHistoryImages };
156+
return {
157+
attachments: results,
158+
...(params.includeAttachmentIndexes ? { attachmentIndexes: resultIndexes } : {}),
159+
recentHistoryImages: resolvedHistoryImages,
160+
};
153161
}
154162

155163
/** Converts inline image content into ACP attachment payloads. */

‎src/auto-reply/reply/current-turn-images.test.ts‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,4 +60,74 @@ describe("resolveCurrentTurnImages", () => {
6060
});
6161
});
6262
});
63+
64+
it("appends extracted PDF page images without dropping current image attachments", async () => {
65+
await withTempDir({ prefix: "openclaw-current-turn-pdf-images-" }, async (base) => {
66+
const imagePath = path.join(base, "photo.png");
67+
const imageBytes = Buffer.from("current-photo");
68+
await fs.writeFile(imagePath, imageBytes);
69+
70+
const pdfPage = {
71+
type: "image" as const,
72+
data: Buffer.from("pdf-page").toString("base64"),
73+
mimeType: "image/png",
74+
attachmentIndex: 1,
75+
};
76+
77+
const result = await resolveCurrentTurnImages({
78+
ctx: {
79+
Body: "caption",
80+
MediaPaths: [imagePath, path.join(base, "scan.pdf")],
81+
MediaTypes: ["image/png", "application/pdf"],
82+
MediaWorkspaceDir: base,
83+
} satisfies MsgContext,
84+
cfg: {} as OpenClawConfig,
85+
extractedFileImages: [pdfPage],
86+
});
87+
88+
expect(result.images).toEqual([
89+
{
90+
type: "image",
91+
data: imageBytes.toString("base64"),
92+
mimeType: "image/png",
93+
},
94+
{
95+
type: "image",
96+
data: pdfPage.data,
97+
mimeType: "image/png",
98+
},
99+
]);
100+
expect(result.imageOrder).toEqual(["inline", "inline"]);
101+
});
102+
});
103+
104+
it("orders extracted PDF page images before later current image attachments", async () => {
105+
await withTempDir({ prefix: "openclaw-current-turn-pdf-order-" }, async (base) => {
106+
const imagePath = path.join(base, "photo.png");
107+
await fs.writeFile(imagePath, "current-photo");
108+
const pdfPage = {
109+
type: "image" as const,
110+
data: Buffer.from("pdf-page").toString("base64"),
111+
mimeType: "image/png",
112+
attachmentIndex: 0,
113+
};
114+
115+
const result = await resolveCurrentTurnImages({
116+
ctx: {
117+
Body: "caption",
118+
MediaPaths: [path.join(base, "scan.pdf"), imagePath],
119+
MediaTypes: ["application/pdf", "image/png"],
120+
MediaWorkspaceDir: base,
121+
} satisfies MsgContext,
122+
cfg: {} as OpenClawConfig,
123+
extractedFileImages: [pdfPage],
124+
});
125+
126+
expect(result.images?.map((image) => Buffer.from(image.data, "base64").toString())).toEqual([
127+
"pdf-page",
128+
"current-photo",
129+
]);
130+
expect(result.imageOrder).toEqual(["inline", "inline"]);
131+
});
132+
});
63133
});

‎src/auto-reply/reply/current-turn-images.ts‎

Lines changed: 77 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,10 @@ import type { OpenClawConfig } from "../../config/types.openclaw.js";
55
import { logVerbose } from "../../globals.js";
66
import { formatErrorMessage } from "../../infra/errors.js";
77
import type { ImageContent } from "../../llm/types.js";
8+
import {
9+
stripExtractedFileImageMetadata,
10+
type ExtractedFileImage,
11+
} from "../../media-understanding/extracted-file-images.js";
812
import type { PromptImageOrderEntry } from "../../media/prompt-image-order.js";
913
import type { MsgContext } from "../templating.js";
1014
import { resolveAgentTurnAttachments } from "./agent-turn-attachments.js";
@@ -15,6 +19,13 @@ type CurrentImageAttachment = {
1519
mediaType: string;
1620
};
1721

22+
type OrderedTurnImage = {
23+
image: ImageContent;
24+
imageOrder: PromptImageOrderEntry;
25+
sourceIndex?: number;
26+
sequence: number;
27+
};
28+
1829
function isGenericMediaType(mediaType: string | undefined): boolean {
1930
if (!mediaType) {
2031
return true;
@@ -88,30 +99,82 @@ function createUndescribedImageContext(
8899
};
89100
}
90101

102+
function appendOrderedImages(params: {
103+
entries: OrderedTurnImage[];
104+
images: ImageContent[] | undefined;
105+
imageOrder?: PromptImageOrderEntry[];
106+
sourceIndex?: number;
107+
}) {
108+
if (!params.images || params.images.length === 0) {
109+
return;
110+
}
111+
for (const [index, image] of params.images.entries()) {
112+
params.entries.push({
113+
image,
114+
imageOrder: params.imageOrder?.[index] ?? "inline",
115+
sourceIndex: params.sourceIndex,
116+
sequence: params.entries.length,
117+
});
118+
}
119+
}
120+
121+
function resolveMergedTurnImages(entries: OrderedTurnImage[]): {
122+
images?: ImageContent[];
123+
imageOrder?: PromptImageOrderEntry[];
124+
} {
125+
if (entries.length === 0) {
126+
return {};
127+
}
128+
const merged = entries.toSorted((left, right) => {
129+
if (left.sourceIndex !== undefined && right.sourceIndex !== undefined) {
130+
return left.sourceIndex - right.sourceIndex || left.sequence - right.sequence;
131+
}
132+
if (left.sourceIndex !== undefined || right.sourceIndex !== undefined) {
133+
return left.sequence - right.sequence;
134+
}
135+
return left.sequence - right.sequence;
136+
});
137+
return {
138+
images: merged.map((entry) => entry.image),
139+
imageOrder: merged.map((entry) => entry.imageOrder),
140+
};
141+
}
142+
91143
/** Resolves current-turn image attachments that were not already described by media understanding. */
92144
export async function resolveCurrentTurnImages(params: {
93145
ctx: MsgContext;
94146
cfg: OpenClawConfig;
95147
images?: ImageContent[];
96148
imageOrder?: PromptImageOrderEntry[];
149+
extractedFileImages?: ExtractedFileImage[];
97150
}): Promise<{
98151
images?: ImageContent[];
99152
imageOrder?: PromptImageOrderEntry[];
100153
}> {
101-
if (Array.isArray(params.images) && params.images.length > 0) {
102-
return { images: params.images, imageOrder: params.imageOrder };
154+
const entries: OrderedTurnImage[] = [];
155+
appendOrderedImages({
156+
entries,
157+
images: params.images,
158+
imageOrder: params.imageOrder,
159+
});
160+
for (const image of params.extractedFileImages ?? []) {
161+
appendOrderedImages({
162+
entries,
163+
images: [stripExtractedFileImageMetadata(image)],
164+
sourceIndex: image.attachmentIndex,
165+
});
103166
}
104167

105168
const currentImageAttachments = collectCurrentImageAttachments(params.ctx);
106169
if (currentImageAttachments.length === 0) {
107-
return { images: params.images, imageOrder: params.imageOrder };
170+
return resolveMergedTurnImages(entries);
108171
}
109172
const describedImageIndexes = collectDescribedImageAttachmentIndexes(params.ctx);
110173
const undescribedImageAttachments = currentImageAttachments.filter(
111174
(attachment) => !describedImageIndexes.has(attachment.index),
112175
);
113176
if (undescribedImageAttachments.length === 0) {
114-
return { images: params.images, imageOrder: params.imageOrder };
177+
return resolveMergedTurnImages(entries);
115178
}
116179

117180
try {
@@ -132,15 +195,20 @@ export async function resolveCurrentTurnImages(params: {
132195
logVerbose(
133196
`agent-runner: native OpenClaw media resolution produced ${images.length}/${undescribedImageAttachments.length} current image attachment(s); falling back to prompt image refs`,
134197
);
135-
return { images: params.images, imageOrder: params.imageOrder };
198+
return resolveMergedTurnImages(entries);
199+
}
200+
for (const [index, image] of images.entries()) {
201+
appendOrderedImages({
202+
entries,
203+
images: [image],
204+
sourceIndex: undescribedImageAttachments[index]?.index,
205+
});
136206
}
137-
return images.length > 0
138-
? { images, imageOrder: images.map(() => "inline" as const) }
139-
: { images: params.images, imageOrder: params.imageOrder };
207+
return resolveMergedTurnImages(entries);
140208
} catch (error) {
141209
logVerbose(
142210
`agent-runner: media attachment image resolution failed, proceeding without native images: ${formatErrorMessage(error)}`,
143211
);
144-
return { images: params.images, imageOrder: params.imageOrder };
212+
return resolveMergedTurnImages(entries);
145213
}
146214
}

‎src/auto-reply/reply/dispatch-acp.test.ts‎

Lines changed: 45 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import { AcpRuntimeError } from "../../acp/runtime/errors.js";
88
import type { AcpSessionStoreEntry } from "../../acp/runtime/session-meta.js";
99
import type { OpenClawConfig } from "../../config/config.js";
1010
import type { SessionBindingRecord } from "../../infra/outbound/session-binding-service.js";
11+
import type { ApplyMediaUnderstandingResult } from "../../media-understanding/apply.js";
1112
import { withFetchPreconnect } from "../../test-utils/fetch-mock.js";
1213
import {
1314
resolveAgentTurnAttachments,
@@ -80,7 +81,9 @@ const ttsMocks = vi.hoisted(() => ({
8081
}));
8182

8283
const mediaUnderstandingMocks = vi.hoisted(() => ({
83-
applyMediaUnderstanding: vi.fn(async (_params: unknown) => undefined),
84+
applyMediaUnderstanding: vi.fn<
85+
(_params: unknown) => Promise<ApplyMediaUnderstandingResult | undefined>
86+
>(async () => undefined),
8487
}));
8588

8689
const acpAttachmentBuffers = vi.hoisted(() => new Map<string, Buffer>());
@@ -1192,6 +1195,47 @@ describe("tryDispatchAcpReply", () => {
11921195
]);
11931196
});
11941197

1198+
it("forwards media-understanding PDF page images alongside current image attachments", async () => {
1199+
setReadyAcpResolution();
1200+
const currentPath = "/tmp/openclaw-current-image.png";
1201+
const currentImage = Buffer.from("current-image");
1202+
const pdfPage = {
1203+
type: "image" as const,
1204+
mimeType: "image/png",
1205+
data: Buffer.from("pdf-page").toString("base64"),
1206+
attachmentIndex: 1,
1207+
};
1208+
acpAttachmentBuffers.set(currentPath, currentImage);
1209+
mediaUnderstandingMocks.applyMediaUnderstanding.mockResolvedValueOnce({
1210+
outputs: [],
1211+
decisions: [],
1212+
extractedFileImages: [pdfPage],
1213+
appliedImage: false,
1214+
appliedAudio: false,
1215+
appliedVideo: false,
1216+
appliedFile: true,
1217+
});
1218+
1219+
await runDispatch({
1220+
bodyForAgent: "describe current image and scanned PDF",
1221+
ctxOverrides: {
1222+
MediaPath: currentPath,
1223+
MediaType: "image/png",
1224+
},
1225+
});
1226+
1227+
expect(runTurnCall().attachments).toEqual([
1228+
{
1229+
mediaType: "image/png",
1230+
data: currentImage.toString("base64"),
1231+
},
1232+
{
1233+
mediaType: "image/png",
1234+
data: pdfPage.data,
1235+
},
1236+
]);
1237+
});
1238+
11951239
it("preserves chat.send inline image attachments over recent history images", async () => {
11961240
setReadyAcpResolution();
11971241
const image = {

0 commit comments

Comments
 (0)