Skip to content

Commit 1a36c8d

Browse files
wanyongstarwanyongstar
committed
fix(media): forward scanned PDF page images as current-turn images
1 parent 2e2c824 commit 1a36c8d

4 files changed

Lines changed: 65 additions & 24 deletions

File tree

‎src/auto-reply/reply/current-turn-images.ts‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,23 @@ export async function resolveCurrentTurnImages(params: {
9898
images?: ImageContent[];
9999
imageOrder?: PromptImageOrderEntry[];
100100
}> {
101+
// Consume extracted current-turn images (e.g., PDF page images from file extraction).
102+
const currentTurnImages = params.ctx.CurrentTurnImages;
103+
if (currentTurnImages && currentTurnImages.length > 0) {
104+
// Clear so they are consumed once.
105+
delete params.ctx.CurrentTurnImages;
106+
const mergedImages = [
107+
...(Array.isArray(params.images) ? params.images : []),
108+
...currentTurnImages,
109+
];
110+
if (mergedImages.length > 0) {
111+
return {
112+
images: mergedImages,
113+
imageOrder: mergedImages.map(() => "inline" as const),
114+
};
115+
}
116+
}
117+
101118
if (Array.isArray(params.images) && params.images.length > 0) {
102119
return { images: params.images, imageOrder: params.imageOrder };
103120
}

‎src/auto-reply/reply/dispatch-acp.ts‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -558,20 +558,38 @@ export async function tryDispatchAcpReply(params: {
558558
}
559559

560560
const promptText = resolveAcpPromptText(params.ctx);
561+
562+
// Consume extracted current-turn images (e.g., PDF page images from file extraction)
563+
// forwarded by applyMediaUnderstanding via ctx.CurrentTurnImages.
564+
const currentTurnImages = params.ctx.CurrentTurnImages;
565+
if (currentTurnImages && currentTurnImages.length > 0) {
566+
delete params.ctx.CurrentTurnImages;
567+
}
568+
561569
const resolvedTurnAttachments = await resolveAgentTurnAttachments({
562570
ctx: params.ctx,
563571
cfg: params.cfg,
564572
});
565573
const mediaAttachments = resolvedTurnAttachments.attachments;
566574
const inlineAttachments = resolveInlineAgentImageAttachments(params.images);
575+
// Merge extracted current-turn images (PDF page rasters) into inline attachments.
576+
const allInlineAttachments = currentTurnImages && currentTurnImages.length > 0
577+
? [
578+
...inlineAttachments,
579+
...currentTurnImages.map((img) => ({
580+
mediaType: img.mimeType,
581+
data: img.data,
582+
})),
583+
]
584+
: inlineAttachments;
567585
const mediaAttachmentsAreOnlyRecentHistory =
568586
mediaAttachments.length > 0 &&
569587
mediaAttachments.length === resolvedTurnAttachments.recentHistoryImages.length;
570588
const attachments =
571589
mediaAttachments.length > 0 &&
572-
!(mediaAttachmentsAreOnlyRecentHistory && inlineAttachments.length > 0)
590+
!(mediaAttachmentsAreOnlyRecentHistory && allInlineAttachments.length > 0)
573591
? mediaAttachments
574-
: inlineAttachments;
592+
: allInlineAttachments;
575593
const turnPromptText =
576594
attachments === mediaAttachments
577595
? appendRecentHistoryImageContext({

‎src/auto-reply/templating.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import type {
55
MediaUnderstandingOutput,
66
} from "../media-understanding/types.js";
77
import type { PluginHookChannelContext } from "../plugins/hook-channel-context.types.js";
8+
import type { ImageContent } from "../llm/types.js";
89
import type { InputProvenance } from "../sessions/input-provenance.js";
910
import type { CommandTurnContext } from "./command-turn-context.js";
1011
import type { CommandArgs } from "./commands-args.types.js";
@@ -217,6 +218,8 @@ export type MsgContext = {
217218
/** Remote host for SCP when media lives on a different machine (e.g., [email protected]). */
218219
MediaRemoteHost?: string;
219220
Transcript?: string;
221+
/** Extracted PDF page images from file extraction, forwarded as current-turn image inputs for vision-capable model paths. */
222+
CurrentTurnImages?: ImageContent[];
220223
MediaUnderstanding?: MediaUnderstandingOutput[];
221224
MediaUnderstandingDecisions?: MediaUnderstandingDecision[];
222225
LinkUnderstanding?: string[];

‎src/media-understanding/apply.ts‎

Lines changed: 25 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -5,19 +5,20 @@ import {
55
normalizeLowercaseStringOrEmpty,
66
normalizeOptionalString,
77
} from "@openclaw/normalization-core/string-coerce";
8+
import type { ActiveMediaModel } from "../../packages/media-understanding-common/src/active-model.js";
9+
import {
10+
extractMediaUserText,
11+
formatAudioTranscripts,
12+
formatMediaUnderstandingBody,
13+
} from "../../packages/media-understanding-common/src/format.js";
814
import { finalizeInboundContext } from "../auto-reply/reply/inbound-context.js";
915
import type { MsgContext } from "../auto-reply/templating.js";
1016
import type { OpenClawConfig } from "../config/types.js";
1117
import { logVerbose, shouldLogVerbose } from "../globals.js";
18+
import type { ImageContent } from "../llm/types.js";
1219
import { renderFileContextBlock } from "../media/file-context.js";
1320
import { extractFileContentFromSource, normalizeMimeType } from "../media/input-files.js";
1421
import { wrapExternalContent } from "../security/external-content.js";
15-
import type { ActiveMediaModel } from "../../packages/media-understanding-common/src/active-model.js";
16-
import {
17-
extractMediaUserText,
18-
formatAudioTranscripts,
19-
formatMediaUnderstandingBody,
20-
} from "../../packages/media-understanding-common/src/format.js";
2122
import { resolveAttachmentKind } from "./attachments.js";
2223
import { runWithConcurrency } from "./concurrency.js";
2324
import { DEFAULT_ECHO_TRANSCRIPT_FORMAT, sendTranscriptEcho } from "./echo-transcript.js";
@@ -49,6 +50,11 @@ type ApplyMediaUnderstandingResult = {
4950
appliedFile: boolean;
5051
};
5152

53+
type ExtractedFileBlocks = {
54+
blocks: string[];
55+
images: ImageContent[];
56+
};
57+
5258
const CAPABILITY_ORDER: MediaUnderstandingCapability[] = ["image", "audio", "video"];
5359
const EMPTY_VOICE_NOTE_PLACEHOLDER =
5460
"[Voice note could not be transcribed because the audio attachment was too small]";
@@ -383,12 +389,13 @@ async function extractFileBlocks(params: {
383389
cfg: OpenClawConfig;
384390
limits: FileExtractionLimits;
385391
skipAttachmentIndexes?: Set<number>;
386-
}): Promise<string[]> {
392+
}): Promise<ExtractedFileBlocks> {
387393
const { attachments, cache, cfg, limits, skipAttachmentIndexes } = params;
388394
if (!attachments || attachments.length === 0) {
389-
return [];
395+
return { blocks: [], images: [] };
390396
}
391397
const blocks: string[] = [];
398+
const images: ImageContent[] = [];
392399
for (const attachment of attachments) {
393400
if (!attachment) {
394401
continue;
@@ -494,21 +501,13 @@ async function extractFileBlocks(params: {
494501
continue;
495502
}
496503
const text = extracted?.text?.trim() ?? "";
504+
if (extracted?.images && extracted.images.length > 0) {
505+
images.push(...extracted.images);
506+
}
497507
let blockText = text ? wrapUntrustedAttachmentContent(text) : "";
498508
if (!blockText) {
499509
if (extracted?.images && extracted.images.length > 0) {
500-
// Forward extracted PDF page images as inline base64 data URIs so
501-
// vision-capable models can read scanned documents. Mirrors the
502-
// /v1/responses path behavior (openresponses-http.ts:629-630) where
503-
// the same extractor output is forwarded as separate image content.
504-
//
505-
// Base64 data URIs are used because the chat path has no image-carrying
506-
// field on ApplyMediaUnderstandingResult; embedding them in the text block
507-
// ensures they reach the model without structural type changes.
508-
const imageContext = extracted.images
509-
.map((img, i) => `![Page ${i + 1}](data:${img.mimeType ?? "image/png"};base64,${img.base64})`)
510-
.join("\n\n");
511-
blockText = `[PDF content rendered to images]\n\n${imageContext}`;
510+
blockText = "[PDF content rendered to images]";
512511
} else {
513512
blockText = "[No extractable text]";
514513
}
@@ -522,7 +521,7 @@ async function extractFileBlocks(params: {
522521
}),
523522
);
524523
}
525-
return blocks;
524+
return { blocks, images };
526525
}
527526

528527
export async function applyMediaUnderstanding(params: {
@@ -681,13 +680,17 @@ export async function applyMediaUnderstanding(params: {
681680
)
682681
.map((output) => output.attachmentIndex),
683682
);
684-
const fileBlocks = await extractFileBlocks({
683+
const extractedFiles = await extractFileBlocks({
685684
attachments,
686685
cache,
687686
cfg,
688687
limits: resolveFileExtractionLimits(cfg),
689688
skipAttachmentIndexes: audioAttachmentIndexes.size > 0 ? audioAttachmentIndexes : undefined,
690689
});
690+
const fileBlocks = extractedFiles.blocks;
691+
if (extractedFiles.images.length > 0) {
692+
ctx.CurrentTurnImages = [...(ctx.CurrentTurnImages ?? []), ...extractedFiles.images];
693+
}
691694
if (fileBlocks.length > 0) {
692695
ctx.Body = appendFileBlocks(ctx.Body, fileBlocks);
693696
}

0 commit comments

Comments
 (0)