Skip to content

Commit b7ab5a8

Browse files
authored
fix: reject unsupported MIME types in image blocks (#1961)
* fix: reject unsupported MIME types in image blocks Extract the shared image MIME guard from PR #1950 as a standalone prerequisite for #1948. Preserve the supported SDK payload contract and cover both formatter entry points, malformed headers, Unicode lookalikes, and opaque payload boundaries. * fix: skip unsupported image MIME types in mentions
1 parent 1a98c55 commit b7ab5a8

5 files changed

Lines changed: 286 additions & 24 deletions

File tree

‎src/core/mentions/__tests__/resolveImageMentions.spec.ts‎

Lines changed: 126 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import * as path from "path"
22

33
import { resolveImageMentions } from "../resolveImageMentions"
4+
import { formatResponse } from "../../prompts/responses"
45

56
vi.mock("../../tools/helpers/imageHelpers", () => ({
67
isSupportedImageFormat: vi.fn((ext: string) =>
@@ -11,9 +12,12 @@ vi.mock("../../tools/helpers/imageHelpers", () => ({
1112
readImageAsDataUrlWithBuffer: vi.fn(),
1213
validateImageForProcessing: vi.fn(),
1314
ImageMemoryTracker: vi.fn().mockImplementation(function () {
15+
let totalMemoryUsed = 0
1416
return {
15-
getTotalMemoryUsed: vi.fn().mockReturnValue(0),
16-
addMemoryUsage: vi.fn(),
17+
getTotalMemoryUsed: vi.fn(() => totalMemoryUsed),
18+
addMemoryUsage: vi.fn((sizeInMB: number) => {
19+
totalMemoryUsed += sizeInMB
20+
}),
1721
}
1822
}),
1923
DEFAULT_MAX_IMAGE_FILE_SIZE_MB: 5,
@@ -28,50 +32,154 @@ const mockValidateImage = vi.mocked(validateImageForProcessing)
2832
describe("resolveImageMentions", () => {
2933
beforeEach(() => {
3034
vi.clearAllMocks()
35+
mockReadImageAsDataUrl.mockReset()
36+
mockValidateImage.mockReset()
3137
// Default: validation passes
3238
mockValidateImage.mockResolvedValue({ isValid: true, sizeInMB: 0.1 })
3339
})
3440

35-
it("should append a data URL when a local png mention is present", async () => {
36-
const dataUrl = `data:image/png;base64,${Buffer.from("png-bytes").toString("base64")}`
37-
mockReadImageAsDataUrl.mockResolvedValue({ dataUrl, buffer: Buffer.from("png-bytes") })
41+
it.each([
42+
["png", "image/png"],
43+
["PNG", "image/png"],
44+
["jpg", "image/jpeg"],
45+
["jpeg", "image/jpeg"],
46+
["gif", "image/gif"],
47+
["webp", "image/webp"],
48+
])("should append a supported %s image mention", async (extension, mimeType) => {
49+
const buffer = Buffer.from("image-bytes")
50+
const dataUrl = `data:${mimeType};base64,${buffer.toString("base64")}`
51+
mockReadImageAsDataUrl.mockResolvedValue({ dataUrl, buffer })
52+
const text = `Please look at @/assets/cat.${extension}`
3853

3954
const result = await resolveImageMentions({
40-
text: "Please look at @/assets/cat.png",
55+
text,
4156
images: [],
4257
cwd: "/workspace",
4358
})
4459

4560
expect(mockValidateImage).toHaveBeenCalled()
46-
expect(mockReadImageAsDataUrl).toHaveBeenCalledWith(path.resolve("/workspace", "assets/cat.png"))
47-
expect(result.text).toBe("Please look at @/assets/cat.png")
61+
expect(mockReadImageAsDataUrl).toHaveBeenCalledWith(path.resolve("/workspace", `assets/cat.${extension}`))
62+
expect(result.text).toBe(text)
4863
expect(result.images).toEqual([dataUrl])
64+
expect(formatResponse.imageBlocks(result.images)).toEqual([
65+
{
66+
type: "image",
67+
source: { type: "base64", media_type: mimeType, data: buffer.toString("base64") },
68+
},
69+
])
4970
})
5071

51-
it("should support gif images (matching read_file)", async () => {
52-
const dataUrl = `data:image/gif;base64,${Buffer.from("gif-bytes").toString("base64")}`
53-
mockReadImageAsDataUrl.mockResolvedValue({ dataUrl, buffer: Buffer.from("gif-bytes") })
72+
it.each([
73+
["svg", "image/svg+xml"],
74+
["bmp", "image/bmp"],
75+
["ico", "image/x-icon"],
76+
["tiff", "image/tiff"],
77+
["tif", "image/tiff"],
78+
["avif", "image/avif"],
79+
])("should skip %s mentions with unsupported MIME types", async (extension, mimeType) => {
80+
const buffer = Buffer.from("unsupported-image-bytes")
81+
const dataUrl = `data:${mimeType};base64,${buffer.toString("base64")}`
82+
mockReadImageAsDataUrl.mockResolvedValue({ dataUrl, buffer })
83+
const text = `See @/image.${extension}`
5484

5585
const result = await resolveImageMentions({
56-
text: "See @/animation.gif",
86+
text,
5787
images: [],
5888
cwd: "/workspace",
5989
})
6090

61-
expect(result.images).toEqual([dataUrl])
91+
expect(result.text).toBe(text)
92+
expect(result.images).toEqual([])
93+
expect(formatResponse.imageBlocks(result.images)).toEqual([])
94+
})
95+
96+
it.each(["not-a-data-url", "data:;base64,aW1hZ2U="])(
97+
"should skip image data URLs without a recognized MIME type: %s",
98+
async (dataUrl) => {
99+
mockReadImageAsDataUrl.mockResolvedValue({ dataUrl, buffer: Buffer.from("image-bytes") })
100+
101+
const result = await resolveImageMentions({
102+
text: "See @/image.png",
103+
images: [],
104+
cwd: "/workspace",
105+
})
106+
107+
expect(result.images).toEqual([])
108+
},
109+
)
110+
111+
it("should preserve existing images and supported mentions when unsupported mentions are mixed in", async () => {
112+
const buffer = Buffer.from("image-bytes")
113+
const base64 = buffer.toString("base64")
114+
const existingImage = `data:image/png;base64,${base64}`
115+
const supportedImage = `data:image/webp;base64,${base64}`
116+
mockReadImageAsDataUrl
117+
.mockResolvedValueOnce({ dataUrl: `data:image/svg+xml;base64,${base64}`, buffer })
118+
.mockResolvedValueOnce({ dataUrl: supportedImage, buffer })
119+
.mockResolvedValueOnce({ dataUrl: `data:image/bmp;base64,${base64}`, buffer })
120+
const text = "Compare @/icon.svg, @/photo.webp and @/bitmap.bmp"
121+
122+
const result = await resolveImageMentions({
123+
text,
124+
images: [existingImage],
125+
cwd: "/workspace",
126+
})
127+
128+
expect(mockReadImageAsDataUrl).toHaveBeenCalledTimes(3)
129+
expect(result.text).toBe(text)
130+
expect(result.images).toEqual([existingImage, supportedImage])
131+
expect(formatResponse.imageBlocks(result.images)).toHaveLength(2)
62132
})
63133

64-
it("should support svg images (matching read_file)", async () => {
65-
const dataUrl = `data:image/svg+xml;base64,${Buffer.from("svg-bytes").toString("base64")}`
66-
mockReadImageAsDataUrl.mockResolvedValue({ dataUrl, buffer: Buffer.from("svg-bytes") })
134+
it("should not count unsupported image mentions toward the total memory limit", async () => {
135+
mockValidateImage.mockImplementation(
136+
async (_filePath, _supportsImages, _maxImageFileSize, maxTotalImageSize, currentTotalMemoryUsed) => ({
137+
isValid: currentTotalMemoryUsed + 10 <= maxTotalImageSize,
138+
sizeInMB: 10,
139+
}),
140+
)
141+
const buffer = Buffer.from("image-bytes")
142+
const base64 = buffer.toString("base64")
143+
const firstSupportedImage = `data:image/png;base64,${base64}`
144+
const secondSupportedImage = `data:image/jpeg;base64,${base64}`
145+
mockReadImageAsDataUrl
146+
.mockResolvedValueOnce({ dataUrl: `data:image/svg+xml;base64,${base64}`, buffer })
147+
.mockResolvedValueOnce({ dataUrl: firstSupportedImage, buffer })
148+
.mockResolvedValueOnce({ dataUrl: secondSupportedImage, buffer })
67149

68150
const result = await resolveImageMentions({
69-
text: "See @/icon.svg",
151+
text: "See @/icon.svg, @/first.png and @/second.jpg",
70152
images: [],
71153
cwd: "/workspace",
154+
maxImageFileSize: 10,
155+
maxTotalImageSize: 20,
72156
})
73157

74-
expect(result.images).toEqual([dataUrl])
158+
expect(result.images).toEqual([firstSupportedImage, secondSupportedImage])
159+
expect(mockValidateImage).toHaveBeenNthCalledWith(2, path.resolve("/workspace", "first.png"), true, 10, 20, 0)
160+
expect(mockValidateImage).toHaveBeenNthCalledWith(3, path.resolve("/workspace", "second.jpg"), true, 10, 20, 10)
161+
})
162+
163+
it("should not count unsupported image mentions toward the image count limit", async () => {
164+
const existingImages = Array.from(
165+
{ length: 19 },
166+
(_, index) => `data:image/png;base64,${Buffer.from(`existing-${index}`).toString("base64")}`,
167+
)
168+
const buffer = Buffer.from("new-image-bytes")
169+
const base64 = buffer.toString("base64")
170+
const supportedImage = `data:image/png;base64,${base64}`
171+
mockReadImageAsDataUrl
172+
.mockResolvedValueOnce({ dataUrl: `data:image/svg+xml;base64,${base64}`, buffer })
173+
.mockResolvedValueOnce({ dataUrl: supportedImage, buffer })
174+
175+
const result = await resolveImageMentions({
176+
text: "See @/icon.svg, @/last.png and @/over-limit.jpg",
177+
images: existingImages,
178+
cwd: "/workspace",
179+
})
180+
181+
expect(result.images).toEqual([...existingImages, supportedImage])
182+
expect(mockReadImageAsDataUrl).toHaveBeenCalledTimes(2)
75183
})
76184

77185
it("should ignore non-image mentions", async () => {

‎src/core/mentions/resolveImageMentions.ts‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import * as path from "path"
22

33
import { mentionRegexGlobal, unescapeSpaces } from "../../shared/context-mentions"
4+
import { getImageMimeType, isSupportedImageMimeType } from "../../utils/imageMime"
45
import {
56
isSupportedImageFormat,
67
readImageAsDataUrlWithBuffer,
@@ -50,8 +51,11 @@ function dedupePreserveOrder(values: string[]): string[] {
5051
* Resolves local image file mentions like `@/path/to/image.png` found in `text` into `data:image/...;base64,...`
5152
* and appends them to the outgoing `images` array.
5253
*
53-
* Behavior matches the read_file tool:
54-
* - Supports the same image formats: png, jpg, jpeg, gif, webp, svg, bmp, ico, tiff, avif
54+
* Recognizes the same image extensions as the read_file tool, but only appends images
55+
* supported by the shared image-block contract (JPEG, PNG, GIF, and WebP).
56+
* Other image MIME types are skipped without changing the original text.
57+
*
58+
* Like the read_file tool:
5559
* - Respects per-file size limits (default 5MB)
5660
* - Respects total memory limits (default 20MB)
5761
* - Skips images if model doesn't support them
@@ -128,6 +132,10 @@ export async function resolveImageMentions({
128132
}
129133

130134
const { dataUrl } = await readImageAsDataUrlWithBuffer(absPath)
135+
if (!isSupportedImageMimeType(getImageMimeType(dataUrl))) {
136+
// Fail-soft: skip formats unsupported by the shared image-block contract.
137+
continue
138+
}
131139
newImages.push(dataUrl)
132140

133141
// Track memory usage
Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
import { formatResponse } from "../responses"
2+
import { IMAGE_MIME_TYPES } from "../../tools/helpers/imageHelpers"
3+
import { getImageMimeType, isSupportedImageMimeType } from "../../../utils/imageMime"
4+
5+
const supportedMimeTypes = ["image/jpeg", "image/png", "image/gif", "image/webp"] as const
6+
const supportedMimeTypeSet = new Set<string>(supportedMimeTypes)
7+
const unsupportedMimeTypes = [
8+
...new Set([...Object.values(IMAGE_MIME_TYPES), "image/heic", "application/octet-stream", "image/svg+xml"]),
9+
].filter((mimeType) => !supportedMimeTypeSet.has(mimeType))
10+
const payload = "c3ludGhldGlj"
11+
12+
describe("image MIME utilities", () => {
13+
it.each(supportedMimeTypes)("extracts and accepts the SDK MIME type %s", (mimeType) => {
14+
expect(getImageMimeType(`data:${mimeType};base64,${payload}`)).toBe(mimeType)
15+
expect(isSupportedImageMimeType(mimeType)).toBe(true)
16+
})
17+
18+
it.each(unsupportedMimeTypes)("extracts but rejects the unsupported MIME type %s", (mimeType) => {
19+
expect(getImageMimeType(`data:${mimeType};base64,${payload}`)).toBe(mimeType)
20+
expect(isSupportedImageMimeType(mimeType)).toBe(false)
21+
})
22+
23+
it("returns no MIME for a missing header and rejects absent or empty MIME values", () => {
24+
expect(getImageMimeType("")).toBeUndefined()
25+
expect(isSupportedImageMimeType(undefined)).toBe(false)
26+
expect(isSupportedImageMimeType("")).toBe(false)
27+
})
28+
})
29+
30+
describe("shared image-block MIME contract", () => {
31+
it.each(supportedMimeTypes)("preserves the complete image block for %s through both entry points", (mimeType) => {
32+
const dataUrl = `data:${mimeType};base64,${payload}`
33+
const image = { type: "image", source: { type: "base64", media_type: mimeType, data: payload } }
34+
expect(formatResponse.imageBlocks([dataUrl])).toEqual([image])
35+
expect(formatResponse.toolResult("read result", [dataUrl])).toEqual([
36+
{ type: "text", text: "read result" },
37+
image,
38+
])
39+
})
40+
41+
it.each(unsupportedMimeTypes)("rejects %s through both entry points before returning a payload", (mimeType) => {
42+
const dataUrl = `data:${mimeType};base64,${payload}`
43+
const error = new Error(`Unsupported image MIME type: ${mimeType}. Use JPEG, PNG, GIF, or WebP.`)
44+
expect(() => formatResponse.imageBlocks([dataUrl])).toThrowError(error)
45+
expect(() => formatResponse.toolResult("read result", [dataUrl])).toThrowError(error)
46+
})
47+
48+
it.each([
49+
"",
50+
"not a data URL",
51+
"data:;base64,c3ludGhldGlj",
52+
"data:image/png,c3ludGhldGlj",
53+
"data:image/png;base64",
54+
"data:image/png;base64;c3ludGhldGlj",
55+
"data:image/png;charset=utf-8;base64,c3ludGhldGlj",
56+
"DATA:image/png;base64,c3ludGhldGlj",
57+
"data:image/png;BASE64,c3ludGhldGlj",
58+
"prefix data:image/png;base64,c3ludGhldGlj",
59+
" data:image/png;base64,c3ludGhldGlj",
60+
"\uFEFFdata:image/png;base64,c3ludGhldGlj",
61+
"data\uFF1Aimage/png;base64,c3ludGhldGlj",
62+
"data:image/png;base64\uFF0Cc3ludGhldGlj",
63+
])("rejects a malformed or noncanonical header: %j", (dataUrl) => {
64+
const error = new Error("Unsupported image MIME type: unknown. Use JPEG, PNG, GIF, or WebP.")
65+
expect(() => formatResponse.imageBlocks([dataUrl])).toThrowError(error)
66+
expect(() => formatResponse.toolResult("read result", [dataUrl])).toThrowError(error)
67+
})
68+
69+
it.each(["image/PNG", "image/jpg", "image/png ", "image/p\u200Bng", "image/\u0440ng", "image\uFF0Fpng"])(
70+
"does not normalize unsupported MIME spelling %j into an allowed type",
71+
(mimeType) => {
72+
const dataUrl = `data:${mimeType};base64,${payload}`
73+
const error = new Error(`Unsupported image MIME type: ${mimeType}. Use JPEG, PNG, GIF, or WebP.`)
74+
expect(() => formatResponse.imageBlocks([dataUrl])).toThrowError(error)
75+
expect(() => formatResponse.toolResult("read result", [dataUrl])).toThrowError(error)
76+
},
77+
)
78+
79+
it.each([{ images: undefined }, { images: [] }])("preserves the no-image response for $images", ({ images }) => {
80+
expect(formatResponse.imageBlocks(images)).toEqual([])
81+
expect(formatResponse.toolResult("read result", images)).toBe("read result")
82+
expect(formatResponse.toolResult("", images)).toBe("")
83+
})
84+
85+
it("preserves image order, duplicate images, Unicode feedback, and the input array", () => {
86+
const images = [
87+
"data:image/png;base64,Zmlyc3Q=",
88+
"data:image/jpeg;base64,c2Vjb25k",
89+
"data:image/png;base64,Zmlyc3Q=",
90+
]
91+
const originalImages = [...images]
92+
const expected = [
93+
{ type: "image", source: { type: "base64", media_type: "image/png", data: "Zmlyc3Q=" } },
94+
{ type: "image", source: { type: "base64", media_type: "image/jpeg", data: "c2Vjb25k" } },
95+
{ type: "image", source: { type: "base64", media_type: "image/png", data: "Zmlyc3Q=" } },
96+
]
97+
const text = "\u041E\u0442\u0432\u0435\u0442 \uD83D\uDDBC\uFE0F e\u0301"
98+
expect(formatResponse.imageBlocks(images)).toEqual(expected)
99+
expect(formatResponse.toolResult(text, images)).toEqual([{ type: "text", text }, ...expected])
100+
expect(images).toEqual(originalImages)
101+
})
102+
103+
it.each([0, 1, 2])("rejects an unsupported image at index %i without changing the input", (index) => {
104+
const images = Array<string>(3).fill(`data:image/png;base64,${payload}`)
105+
images[index] = `data:image/bmp;base64,${payload}`
106+
const originalImages = [...images]
107+
const error = new Error("Unsupported image MIME type: image/bmp. Use JPEG, PNG, GIF, or WebP.")
108+
expect(() => formatResponse.imageBlocks(images)).toThrowError(error)
109+
expect(() => formatResponse.toolResult("read result", images)).toThrowError(error)
110+
expect(images).toEqual(originalImages)
111+
})
112+
113+
it.each(["", "not decoded here", "first,second", "\uD83D\uDE00e\u0301"])(
114+
"preserves the opaque payload %j without adding byte or base64 validation",
115+
(data) => {
116+
const dataUrl = `data:image/png;base64,${data}`
117+
const image = { type: "image", source: { type: "base64", media_type: "image/png", data } }
118+
expect(formatResponse.imageBlocks([dataUrl])).toEqual([image])
119+
expect(formatResponse.toolResult("", [dataUrl])).toEqual([{ type: "text", text: "" }, image])
120+
},
121+
)
122+
})

‎src/core/prompts/responses.ts‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import * as path from "path"
33
import * as diff from "diff"
44
import { RooIgnoreController, LOCK_TEXT_SYMBOL } from "../ignore/RooIgnoreController"
55
import { RooProtectedController } from "../protect/RooProtectedController"
6+
import { getImageMimeType, isSupportedImageMimeType } from "../../utils/imageMime"
67

78
export const formatResponse = {
89
toolDenied: () =>
@@ -226,14 +227,19 @@ Otherwise, if you have not completed the task and do not need additional informa
226227
// to avoid circular dependency
227228
const formatImagesIntoBlocks = (images?: string[]): Anthropic.ImageBlockParam[] => {
228229
return images
229-
? images.map((dataUrl) => {
230+
? images.map((dataUrl): Anthropic.ImageBlockParam => {
230231
// data:image/png;base64,base64string
231-
const [rest, base64] = dataUrl.split(",")
232-
const mimeType = rest.split(":")[1].split(";")[0]
232+
const mimeType = getImageMimeType(dataUrl)
233+
if (!isSupportedImageMimeType(mimeType)) {
234+
throw new Error(
235+
`Unsupported image MIME type: ${mimeType ?? "unknown"}. Use JPEG, PNG, GIF, or WebP.`,
236+
)
237+
}
238+
const base64 = dataUrl.slice(dataUrl.indexOf(",") + 1)
233239
return {
234240
type: "image",
235241
source: { type: "base64", media_type: mimeType, data: base64 },
236-
} as Anthropic.ImageBlockParam
242+
}
237243
})
238244
: []
239245
}

0 commit comments

Comments
 (0)