Skip to content

Commit 50e7aea

Browse files
committed
fix: skip unsupported image MIME types in mentions
1 parent c6baabf commit 50e7aea

2 files changed

Lines changed: 136 additions & 20 deletions

File tree

‎src/core/mentions/__tests__/resolveImageMentions.spec.ts‎

Lines changed: 126 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import * as path from "path"
22

33
import { resolveImageMentions } from "../resolveImageMentions"
4+
import { formatResponse } from "../../prompts/responses"
45

56
vi.mock("../../tools/helpers/imageHelpers", () => ({
67
isSupportedImageFormat: vi.fn((ext: string) =>
@@ -11,9 +12,12 @@ vi.mock("../../tools/helpers/imageHelpers", () => ({
1112
readImageAsDataUrlWithBuffer: vi.fn(),
1213
validateImageForProcessing: vi.fn(),
1314
ImageMemoryTracker: vi.fn().mockImplementation(function () {
15+
let totalMemoryUsed = 0
1416
return {
15-
getTotalMemoryUsed: vi.fn().mockReturnValue(0),
16-
addMemoryUsage: vi.fn(),
17+
getTotalMemoryUsed: vi.fn(() => totalMemoryUsed),
18+
addMemoryUsage: vi.fn((sizeInMB: number) => {
19+
totalMemoryUsed += sizeInMB
20+
}),
1721
}
1822
}),
1923
DEFAULT_MAX_IMAGE_FILE_SIZE_MB: 5,
@@ -28,50 +32,154 @@ const mockValidateImage = vi.mocked(validateImageForProcessing)
2832
describe("resolveImageMentions", () => {
2933
beforeEach(() => {
3034
vi.clearAllMocks()
35+
mockReadImageAsDataUrl.mockReset()
36+
mockValidateImage.mockReset()
3137
// Default: validation passes
3238
mockValidateImage.mockResolvedValue({ isValid: true, sizeInMB: 0.1 })
3339
})
3440

35-
it("should append a data URL when a local png mention is present", async () => {
36-
const dataUrl = `data:image/png;base64,${Buffer.from("png-bytes").toString("base64")}`
37-
mockReadImageAsDataUrl.mockResolvedValue({ dataUrl, buffer: Buffer.from("png-bytes") })
41+
it.each([
42+
["png", "image/png"],
43+
["PNG", "image/png"],
44+
["jpg", "image/jpeg"],
45+
["jpeg", "image/jpeg"],
46+
["gif", "image/gif"],
47+
["webp", "image/webp"],
48+
])("should append a supported %s image mention", async (extension, mimeType) => {
49+
const buffer = Buffer.from("image-bytes")
50+
const dataUrl = `data:${mimeType};base64,${buffer.toString("base64")}`
51+
mockReadImageAsDataUrl.mockResolvedValue({ dataUrl, buffer })
52+
const text = `Please look at @/assets/cat.${extension}`
3853

3954
const result = await resolveImageMentions({
40-
text: "Please look at @/assets/cat.png",
55+
text,
4156
images: [],
4257
cwd: "/workspace",
4358
})
4459

4560
expect(mockValidateImage).toHaveBeenCalled()
46-
expect(mockReadImageAsDataUrl).toHaveBeenCalledWith(path.resolve("/workspace", "assets/cat.png"))
47-
expect(result.text).toBe("Please look at @/assets/cat.png")
61+
expect(mockReadImageAsDataUrl).toHaveBeenCalledWith(path.resolve("/workspace", `assets/cat.${extension}`))
62+
expect(result.text).toBe(text)
4863
expect(result.images).toEqual([dataUrl])
64+
expect(formatResponse.imageBlocks(result.images)).toEqual([
65+
{
66+
type: "image",
67+
source: { type: "base64", media_type: mimeType, data: buffer.toString("base64") },
68+
},
69+
])
4970
})
5071

51-
it("should support gif images (matching read_file)", async () => {
52-
const dataUrl = `data:image/gif;base64,${Buffer.from("gif-bytes").toString("base64")}`
53-
mockReadImageAsDataUrl.mockResolvedValue({ dataUrl, buffer: Buffer.from("gif-bytes") })
72+
it.each([
73+
["svg", "image/svg+xml"],
74+
["bmp", "image/bmp"],
75+
["ico", "image/x-icon"],
76+
["tiff", "image/tiff"],
77+
["tif", "image/tiff"],
78+
["avif", "image/avif"],
79+
])("should skip %s mentions with unsupported MIME types", async (extension, mimeType) => {
80+
const buffer = Buffer.from("unsupported-image-bytes")
81+
const dataUrl = `data:${mimeType};base64,${buffer.toString("base64")}`
82+
mockReadImageAsDataUrl.mockResolvedValue({ dataUrl, buffer })
83+
const text = `See @/image.${extension}`
5484

5585
const result = await resolveImageMentions({
56-
text: "See @/animation.gif",
86+
text,
5787
images: [],
5888
cwd: "/workspace",
5989
})
6090

61-
expect(result.images).toEqual([dataUrl])
91+
expect(result.text).toBe(text)
92+
expect(result.images).toEqual([])
93+
expect(formatResponse.imageBlocks(result.images)).toEqual([])
94+
})
95+
96+
it.each(["not-a-data-url", "data:;base64,aW1hZ2U="])(
97+
"should skip image data URLs without a recognized MIME type: %s",
98+
async (dataUrl) => {
99+
mockReadImageAsDataUrl.mockResolvedValue({ dataUrl, buffer: Buffer.from("image-bytes") })
100+
101+
const result = await resolveImageMentions({
102+
text: "See @/image.png",
103+
images: [],
104+
cwd: "/workspace",
105+
})
106+
107+
expect(result.images).toEqual([])
108+
},
109+
)
110+
111+
it("should preserve existing images and supported mentions when unsupported mentions are mixed in", async () => {
112+
const buffer = Buffer.from("image-bytes")
113+
const base64 = buffer.toString("base64")
114+
const existingImage = `data:image/png;base64,${base64}`
115+
const supportedImage = `data:image/webp;base64,${base64}`
116+
mockReadImageAsDataUrl
117+
.mockResolvedValueOnce({ dataUrl: `data:image/svg+xml;base64,${base64}`, buffer })
118+
.mockResolvedValueOnce({ dataUrl: supportedImage, buffer })
119+
.mockResolvedValueOnce({ dataUrl: `data:image/bmp;base64,${base64}`, buffer })
120+
const text = "Compare @/icon.svg, @/photo.webp and @/bitmap.bmp"
121+
122+
const result = await resolveImageMentions({
123+
text,
124+
images: [existingImage],
125+
cwd: "/workspace",
126+
})
127+
128+
expect(mockReadImageAsDataUrl).toHaveBeenCalledTimes(3)
129+
expect(result.text).toBe(text)
130+
expect(result.images).toEqual([existingImage, supportedImage])
131+
expect(formatResponse.imageBlocks(result.images)).toHaveLength(2)
62132
})
63133

64-
it("should support svg images (matching read_file)", async () => {
65-
const dataUrl = `data:image/svg+xml;base64,${Buffer.from("svg-bytes").toString("base64")}`
66-
mockReadImageAsDataUrl.mockResolvedValue({ dataUrl, buffer: Buffer.from("svg-bytes") })
134+
it("should not count unsupported image mentions toward the total memory limit", async () => {
135+
mockValidateImage.mockImplementation(
136+
async (_filePath, _supportsImages, _maxImageFileSize, maxTotalImageSize, currentTotalMemoryUsed) => ({
137+
isValid: currentTotalMemoryUsed + 10 <= maxTotalImageSize,
138+
sizeInMB: 10,
139+
}),
140+
)
141+
const buffer = Buffer.from("image-bytes")
142+
const base64 = buffer.toString("base64")
143+
const firstSupportedImage = `data:image/png;base64,${base64}`
144+
const secondSupportedImage = `data:image/jpeg;base64,${base64}`
145+
mockReadImageAsDataUrl
146+
.mockResolvedValueOnce({ dataUrl: `data:image/svg+xml;base64,${base64}`, buffer })
147+
.mockResolvedValueOnce({ dataUrl: firstSupportedImage, buffer })
148+
.mockResolvedValueOnce({ dataUrl: secondSupportedImage, buffer })
67149

68150
const result = await resolveImageMentions({
69-
text: "See @/icon.svg",
151+
text: "See @/icon.svg, @/first.png and @/second.jpg",
70152
images: [],
71153
cwd: "/workspace",
154+
maxImageFileSize: 10,
155+
maxTotalImageSize: 20,
72156
})
73157

74-
expect(result.images).toEqual([dataUrl])
158+
expect(result.images).toEqual([firstSupportedImage, secondSupportedImage])
159+
expect(mockValidateImage).toHaveBeenNthCalledWith(2, path.resolve("/workspace", "first.png"), true, 10, 20, 0)
160+
expect(mockValidateImage).toHaveBeenNthCalledWith(3, path.resolve("/workspace", "second.jpg"), true, 10, 20, 10)
161+
})
162+
163+
it("should not count unsupported image mentions toward the image count limit", async () => {
164+
const existingImages = Array.from(
165+
{ length: 19 },
166+
(_, index) => `data:image/png;base64,${Buffer.from(`existing-${index}`).toString("base64")}`,
167+
)
168+
const buffer = Buffer.from("new-image-bytes")
169+
const base64 = buffer.toString("base64")
170+
const supportedImage = `data:image/png;base64,${base64}`
171+
mockReadImageAsDataUrl
172+
.mockResolvedValueOnce({ dataUrl: `data:image/svg+xml;base64,${base64}`, buffer })
173+
.mockResolvedValueOnce({ dataUrl: supportedImage, buffer })
174+
175+
const result = await resolveImageMentions({
176+
text: "See @/icon.svg, @/last.png and @/over-limit.jpg",
177+
images: existingImages,
178+
cwd: "/workspace",
179+
})
180+
181+
expect(result.images).toEqual([...existingImages, supportedImage])
182+
expect(mockReadImageAsDataUrl).toHaveBeenCalledTimes(2)
75183
})
76184

77185
it("should ignore non-image mentions", async () => {

‎src/core/mentions/resolveImageMentions.ts‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import * as path from "path"
22

33
import { mentionRegexGlobal, unescapeSpaces } from "../../shared/context-mentions"
4+
import { getImageMimeType, isSupportedImageMimeType } from "../../utils/imageMime"
45
import {
56
isSupportedImageFormat,
67
readImageAsDataUrlWithBuffer,
@@ -50,8 +51,11 @@ function dedupePreserveOrder(values: string[]): string[] {
5051
* Resolves local image file mentions like `@/path/to/image.png` found in `text` into `data:image/...;base64,...`
5152
* and appends them to the outgoing `images` array.
5253
*
53-
* Behavior matches the read_file tool:
54-
* - Supports the same image formats: png, jpg, jpeg, gif, webp, svg, bmp, ico, tiff, avif
54+
* Recognizes the same image extensions as the read_file tool, but only appends images
55+
* supported by the shared image-block contract (JPEG, PNG, GIF, and WebP).
56+
* Other image MIME types are skipped without changing the original text.
57+
*
58+
* Like the read_file tool:
5559
* - Respects per-file size limits (default 5MB)
5660
* - Respects total memory limits (default 20MB)
5761
* - Skips images if model doesn't support them
@@ -128,6 +132,10 @@ export async function resolveImageMentions({
128132
}
129133

130134
const { dataUrl } = await readImageAsDataUrlWithBuffer(absPath)
135+
if (!isSupportedImageMimeType(getImageMimeType(dataUrl))) {
136+
// Fail-soft: skip formats unsupported by the shared image-block contract.
137+
continue
138+
}
131139
newImages.push(dataUrl)
132140

133141
// Track memory usage

0 commit comments

Comments
 (0)