Skip to content

Commit 97c89d7

Browse files
committed
fix(anthropic): prioritize inline image data over remote URL in dual-source ImageContent
1 parent dfe3f5f commit 97c89d7

2 files changed

Lines changed: 81 additions & 12 deletions

File tree

‎python/semantic_kernel/connectors/ai/anthropic/services/utils.py‎

Lines changed: 37 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -53,19 +53,17 @@ def _infer_image_mime_type(raw_bytes: bytes | bytearray | None, mime_type: str |
5353
)
5454

5555

56-
def _create_image_content(image_content: ImageContent) -> dict[str, Any]:
57-
"""Create an Anthropic image content block from an ImageContent object."""
58-
if image_content.uri:
59-
uri_str = str(image_content.uri)
60-
if uri_str.startswith(("http://", "https://")):
61-
return {
62-
"type": "image",
63-
"source": {
64-
"type": "url",
65-
"url": uri_str,
66-
},
67-
}
56+
def _has_inline_image_data(image_content: ImageContent) -> bool:
57+
"""Check whether the ImageContent carries inline image data (not just a remote reference)."""
58+
data = image_content.data
59+
if isinstance(data, (bytes, bytearray)):
60+
# BinaryContent defaults data to b"" when only a URI is given.
61+
return len(data) > 0
62+
return bool(image_content.data_string) or data is not None
63+
6864

65+
def _create_base64_image_content(image_content: ImageContent) -> dict[str, Any]:
66+
"""Create an Anthropic base64 image content block from inline image data."""
6967
raw_bytes = image_content.data if isinstance(image_content.data, (bytes, bytearray)) else None
7068
mime_type = _infer_image_mime_type(raw_bytes, image_content.mime_type)
7169

@@ -95,6 +93,33 @@ def _create_image_content(image_content: ImageContent) -> dict[str, Any]:
9593
}
9694

9795

96+
def _create_image_content(image_content: ImageContent) -> dict[str, Any]:
97+
"""Create an Anthropic image content block from an ImageContent object.
98+
99+
Inline image data takes precedence over a remote URL: per ImageContent's
100+
contract, its URI may refer to different content than its data, so sending
101+
the URL when bytes are available can send the wrong image (or fail for
102+
URLs requiring authentication).
103+
"""
104+
if _has_inline_image_data(image_content):
105+
return _create_base64_image_content(image_content)
106+
107+
if image_content.uri:
108+
uri_str = str(image_content.uri)
109+
if uri_str.startswith(("http://", "https://")):
110+
return {
111+
"type": "image",
112+
"source": {
113+
"type": "url",
114+
"url": uri_str,
115+
},
116+
}
117+
118+
raise ServiceInvalidRequestError(
119+
"ImageContent without data, data_uri, or valid http(s) uri while formatting message for Anthropic."
120+
)
121+
122+
98123
def _format_user_message(message: ChatMessageContent) -> dict[str, Any]:
99124
"""Format a user message to the expected object for the Anthropic client.
100125

‎python/tests/unit/connectors/ai/anthropic/services/test_anthropic_chat_completion.py‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -373,6 +373,50 @@ def test_format_user_message_image_url_only():
373373
assert formatted["content"][0]["source"]["url"] == url
374374

375375

376+
def test_format_user_message_dual_source_prefers_inline_bytes():
377+
# ImageContent documents that its URI may refer to different content than
378+
# its data, so inline bytes must win over the remote URL.
379+
raw_png = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15c4"
380+
message = ChatMessageContent(
381+
role=AuthorRole.USER,
382+
items=[
383+
ImageContent(
384+
uri="https://example.com/remote.png",
385+
data=raw_png,
386+
mime_type="image/png",
387+
),
388+
],
389+
)
390+
formatted = _format_user_message(message)
391+
assert formatted["role"] == "user"
392+
assert len(formatted["content"]) == 1
393+
assert formatted["content"][0]["type"] == "image"
394+
assert formatted["content"][0]["source"]["type"] == "base64"
395+
assert formatted["content"][0]["source"]["media_type"] == "image/png"
396+
import base64
397+
assert formatted["content"][0]["source"]["data"] == base64.b64encode(raw_png).decode("utf-8")
398+
399+
400+
def test_format_user_message_dual_source_prefers_data_uri():
401+
dummy_b64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
402+
message = ChatMessageContent(
403+
role=AuthorRole.USER,
404+
items=[
405+
ImageContent(
406+
uri="https://example.com/remote.png",
407+
data_uri=f"data:image/png;base64,{dummy_b64}",
408+
),
409+
],
410+
)
411+
formatted = _format_user_message(message)
412+
assert formatted["role"] == "user"
413+
assert len(formatted["content"]) == 1
414+
assert formatted["content"][0]["type"] == "image"
415+
assert formatted["content"][0]["source"]["type"] == "base64"
416+
assert formatted["content"][0]["source"]["media_type"] == "image/png"
417+
assert formatted["content"][0]["source"]["data"] == dummy_b64
418+
419+
376420
def test_format_user_message_raw_bytes_default_mime_type():
377421
raw_png = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15c4"
378422
# When mime_type is omitted, BinaryContent defaults to text/plain

0 commit comments

Comments
 (0)