Skip to content

Commit 54b558b

Browse files
authored
Python: Decode Ollama inline image data before serialization (#9459)
* Python: decode Ollama inline images before serialization * Test Ollama URI validation at connector boundary
1 parent f7a5fcb commit 54b558b

2 files changed

Lines changed: 112 additions & 4 deletions

File tree

‎python/packages/ollama/agent_framework_ollama/_chat_client.py‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22

33
from __future__ import annotations
44

5+
import base64
6+
import binascii
57
import json
68
import logging
79
import sys
@@ -45,6 +47,7 @@
4547

4648
# Rename imported types to avoid naming conflicts with Agent Framework types
4749
from ollama._types import ChatResponse as OllamaChatResponse
50+
from ollama._types import Image as OllamaImage
4851
from ollama._types import Message as OllamaMessage
4952
from pydantic import BaseModel
5053

@@ -533,10 +536,32 @@ def _format_user_message(self, message: Message) -> list[OllamaMessage]:
533536
raise ChatClientInvalidRequestException(
534537
"Only image data content is supported for user messages in Ollama."
535538
)
536-
# Ollama expects base64 strings without prefix
537-
user_message["images"] = [c.uri.split(",")[1] for c in data_contents if c.uri]
539+
user_message["images"] = [OllamaImage(value=self._decode_image_data_uri(c)) for c in data_contents]
538540
return [user_message]
539541

542+
@staticmethod
543+
def _decode_image_data_uri(content: Content) -> bytes:
544+
uri = content.uri
545+
if not uri or not uri.startswith("data:") or "," not in uri:
546+
raise ChatClientInvalidRequestException("Ollama image content must be a base64-encoded data:image URI.")
547+
548+
metadata, payload = uri.split(",", 1)
549+
metadata_parts = metadata[5:].split(";")
550+
parameters = metadata_parts[1:]
551+
if (
552+
not metadata_parts[0].lower().startswith("image/")
553+
or not parameters
554+
or parameters[-1] != "base64"
555+
or any(not parameter or "=" not in parameter for parameter in parameters[:-1])
556+
or not payload
557+
):
558+
raise ChatClientInvalidRequestException("Ollama image content must be a base64-encoded data:image URI.")
559+
560+
try:
561+
return base64.b64decode(payload, validate=True)
562+
except (binascii.Error, ValueError) as exc:
563+
raise ChatClientInvalidRequestException("Ollama image content contains invalid base64 data.") from exc
564+
540565
def _format_assistant_message(self, message: Message) -> list[OllamaMessage]:
541566
text_content = message.text
542567
# Ollama shouldn't have encrypted reasoning, so we just process text.

‎python/packages/ollama/tests/test_ollama_chat_client.py‎

Lines changed: 85 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
# Copyright (c) Microsoft. All rights reserved.
22

3+
import base64
34
import os
45
import uuid
56
from collections.abc import AsyncIterable
7+
from pathlib import Path
68
from typing import Any, cast
79
from unittest.mock import AsyncMock, MagicMock, patch
810

@@ -25,6 +27,7 @@
2527
)
2628
from ollama import AsyncClient
2729
from ollama._types import ChatResponse as OllamaChatResponse
30+
from ollama._types import Image as OllamaImage
2831
from ollama._types import Message as OllamaMessage
2932
from openai import AsyncStream
3033
from pydantic import BaseModel
@@ -634,18 +637,98 @@ async def test_cmc_with_data_content_type(
634637
chat_history: list[Message],
635638
mock_chat_completion_response: OllamaChatResponse,
636639
) -> None:
640+
image_bytes = b"\x89PNG\r\n\x1a\n"
637641
mock_chat.return_value = mock_chat_completion_response
638642
chat_history.append(
639643
Message(
640-
contents=[Content.from_uri(uri="data:image/png;base64,xyz", media_type="image/png")],
644+
contents=[
645+
Content.from_uri(
646+
uri=f"data:image/png;base64,{base64.b64encode(image_bytes).decode()}",
647+
media_type="image/png",
648+
)
649+
],
641650
role="user",
642651
)
643652
)
644653

645654
ollama_client = OllamaChatClient()
646655

647-
result = await ollama_client.get_response(messages=chat_history)
656+
with (
657+
patch.object(Path, "exists", side_effect=AssertionError("Path.exists must not be called")) as path_exists,
658+
patch.object(
659+
Path, "read_bytes", side_effect=AssertionError("Path.read_bytes must not be called")
660+
) as read_bytes,
661+
):
662+
result = await ollama_client.get_response(messages=chat_history)
663+
sent_image = mock_chat.call_args.kwargs["messages"][0].images[0]
664+
assert isinstance(sent_image, OllamaImage)
665+
assert sent_image.value == image_bytes
666+
assert sent_image.model_dump() == base64.b64encode(image_bytes).decode()
667+
648668
assert result.text == "test"
669+
path_exists.assert_not_called()
670+
read_bytes.assert_not_called()
671+
672+
673+
@pytest.mark.parametrize(
674+
"uri",
675+
[
676+
"data:image/png,/tmp/image.png",
677+
"data:image/png;base64,/tmp/image.png",
678+
r"data:image/png;base64,C:\temp\image.png",
679+
r"data:image/png;base64,\\server\share\image.png",
680+
"data:image/png;base64,../image.png",
681+
"data:image/png;base64,",
682+
"data:text/plain;base64,aGVsbG8=",
683+
],
684+
)
685+
def test_cmc_rejects_invalid_image_data_uri(
686+
uri: str,
687+
ollama_unit_test_env: dict[str, str],
688+
) -> None:
689+
message = Message(
690+
contents=[Content.from_uri(uri=uri, media_type="image/png")],
691+
role="user",
692+
)
693+
ollama_client = OllamaChatClient()
694+
695+
with pytest.raises(ChatClientInvalidRequestException):
696+
ollama_client._format_user_message(message)
697+
698+
699+
@patch.object(AsyncClient, "chat", new_callable=AsyncMock)
700+
async def test_cmc_treats_base64_shaped_path_as_inline_bytes(
701+
mock_chat: AsyncMock,
702+
ollama_unit_test_env: dict[str, str],
703+
chat_history: list[Message],
704+
mock_chat_completion_response: OllamaChatResponse,
705+
) -> None:
706+
payload = "////tmpx"
707+
expected_bytes = base64.b64decode(payload, validate=True)
708+
mock_chat.return_value = mock_chat_completion_response
709+
chat_history.append(
710+
Message(
711+
contents=[Content.from_uri(uri=f"data:image/png;base64,{payload}", media_type="image/png")],
712+
role="user",
713+
)
714+
)
715+
716+
ollama_client = OllamaChatClient()
717+
718+
with (
719+
patch.object(Path, "exists", side_effect=AssertionError("Path.exists must not be called")) as path_exists,
720+
patch.object(
721+
Path, "read_bytes", side_effect=AssertionError("Path.read_bytes must not be called")
722+
) as read_bytes,
723+
):
724+
await ollama_client.get_response(messages=chat_history)
725+
sent_image = mock_chat.call_args.kwargs["messages"][0].images[0]
726+
assert isinstance(sent_image, OllamaImage)
727+
assert sent_image.value == expected_bytes
728+
assert sent_image.model_dump() == payload
729+
730+
path_exists.assert_not_called()
731+
read_bytes.assert_not_called()
649732

650733

651734
@patch.object(AsyncClient, "chat", new_callable=AsyncMock)

0 commit comments

Comments
 (0)