Skip to content

Support ImageContent and AudioContent in OpenAISamplingHandler #3544

@zhengmz

Description

@zhengmz

Enhancement

Support ImageContent and AudioContent in OpenAISamplingHandler

server

PROMPT_ANALYZE_DEFAULT = "Please identify obstacles in the image and estimate the distance to the obstacles."
@mcp.tool(annotations={"readOnlyHint": True})
async def get_or_analyze_image(
    format: str = 'jpeg', jpeg_quality: int = 85, scale: float = 1.0,
    analyze_prompt: str = PROMPT_ANALYZE_DEFAULT,
    ctx: Context = CurrentContext()
) -> str:
    """
    Get or analysis the latest color image from the drone's camera, return base64-encoded image data or analysis report
    Args:
        format: Image format, default is 'jpeg', supports 'jpeg', 'jpg', 'png'
        jpeg_quality: JPEG quality, default is 85, valid range 0-100, and for format is 'jpeg' or 'jpg'
        scale: Image scaling ratio, default is 1.0, valid range 0.1 to 4.0
        analyze_prompt: If set '' or None, just return base64 encoded image uri, default is analyzing image using MCP sampling mechanism
    """
    image_data = tools_executor.execute_tool("get_image", {"format": format, "jpeg_quality": jpeg_quality, "scale": scale})
    if isinstance(image_data, str):
        return image_data
    
    if analyze_prompt:
        # If analyze_flag is True, perform image analysis using the MCP sampling mechanism
        image_content = ImageContent(
            type = "image",
            data = image_data["data_str"],
            mimeType = f"image/{image_data['format']}"
        )
        messages = SamplingMessage(
            role="user",
            content=[
                TextContent(type="text", text=analyze_prompt),
                image_content
            ]
        )

        result = await ctx.sample(
            messages=[messages],
            system_prompt="You are an image analysis expert."
        )

        return result.text if result else "No analysis result"

    return f"data:image/{image_data['format']};base64,{image_data['data_str']}"

Client

from fastmcp import Client
from fastmcp.client.sampling.handlers.openai import OpenAISamplingHandler

mcp_client = Client(
      transport,
      sampling_handler=OpenAISamplingHandler(
            default_model=model,
            client=AsyncOpenAI(api_key=api_key, base_url=base_url)
       )
 )

question

client not work

Metadata

Metadata

Assignees

No one assigned

    Labels

    clientRelated to the FastMCP client SDK or client-side functionality.enhancementImprovement to existing functionality. For issues and smaller PR improvements.

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions