Skip to content

Commit aa1dc53

Browse files
Python: Fix OpenAI chat client compatibility with third-party endpoints and OTel 0.4.14 (microsoft#4161)
* Fix system message content sent as list instead of string Some OpenAI-compatible endpoints (e.g. NVIDIA NIM) reject system messages when content is a list of content parts. This change flattens system and developer message content to a plain string in the Chat Completions client. Fixes microsoft#1407 Co-authored-by: Copilot <[email protected]> * Fix compatibility with opentelemetry-semantic-conventions-ai 0.4.14 Version 0.4.14 removed several LLM_* attributes from SpanAttributes (LLM_SYSTEM, LLM_REQUEST_MODEL, LLM_RESPONSE_MODEL, LLM_REQUEST_MAX_TOKENS, LLM_REQUEST_TEMPERATURE, LLM_REQUEST_TOP_P, LLM_TOKEN_TYPE). Move these to the OtelAttr enum with their well-known gen_ai.* string values and update all references in observability.py and tests. Fixes microsoft#4160 Co-authored-by: Copilot <[email protected]> * Flatten text-only message content to string for all roles Extend the system/developer fix to all message roles. Text-only content lists are now post-processed into plain strings, while multimodal content (text + images/audio) remains as a list. This fixes compatibility with OpenAI-like endpoints that cannot deserialize list content (e.g. Foundry Local's Neutron backend). Partially fixes microsoft#4084 Co-authored-by: Copilot <[email protected]> * Fix streaming text lost when usage data in same chunk Some providers (e.g. Gemini) include both usage data and text content in the same streaming chunk. The early return on chunk.usage caused text and tool call parsing to be skipped entirely. Remove the early return and process usage alongside text/tool calls. Fixes microsoft#3434 Co-authored-by: Copilot <[email protected]> * Fix mypy errors in _chat_client.py Rename shadowed variable 'args' in system/developer branch to 'sys_args' and rename loop variable 'content' to 'msg_content' to avoid type conflict. Co-authored-by: Copilot <[email protected]> --------- Co-authored-by: Copilot <[email protected]>
1 parent 2b1c5ab commit aa1dc53

4 files changed

Lines changed: 214 additions & 45 deletions

File tree

‎python/packages/core/agent_framework/observability.py‎

Lines changed: 20 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,14 @@ class OtelAttr(str, Enum):
210210
INPUT_MESSAGES = "gen_ai.input.messages"
211211
OUTPUT_MESSAGES = "gen_ai.output.messages"
212212
SYSTEM_INSTRUCTIONS = "gen_ai.system_instructions"
213+
# Attributes previously from opentelemetry-semantic-conventions-ai SpanAttributes,
214+
# removed in v0.4.14. Defined here for forward compatibility.
215+
SYSTEM = "gen_ai.system"
216+
REQUEST_MAX_TOKENS = "gen_ai.request.max_tokens"
217+
REQUEST_TEMPERATURE = "gen_ai.request.temperature"
218+
REQUEST_TOP_P = "gen_ai.request.top_p"
219+
REQUEST_MODEL = "gen_ai.request.model"
220+
RESPONSE_MODEL = "gen_ai.response.model"
213221

214222
# Workflow attributes
215223
WORKFLOW_ID = "workflow.id"
@@ -1175,7 +1183,7 @@ def get_response(
11751183
# in a different async context than creation — using use_span() would
11761184
# cause "Failed to detach context" errors from OpenTelemetry.
11771185
operation = attributes.get(OtelAttr.OPERATION, "operation")
1178-
span_name = attributes.get(SpanAttributes.LLM_REQUEST_MODEL, "unknown")
1186+
span_name = attributes.get(OtelAttr.REQUEST_MODEL, "unknown")
11791187
span = get_tracer().start_span(f"{operation} {span_name}")
11801188
span.set_attributes(attributes)
11811189
if OBSERVABILITY_SETTINGS.SENSITIVE_DATA_ENABLED and messages:
@@ -1237,7 +1245,7 @@ async def _finalize_stream() -> None:
12371245
return wrapped_stream
12381246

12391247
async def _get_response() -> ChatResponse:
1240-
with _get_span(attributes=attributes, span_name_attribute=SpanAttributes.LLM_REQUEST_MODEL) as span:
1248+
with _get_span(attributes=attributes, span_name_attribute=OtelAttr.REQUEST_MODEL) as span:
12411249
if OBSERVABILITY_SETTINGS.SENSITIVE_DATA_ENABLED and messages:
12421250
_capture_messages(
12431251
span=span,
@@ -1611,16 +1619,16 @@ def _get_instructions_from_options(options: Any) -> str | None:
16111619
OTEL_ATTR_MAP: dict[str | tuple[str, ...], tuple[str, Callable[[Any], Any] | None, bool, Any]] = {
16121620
"choice_count": (OtelAttr.CHOICE_COUNT, None, False, 1),
16131621
"operation_name": (OtelAttr.OPERATION, None, False, None),
1614-
"system_name": (SpanAttributes.LLM_SYSTEM, None, False, None),
1622+
"system_name": (OtelAttr.SYSTEM, None, False, None),
16151623
"provider_name": (OtelAttr.PROVIDER_NAME, None, False, None),
16161624
"service_url": (OtelAttr.ADDRESS, None, False, None),
16171625
"conversation_id": (OtelAttr.CONVERSATION_ID, None, True, None),
16181626
"seed": (OtelAttr.SEED, None, True, None),
16191627
"frequency_penalty": (OtelAttr.FREQUENCY_PENALTY, None, True, None),
1620-
"max_tokens": (SpanAttributes.LLM_REQUEST_MAX_TOKENS, None, True, None),
1628+
"max_tokens": (OtelAttr.REQUEST_MAX_TOKENS, None, True, None),
16211629
"stop": (OtelAttr.STOP_SEQUENCES, None, True, None),
1622-
"temperature": (SpanAttributes.LLM_REQUEST_TEMPERATURE, None, True, None),
1623-
"top_p": (SpanAttributes.LLM_REQUEST_TOP_P, None, True, None),
1630+
"temperature": (OtelAttr.REQUEST_TEMPERATURE, None, True, None),
1631+
"top_p": (OtelAttr.REQUEST_TOP_P, None, True, None),
16241632
"presence_penalty": (OtelAttr.PRESENCE_PENALTY, None, True, None),
16251633
"top_k": (OtelAttr.TOP_K, None, True, None),
16261634
"encoding_formats": (
@@ -1633,7 +1641,7 @@ def _get_instructions_from_options(options: Any) -> str | None:
16331641
"agent_name": (OtelAttr.AGENT_NAME, None, False, None),
16341642
"agent_description": (OtelAttr.AGENT_DESCRIPTION, None, False, None),
16351643
# Multiple source keys - checks model_id in options, then model in kwargs, then model_id in kwargs
1636-
("model_id", "model"): (SpanAttributes.LLM_REQUEST_MODEL, None, True, None),
1644+
("model_id", "model"): (OtelAttr.REQUEST_MODEL, None, True, None),
16371645
# Tools with validation - returns None if no valid tools
16381646
"tools": (
16391647
OtelAttr.TOOL_DEFINITIONS,
@@ -1790,7 +1798,7 @@ def _get_response_attributes(
17901798
if finish_reason:
17911799
attributes[OtelAttr.FINISH_REASONS] = json.dumps([finish_reason])
17921800
if model_id := getattr(response, "model_id", None):
1793-
attributes[SpanAttributes.LLM_RESPONSE_MODEL] = model_id
1801+
attributes[OtelAttr.RESPONSE_MODEL] = model_id
17941802
if capture_usage and (usage := response.usage_details):
17951803
if usage.get("input_token_count"):
17961804
attributes[OtelAttr.INPUT_TOKENS] = usage["input_token_count"]
@@ -1802,8 +1810,8 @@ def _get_response_attributes(
18021810
GEN_AI_METRIC_ATTRIBUTES = (
18031811
OtelAttr.OPERATION,
18041812
OtelAttr.PROVIDER_NAME,
1805-
SpanAttributes.LLM_REQUEST_MODEL,
1806-
SpanAttributes.LLM_RESPONSE_MODEL,
1813+
OtelAttr.REQUEST_MODEL,
1814+
OtelAttr.RESPONSE_MODEL,
18071815
OtelAttr.ADDRESS,
18081816
OtelAttr.PORT,
18091817
)
@@ -1821,10 +1829,10 @@ def _capture_response(
18211829
attrs: dict[str, Any] = {k: v for k, v in attributes.items() if k in GEN_AI_METRIC_ATTRIBUTES}
18221830
if token_usage_histogram and (input_tokens := attributes.get(OtelAttr.INPUT_TOKENS)):
18231831
token_usage_histogram.record(
1824-
input_tokens, attributes={**attrs, SpanAttributes.LLM_TOKEN_TYPE: OtelAttr.T_TYPE_INPUT}
1832+
input_tokens, attributes={**attrs, OtelAttr.T_TYPE: OtelAttr.T_TYPE_INPUT}
18251833
)
18261834
if token_usage_histogram and (output_tokens := attributes.get(OtelAttr.OUTPUT_TOKENS)):
1827-
token_usage_histogram.record(output_tokens, {**attrs, SpanAttributes.LLM_TOKEN_TYPE: OtelAttr.T_TYPE_OUTPUT})
1835+
token_usage_histogram.record(output_tokens, {**attrs, OtelAttr.T_TYPE: OtelAttr.T_TYPE_OUTPUT})
18281836
if operation_duration_histogram and duration is not None:
18291837
if OtelAttr.ERROR_TYPE in attributes:
18301838
attrs[OtelAttr.ERROR_TYPE] = attributes[OtelAttr.ERROR_TYPE]

‎python/packages/core/agent_framework/openai/_chat_client.py‎

Lines changed: 41 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,14 @@
55
import json
66
import logging
77
import sys
8-
from collections.abc import AsyncIterable, Awaitable, Callable, Mapping, MutableMapping, Sequence
8+
from collections.abc import (
9+
AsyncIterable,
10+
Awaitable,
11+
Callable,
12+
Mapping,
13+
MutableMapping,
14+
Sequence,
15+
)
916
from datetime import datetime, timezone
1017
from itertools import chain
1118
from typing import Any, Generic, Literal
@@ -16,7 +23,9 @@
1623
from openai.types.chat.chat_completion import ChatCompletion, Choice
1724
from openai.types.chat.chat_completion_chunk import ChatCompletionChunk
1825
from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice
19-
from openai.types.chat.chat_completion_message_custom_tool_call import ChatCompletionMessageCustomToolCall
26+
from openai.types.chat.chat_completion_message_custom_tool_call import (
27+
ChatCompletionMessageCustomToolCall,
28+
)
2029
from openai.types.chat.completion_create_params import WebSearchOptions
2130
from pydantic import BaseModel
2231

@@ -395,21 +404,18 @@ def _parse_response_update_from_openai(
395404
) -> ChatResponseUpdate:
396405
"""Parse a streaming response update from OpenAI."""
397406
chunk_metadata = self._get_metadata_from_streaming_chat_response(chunk)
398-
if chunk.usage:
399-
return ChatResponseUpdate(
400-
role="assistant",
401-
contents=[
402-
Content.from_usage(
403-
usage_details=self._parse_usage_from_openai(chunk.usage), raw_representation=chunk
404-
)
405-
],
406-
model_id=chunk.model,
407-
additional_properties=chunk_metadata,
408-
response_id=chunk.id,
409-
message_id=chunk.id,
410-
)
411407
contents: list[Content] = []
412408
finish_reason: FinishReason | None = None
409+
410+
# Process usage data (may coexist with text/tool content in providers like Gemini).
411+
# See https://github.com/microsoft/agent-framework/issues/3434
412+
if chunk.usage:
413+
contents.append(
414+
Content.from_usage(
415+
usage_details=self._parse_usage_from_openai(chunk.usage), raw_representation=chunk
416+
)
417+
)
418+
413419
for choice in chunk.choices:
414420
chunk_metadata.update(self._get_metadata_from_chat_choice(choice))
415421
contents.extend(self._parse_tool_calls_from_openai(choice))
@@ -532,6 +538,17 @@ def _prepare_messages_for_openai(
532538

533539
def _prepare_message_for_openai(self, message: Message) -> list[dict[str, Any]]:
534540
"""Prepare a chat message for OpenAI."""
541+
# System/developer messages must use plain string content because some
542+
# OpenAI-compatible endpoints reject list content for non-user roles.
543+
if message.role in ("system", "developer"):
544+
texts = [content.text for content in message.contents if content.type == "text" and content.text]
545+
if texts:
546+
sys_args: dict[str, Any] = {"role": message.role, "content": "\n".join(texts)}
547+
if message.author_name:
548+
sys_args["name"] = message.author_name
549+
return [sys_args]
550+
return []
551+
535552
all_messages: list[dict[str, Any]] = []
536553
for content in message.contents:
537554
# Skip approval content - it's internal framework state, not for the LLM
@@ -568,6 +585,15 @@ def _prepare_message_for_openai(self, message: Message) -> list[dict[str, Any]]:
568585
args["content"].append(self._prepare_content_for_openai(content)) # type: ignore
569586
if "content" in args or "tool_calls" in args:
570587
all_messages.append(args)
588+
589+
# Flatten text-only content lists to plain strings for broader
590+
# compatibility with OpenAI-like endpoints (e.g. Foundry Local).
591+
# See https://github.com/microsoft/agent-framework/issues/4084
592+
for msg in all_messages:
593+
msg_content: Any = msg.get("content")
594+
if isinstance(msg_content, list) and all(isinstance(c, dict) and c.get("type") == "text" for c in msg_content):
595+
msg["content"] = "\n".join(c.get("text", "") for c in msg_content)
596+
571597
return all_messages
572598

573599
def _prepare_content_for_openai(self, content: Content) -> dict[str, Any]:

‎python/packages/core/tests/core/test_observability.py‎

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77

88
import pytest
99
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
10-
from opentelemetry.semconv_ai import SpanAttributes
1110
from opentelemetry.trace import StatusCode
1211

1312
from agent_framework import (
@@ -48,8 +47,8 @@ def test_role_event_map():
4847
def test_enum_values():
4948
"""Test that OtelAttr enum has expected values."""
5049
assert OtelAttr.OPERATION == "gen_ai.operation.name"
51-
assert SpanAttributes.LLM_SYSTEM == "gen_ai.system"
52-
assert SpanAttributes.LLM_REQUEST_MODEL == "gen_ai.request.model"
50+
assert OtelAttr.SYSTEM == "gen_ai.system"
51+
assert OtelAttr.REQUEST_MODEL == "gen_ai.request.model"
5352
assert OtelAttr.CHAT_COMPLETION_OPERATION == "chat"
5453
assert OtelAttr.TOOL_EXECUTION_OPERATION == "execute_tool"
5554
assert OtelAttr.AGENT_INVOKE_OPERATION == "invoke_agent"
@@ -213,7 +212,7 @@ async def test_chat_client_observability(mock_chat_client, span_exporter: InMemo
213212
span = spans[0]
214213
assert span.name == "chat Test"
215214
assert span.attributes[OtelAttr.OPERATION.value] == OtelAttr.CHAT_COMPLETION_OPERATION
216-
assert span.attributes[SpanAttributes.LLM_REQUEST_MODEL] == "Test"
215+
assert span.attributes[OtelAttr.REQUEST_MODEL] == "Test"
217216
assert span.attributes[OtelAttr.INPUT_TOKENS] == 10
218217
assert span.attributes[OtelAttr.OUTPUT_TOKENS] == 20
219218
if enable_sensitive_data:
@@ -243,7 +242,7 @@ async def test_chat_client_streaming_observability(
243242
span = spans[0]
244243
assert span.name == "chat Test"
245244
assert span.attributes[OtelAttr.OPERATION.value] == OtelAttr.CHAT_COMPLETION_OPERATION
246-
assert span.attributes[SpanAttributes.LLM_REQUEST_MODEL] == "Test"
245+
assert span.attributes[OtelAttr.REQUEST_MODEL] == "Test"
247246
if enable_sensitive_data:
248247
assert span.attributes[OtelAttr.INPUT_MESSAGES] is not None
249248
assert span.attributes[OtelAttr.OUTPUT_MESSAGES] is not None
@@ -392,7 +391,7 @@ async def test_chat_client_without_model_id_observability(mock_chat_client, span
392391

393392
assert span.name == "chat unknown"
394393
assert span.attributes[OtelAttr.OPERATION.value] == OtelAttr.CHAT_COMPLETION_OPERATION
395-
assert span.attributes[SpanAttributes.LLM_REQUEST_MODEL] == "unknown"
394+
assert span.attributes[OtelAttr.REQUEST_MODEL] == "unknown"
396395

397396

398397
async def test_chat_client_streaming_without_model_id_observability(
@@ -416,7 +415,7 @@ async def test_chat_client_streaming_without_model_id_observability(
416415
span = spans[0]
417416
assert span.name == "chat unknown"
418417
assert span.attributes[OtelAttr.OPERATION.value] == OtelAttr.CHAT_COMPLETION_OPERATION
419-
assert span.attributes[SpanAttributes.LLM_REQUEST_MODEL] == "unknown"
418+
assert span.attributes[OtelAttr.REQUEST_MODEL] == "unknown"
420419

421420

422421
def test_prepend_user_agent_with_none_value():
@@ -491,7 +490,7 @@ async def test_agent_instrumentation_enabled(
491490
assert span.attributes[OtelAttr.AGENT_ID] == "test_agent_id"
492491
assert span.attributes[OtelAttr.AGENT_NAME] == "test_agent"
493492
assert span.attributes[OtelAttr.AGENT_DESCRIPTION] == "Test agent description"
494-
assert span.attributes[SpanAttributes.LLM_REQUEST_MODEL] == "TestModel"
493+
assert span.attributes[OtelAttr.REQUEST_MODEL] == "TestModel"
495494
assert span.attributes[OtelAttr.INPUT_TOKENS] == 15
496495
assert span.attributes[OtelAttr.OUTPUT_TOKENS] == 25
497496
if enable_sensitive_data:
@@ -521,7 +520,7 @@ async def test_agent_streaming_response_with_diagnostics_enabled(
521520
assert span.attributes[OtelAttr.AGENT_ID] == "test_agent_id"
522521
assert span.attributes[OtelAttr.AGENT_NAME] == "test_agent"
523522
assert span.attributes[OtelAttr.AGENT_DESCRIPTION] == "Test agent description"
524-
assert span.attributes[SpanAttributes.LLM_REQUEST_MODEL] == "TestModel"
523+
assert span.attributes[OtelAttr.REQUEST_MODEL] == "TestModel"
525524
if enable_sensitive_data:
526525
assert span.attributes.get(OtelAttr.OUTPUT_MESSAGES) is not None # Streaming, so no usage yet
527526

@@ -1381,8 +1380,6 @@ def test_get_response_attributes_with_model_id():
13811380
"""Test _get_response_attributes includes model_id."""
13821381
from unittest.mock import Mock
13831382

1384-
from opentelemetry.semconv_ai import SpanAttributes
1385-
13861383
from agent_framework.observability import _get_response_attributes
13871384

13881385
response = Mock()
@@ -1395,7 +1392,7 @@ def test_get_response_attributes_with_model_id():
13951392
attrs = {}
13961393
result = _get_response_attributes(attrs, response)
13971394

1398-
assert result[SpanAttributes.LLM_RESPONSE_MODEL] == "gpt-4"
1395+
assert result[OtelAttr.RESPONSE_MODEL] == "gpt-4"
13991396

14001397

14011398
def test_get_response_attributes_with_usage():

0 commit comments

Comments
 (0)