Skip to content

Commit 669c8b9

Browse files
authored
Python: fix(orchestrations): preserve user multimodal content during handoff (microsoft#8551)
clean_conversation_for_handoff dropped every non-text content type to avoid replaying runtime-only tool-control payloads, but that also discarded genuine user input such as images, files, and vector store references, so agents receiving a handoff lost the user's multimodal context. Keep semantic multimodal content (data, uri, hosted_file, hosted_vector_store) on user messages while keeping assistant and other non-user messages text-only, since providers treat multimodal parts as input-only and reject them when replayed on assistant turns. Tool-control payloads (function_call, function_result, approval payloads) remain excluded as before. Fixes microsoft#7822
1 parent 2565daf commit 669c8b9

3 files changed

Lines changed: 163 additions & 16 deletions

File tree

‎python/packages/orchestrations/agent_framework_orchestrations/_orchestrator_helpers.py‎

Lines changed: 32 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -13,37 +13,53 @@
1313
logger = logging.getLogger(__name__)
1414

1515

16+
# Semantic content types that carry real user input and must survive handoff routing.
17+
# Everything else (function calls, function results, approval payloads, tool outputs)
18+
# is runtime-only tool-control state that providers reject when replayed.
19+
_USER_SEMANTIC_CONTENT_TYPES: frozenset[str] = frozenset({"text", "data", "uri", "hosted_file", "hosted_vector_store"})
20+
21+
1622
def clean_conversation_for_handoff(conversation: list[Message]) -> list[Message]:
17-
"""Keep only plain text chat history for handoff routing.
23+
"""Clean the conversation history for handoff routing.
1824
1925
Handoff executors must not replay prior tool-control artifacts (function calls,
2026
tool outputs, approval payloads) into future model turns, or providers may reject
21-
the next request due to unmatched tool-call state.
22-
23-
This helper builds a text-only copy of the conversation:
24-
- Drops all non-text content from every message.
25-
- Drops messages with no remaining text content.
26-
- Preserves original roles and author names for retained text messages.
27+
the next request due to unmatched tool-call state. At the same time, genuine user
28+
input such as images or files must be preserved so the receiving agent keeps the
29+
full context of the request.
30+
31+
This helper builds a cleaned copy of the conversation:
32+
- Keeps text content on every message.
33+
- Additionally keeps semantic multimodal content (data, uri, hosted_file,
34+
hosted_vector_store) on user messages.
35+
- Keeps assistant and other non-user messages text-only, because providers treat
36+
multimodal parts as input-only and reject them when replayed on assistant turns.
37+
- Drops tool-control payloads (function_call, function_result, approval payloads, etc.).
38+
- Drops messages with no remaining content.
39+
- Preserves original roles and author names for retained messages.
2740
2841
Args:
2942
conversation: Full conversation history, including tool-control content
3043
Returns:
31-
Cleaned conversation history with only text content, suitable for handoff routing
44+
Cleaned conversation history suitable for handoff routing, with semantic
45+
multimodal content preserved on user messages.
3246
"""
3347
cleaned: list[Message] = []
3448
for msg in conversation:
35-
# Keep only plain text history for handoff routing. Tool-control content
36-
# (function_call/function_result/approval payloads) is runtime-only and
37-
# must not be replayed in future model turns.
38-
text_parts = [content.text for content in msg.contents if content.type == "text" and content.text]
39-
# TODO(@taochen): This is a simplified check that considers any non-text content as a tool call.
40-
# We need to enhance this logic to specifically identify tool related contents.
41-
if not text_parts:
49+
# Tool-control content (function_call/function_result/approval payloads) is
50+
# runtime-only and must not be replayed in future model turns.
51+
allowed_types = _USER_SEMANTIC_CONTENT_TYPES if str(msg.role).lower() == "user" else frozenset({"text"})
52+
retained = [
53+
content
54+
for content in msg.contents
55+
if content.type in allowed_types and (content.type != "text" or content.text)
56+
]
57+
if not retained:
4258
continue
4359

4460
msg_copy = Message(
4561
role=msg.role,
46-
contents=[" ".join(text_parts)],
62+
contents=retained,
4763
author_name=msg.author_name,
4864
additional_properties=dict(msg.additional_properties) if msg.additional_properties else None,
4965
)

‎python/packages/orchestrations/tests/test_handoff.py‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1142,6 +1142,63 @@ def test_clean_conversation_for_handoff_keeps_text_only_history() -> None:
11421142
]
11431143

11441144

1145+
def test_clean_conversation_for_handoff_preserves_user_multimodal_content() -> None:
1146+
"""Semantic multimodal content on user messages must survive handoff routing (#7822).
1147+
1148+
Tool-control payloads are runtime-only and must still be dropped, and assistant
1149+
messages must stay text-only because providers treat multimodal items as
1150+
input-only and reject them when replayed on assistant turns.
1151+
"""
1152+
user_image = Content.from_uri(uri="https://example.com/damage.png", media_type="image/png")
1153+
user_inline_data = Content.from_data(data=b"\x89PNG-fake", media_type="image/png")
1154+
user_upload = Content.from_hosted_file(file_id="file-abc123")
1155+
user_store = Content.from_hosted_vector_store(vector_store_id="vs-xyz789")
1156+
1157+
conversation = [
1158+
Message(
1159+
role="user",
1160+
contents=[
1161+
"My order arrived damaged, see the attached photos.",
1162+
user_image,
1163+
user_inline_data,
1164+
user_upload,
1165+
user_store,
1166+
],
1167+
),
1168+
Message(
1169+
role="assistant",
1170+
contents=[
1171+
Content.from_text(text="Triage Agent: Routing you to Refund."),
1172+
# Providers reject input-only multimodal parts replayed on assistant turns.
1173+
Content.from_uri(uri="https://example.com/generated.png", media_type="image/png"),
1174+
Content.from_function_call(call_id="handoff-call-1", name="handoff_to_refund_agent"),
1175+
],
1176+
),
1177+
Message(role="tool", contents=[Content.from_function_result(call_id="handoff-call-1", result="ok")]),
1178+
]
1179+
1180+
cleaned = clean_conversation_for_handoff(conversation)
1181+
assert [message.role for message in cleaned] == ["user", "assistant"]
1182+
1183+
cleaned_user = cleaned[0]
1184+
assert [content.type for content in cleaned_user.contents] == [
1185+
"text",
1186+
"uri",
1187+
"data",
1188+
"hosted_file",
1189+
"hosted_vector_store",
1190+
]
1191+
assert cleaned_user.contents[1].uri == "https://example.com/damage.png"
1192+
assert cleaned_user.contents[2].uri is not None
1193+
assert cleaned_user.contents[2].uri.startswith("data:image/png;base64,")
1194+
assert cleaned_user.contents[3].file_id == "file-abc123"
1195+
assert cleaned_user.contents[4].vector_store_id == "vs-xyz789"
1196+
1197+
cleaned_assistant = cleaned[1]
1198+
assert [content.type for content in cleaned_assistant.contents] == ["text"]
1199+
assert cleaned_assistant.text == "Triage Agent: Routing you to Refund."
1200+
1201+
11451202
async def test_autonomous_mode_yields_output_without_user_request():
11461203
"""Ensure autonomous interaction mode yields output without requesting user input."""
11471204
triage = MockHandoffAgent(name="triage", handoff_to="specialist")

‎python/packages/orchestrations/tests/test_sequential.py‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -254,6 +254,80 @@ async def test_sequential_checkpoint_resume_round_trip() -> None:
254254
assert baseline_text == resumed_text
255255

256256

257+
async def test_sequential_handoff_forwards_user_multimodal_content() -> None:
258+
"""Multimodal user input must reach downstream agents after history cleanup (#7822)."""
259+
260+
class _MultimodalInspectorAgent(BaseAgent):
261+
"""Captures the content types of the messages it receives."""
262+
263+
@overload
264+
def run(
265+
self,
266+
messages: AgentRunInputs | None = ...,
267+
*,
268+
stream: Literal[False] = ...,
269+
**kwargs: Any,
270+
) -> Awaitable[AgentResponse[Any]]: ...
271+
@overload
272+
def run(
273+
self,
274+
messages: AgentRunInputs | None = ...,
275+
*,
276+
stream: Literal[True],
277+
**kwargs: Any,
278+
) -> ResponseStream[AgentResponseUpdate, AgentResponse[Any]]: ...
279+
280+
def run(
281+
self,
282+
messages: AgentRunInputs | None = None,
283+
*,
284+
stream: bool = False,
285+
**kwargs: Any,
286+
) -> Awaitable[AgentResponse[Any]] | ResponseStream[AgentResponseUpdate, AgentResponse[Any]]:
287+
# AgentRunInputs also permits a single str/Content/Message, so narrow to the
288+
# sequence of Message objects that SequentialBuilder supplies before iterating.
289+
assert isinstance(messages, Sequence) and not isinstance(messages, str)
290+
received = [m for m in messages if isinstance(m, Message)]
291+
seen_types = sorted({c.type for m in received for c in m.contents})
292+
seen_uris = [c.uri for m in received for c in m.contents if c.type == "uri"]
293+
summary = f"types={seen_types} uris={seen_uris}"
294+
295+
if stream:
296+
297+
async def _stream() -> AsyncIterable[AgentResponseUpdate]:
298+
yield AgentResponseUpdate(contents=[Content.from_text(text=summary)])
299+
300+
return ResponseStream(_stream(), finalizer=AgentResponse.from_updates)
301+
302+
async def _run() -> AgentResponse:
303+
return AgentResponse(messages=[Message("assistant", [summary])])
304+
305+
return _run()
306+
307+
echo = _EchoAgent(id="agent1", name="A1")
308+
inspector = _MultimodalInspectorAgent(id="inspector", name="Inspector")
309+
wf = SequentialBuilder(participants=[echo, inspector]).build()
310+
311+
request = Message(
312+
"user",
313+
[
314+
Content.from_text(text="Please review this screenshot."),
315+
Content.from_uri(uri="https://example.com/screenshot.png", media_type="image/png"),
316+
],
317+
)
318+
output_events = [ev async for ev in wf.run(request, stream=True) if ev.type == "output"]
319+
320+
assert len(output_events) == 1
321+
output_data = output_events[0].data
322+
final_text = (
323+
" ".join(m.text for m in output_data.messages)
324+
if isinstance(output_data, AgentResponse)
325+
else (output_data.text or "")
326+
)
327+
assert "'uri'" in final_text
328+
assert "https://example.com/screenshot.png" in final_text
329+
330+
257331
async def test_sequential_checkpoint_runtime_only() -> None:
258332
"""Test checkpointing configured ONLY at runtime, not at build time."""
259333
storage = InMemoryCheckpointStorage()

0 commit comments

Comments
 (0)