Skip to content

Commit cb59096

Browse files
Python: fix: decode entities once in ChatHistory.from_rendered_prompt (#14624)
### Summary Fixes #14619 The XML parser in `ChatHistory.from_rendered_prompt` already decodes XML/HTML entities when parsing element text. Running an additional `html.unescape` on the parsed text caused double-decoding (e.g. `&amp;` became `&` on the first decode and then unescaped again, altering user input containing literal entity representations such as `What does &amp; mean in HTML?`). ### Changes 1. Removed the redundant second `unescape` on parsed element text in: - `ChatHistory.from_rendered_prompt` - `ChatMessageContent.from_element` - `ReasoningContent.from_element` - `TextContent.from_element` 2. Added CDATA pre-handling to decode CDATA content once before XML parsing so that CDATA sections remain properly decoded. 3. Added comprehensive test coverage in `test_chat_history.py` verifying single-decode semantics across messages, plain inputs, text after message tags, CDATA sections, and prompt round-tripping. ### Verification - `uv run pytest tests/unit/contents/` (414 passed). - `uv run ruff check semantic_kernel/contents/ tests/unit/contents/` (all checks passed). Co-authored-by: VANDRANKI <[email protected]>
1 parent 06368f4 commit cb59096

5 files changed

Lines changed: 78 additions & 9 deletions

File tree

‎python/semantic_kernel/contents/chat_history.py‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
# Copyright (c) Microsoft. All rights reserved.
22

33
import logging
4+
import re
45
from collections.abc import Generator, Iterable
56
from functools import singledispatchmethod
67
from html import unescape
78
from typing import Any, TypeVar
89
from xml.etree.ElementTree import Element, tostring # nosec
10+
from xml.sax.saxutils import escape # nosec
911

1012
from defusedxml.ElementTree import XML, ParseError
1113
from pydantic import Field, field_validator, model_validator
@@ -22,6 +24,8 @@
2224

2325
_T = TypeVar("_T", bound="ChatHistory")
2426

27+
_CDATA_SECTION = re.compile(r"<!\[CDATA\[(.*?)\]\]>", re.DOTALL)
28+
2529

2630
class ChatHistory(KernelBaseModel):
2731
"""This class holds the history of chat messages from a chat conversation.
@@ -340,13 +344,17 @@ def from_rendered_prompt(cls: type[_T], rendered_prompt: str) -> _T:
340344
prompt_tag = "root"
341345
messages: list["ChatMessageContent"] = []
342346
prompt = rendered_prompt.strip()
347+
# The XML parser decodes entities in normal text but keeps a CDATA section literal, and the
348+
# template engine escapes values wherever they land. Decode CDATA once here, so every text
349+
# is decoded exactly once by the parser and not again afterwards.
350+
xml_text = _CDATA_SECTION.sub(lambda match: escape(unescape(match.group(1))), prompt)
343351
try:
344-
xml_prompt = XML(text=f"<{prompt_tag}>{prompt}</{prompt_tag}>")
352+
xml_prompt = XML(text=f"<{prompt_tag}>{xml_text}</{prompt_tag}>")
345353
except ParseError as exc:
346354
logger.info(f"Could not parse prompt {prompt} as xml, treating as text, error was: {exc}")
347355
return cls(messages=[ChatMessageContent(role=AuthorRole.USER, content=unescape(prompt))])
348356
if xml_prompt.text and xml_prompt.text.strip():
349-
messages.append(ChatMessageContent(role=AuthorRole.SYSTEM, content=unescape(xml_prompt.text.strip())))
357+
messages.append(ChatMessageContent(role=AuthorRole.SYSTEM, content=xml_prompt.text.strip()))
350358
for item in xml_prompt:
351359
if item.tag == CHAT_MESSAGE_CONTENT_TAG:
352360
messages.append(ChatMessageContent.from_element(item))
@@ -368,10 +376,10 @@ def from_rendered_prompt(cls: type[_T], rendered_prompt: str) -> _T:
368376
# For unknown tags the tail is part of the surrounding text,
369377
# so keep it in the same message instead of starting a new one.
370378
if item.tail:
371-
messages[-1].content = (messages[-1].content or "") + unescape(item.tail)
379+
messages[-1].content = (messages[-1].content or "") + item.tail
372380
continue
373381
if item.tail and item.tail.strip():
374-
messages.append(ChatMessageContent(role=AuthorRole.USER, content=unescape(item.tail.strip())))
382+
messages.append(ChatMessageContent(role=AuthorRole.USER, content=item.tail.strip()))
375383
if len(messages) == 1 and messages[0].role == AuthorRole.SYSTEM:
376384
messages[0].role = AuthorRole.USER
377385
return cls(messages=messages)

‎python/semantic_kernel/contents/chat_message_content.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -268,7 +268,7 @@ def from_element(cls, element: Element) -> "ChatMessageContent":
268268
kwargs: dict[str, Any] = {key: value for key, value in element.items()}
269269
items: list[KernelContent] = []
270270
if element.text:
271-
items.append(TextContent(text=unescape(element.text)))
271+
items.append(TextContent(text=element.text))
272272
for child in element:
273273
if child.tag not in TAG_CONTENT_MAP:
274274
logger.warning('Unknown tag "%s" in ChatMessageContent, treating as text', child.tag)

‎python/semantic_kernel/contents/reasoning_content.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
"""Reasoning content model."""
44

5-
from html import unescape
65
from typing import ClassVar, Literal, TypeVar
76
from xml.etree.ElementTree import Element # nosec
87

@@ -52,7 +51,7 @@ def from_element(cls: type[_T], element: Element) -> _T:
5251
"""Create an instance from an XML Element."""
5352
if element.tag != cls.tag:
5453
raise ContentInitializationError(f"Element tag is not {cls.tag}") # pragma: no cover
55-
return cls(text=unescape(element.text) if element.text else "")
54+
return cls(text=element.text or "")
5655

5756
def to_dict(self) -> dict[str, str]:
5857
"""Convert the instance to a dictionary suitable for message serialization."""

‎python/semantic_kernel/contents/text_content.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
# Copyright (c) Microsoft. All rights reserved.
22

3-
from html import unescape
43
from typing import ClassVar, Literal, TypeVar
54
from xml.etree.ElementTree import Element # nosec
65

@@ -52,7 +51,7 @@ def from_element(cls: type[_T], element: Element) -> _T:
5251
if element.tag != cls.tag:
5352
raise ContentInitializationError(f"Element tag is not {cls.tag}") # pragma: no cover
5453

55-
return cls(text=unescape(element.text) if element.text else "", encoding=element.get("encoding", None))
54+
return cls(text=element.text or "", encoding=element.get("encoding", None))
5655

5756
def to_dict(self) -> dict[str, str]:
5857
"""Convert the instance to a dictionary."""

‎python/tests/unit/contents/test_chat_history.py‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -670,3 +670,66 @@ def __str__(self) -> str:
670670
])
671671
chat_history.add_tool_message([FunctionResultContent(id="test1", result=custom_result)])
672672
assert "CustomResultTestValue" in chat_history.serialize()
673+
674+
675+
ENTITY_TEXTS = [
676+
"What does &amp; mean in HTML?",
677+
"Write &lt;b&gt; as text",
678+
"Use &#x3a; as a colon",
679+
"Tom & Jerry <3",
680+
]
681+
682+
683+
@pytest.mark.parametrize("text", ENTITY_TEXTS)
684+
async def test_template_message_input_is_decoded_once(text: str):
685+
template = '<message role="user">{{$input}}</message>'
686+
rendered = await KernelPromptTemplate(
687+
prompt_template_config=PromptTemplateConfig(name="test", description="test", template=template)
688+
).render(kernel=Kernel(), arguments=KernelArguments(input=text))
689+
690+
chat_history = ChatHistory.from_rendered_prompt(rendered)
691+
assert chat_history.messages[0].role == AuthorRole.USER
692+
assert chat_history.messages[0].content == text
693+
694+
695+
@pytest.mark.parametrize("text", ENTITY_TEXTS)
696+
async def test_template_plain_input_is_decoded_once(text: str):
697+
template = "{{$input}}"
698+
rendered = await KernelPromptTemplate(
699+
prompt_template_config=PromptTemplateConfig(name="test", description="test", template=template)
700+
).render(kernel=Kernel(), arguments=KernelArguments(input=text))
701+
702+
chat_history = ChatHistory.from_rendered_prompt(rendered)
703+
assert chat_history.messages[0].content == text
704+
705+
706+
@pytest.mark.parametrize("text", ENTITY_TEXTS)
707+
async def test_template_text_after_message_is_decoded_once(text: str):
708+
template = '<message role="system">rules</message>{{$input}}'
709+
rendered = await KernelPromptTemplate(
710+
prompt_template_config=PromptTemplateConfig(name="test", description="test", template=template)
711+
).render(kernel=Kernel(), arguments=KernelArguments(input=text))
712+
713+
chat_history = ChatHistory.from_rendered_prompt(rendered)
714+
assert chat_history.messages[1].content == text
715+
716+
717+
@pytest.mark.parametrize("text", ENTITY_TEXTS)
718+
async def test_template_cdata_input_is_decoded_once(text: str):
719+
template = '<message role="user"><![CDATA[{{$input}}]]></message>'
720+
rendered = await KernelPromptTemplate(
721+
prompt_template_config=PromptTemplateConfig(name="test", description="test", template=template)
722+
).render(kernel=Kernel(), arguments=KernelArguments(input=text))
723+
724+
chat_history = ChatHistory.from_rendered_prompt(rendered)
725+
assert chat_history.messages[0].content == text
726+
727+
728+
@pytest.mark.parametrize("text", ENTITY_TEXTS)
729+
def test_chat_history_prompt_round_trip_keeps_entities(text: str):
730+
chat_history = ChatHistory()
731+
chat_history.add_user_message(text)
732+
chat_history.add_assistant_message(text)
733+
734+
restored = ChatHistory.from_rendered_prompt(chat_history.to_prompt())
735+
assert [message.content for message in restored.messages] == [text, text]

0 commit comments

Comments
 (0)