Skip to content

Commit 52c3a4e

Browse files
seanturner83claude
seanturner83
and
claude
committed
fix(litellm): suppress empty leading content delta so refusal keeps index 0
Per @seratch: rather than open an empty text part for a leading "" content delta and then close/reindex it, drop that delta before it opens a text part. The synthesized content-filter refusal then lands at content index 0 in both the streamed events and response.completed, so raw-event replay and the final response stay aligned (previously the refusal streamed at index 1 while the completed response placed it at position 0). The suppression is scoped to a leading empty ("") delta when no text part has opened yet; empty deltas after real text keep their existing behavior. With the empty part gone, the synthesis block's close/distinct-index logic is dead and removed — the block now just announces the message and emits the refusal at the first content index. Tests updated to assert no text part / no output_text.delta is emitted and that the refusal's streamed content_index (0) matches its position in response.completed. Live-verified against Bedrock claude-fable-5: 0 output_text.delta, 1 output_item.added, one content_part.added (ResponseOutputRefusal) at index 0, completed content[0] a single refusal. Co-Authored-By: Claude Opus 4.8 <[email protected]>
1 parent 728b951 commit 52c3a4e

2 files changed

Lines changed: 54 additions & 59 deletions

File tree

‎src/agents/models/chatcmpl_stream_handler.py‎

Lines changed: 33 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -685,7 +685,17 @@ async def handle_stream(
685685
yield event
686686

687687
# Handle regular content
688-
if delta.content is not None:
688+
if delta.content is not None and not (
689+
not state.text_content_index_and_output and delta.content == ""
690+
):
691+
# An empty leading content delta ("") is dropped rather than
692+
# opening a text content part: materializing an empty part would
693+
# add a spurious ResponseOutputText to response.completed. Bedrock
694+
# content-filter turns emit exactly this "" warm-up chunk before
695+
# the terminal content_filter, so suppressing it here keeps the
696+
# synthesized refusal (below) at content index 0 in both the
697+
# streamed events and the completed response. Empty deltas after a
698+
# text part has already opened keep their existing behavior.
689699
if not state.text_content_index_and_output:
690700
content_index = 0
691701
if state.reasoning_content_index_and_output:
@@ -953,63 +963,39 @@ async def handle_stream(
953963
# calls) — a content_filter that still emitted content is left as-is.
954964
if (
955965
saw_content_filter
956-
and not (
957-
state.text_content_index_and_output and state.text_content_index_and_output[1].text
958-
)
966+
and state.text_content_index_and_output is None
959967
and state.refusal_content_index_and_output is None
960968
and not state.function_calls
961969
):
962-
# A content-filtered turn on Bedrock often opens an empty text part
963-
# (a "" content delta) before terminating, so gate on the accumulated
964-
# text being empty rather than on the part being absent.
970+
# A content-filtered turn (e.g. Bedrock) can terminate with no
971+
# emitted output. Its leading empty "" content delta is suppressed
972+
# above so no text part opens, so we announce a fresh assistant
973+
# message and place the refusal at the first content index — the same
974+
# index it occupies in response.completed, keeping raw-event replay
975+
# and the final response aligned.
965976
refusal_index = 0
966977
if state.reasoning_content_index_and_output:
967978
refusal_index += 1
968-
# If an empty text part was already opened, the assistant message +
969-
# that content part have already been announced on the stream. Close
970-
# the stale text part and reuse its index for the refusal, rather than
971-
# re-announcing the message and leaving the text part unterminated
972-
# (which would produce an incoherent event sequence).
973-
empty_text_part = state.text_content_index_and_output
974-
if empty_text_part is not None:
975-
# Close the stale empty text part and give the refusal a DISTINCT
976-
# index (not the text part's), so a consumer replaying events by
977-
# (item_id, content_index) never sees two different part types at
978-
# the same slot. The empty text part is dropped from the final
979-
# message, so only the refusal remains in response.completed.
980-
yield ResponseContentPartDoneEvent(
981-
content_index=empty_text_part[0],
982-
item_id=FAKE_RESPONSES_ID,
983-
output_index=output_layout.assistant_message_output_index(state),
984-
part=empty_text_part[1],
985-
type="response.content_part.done",
986-
sequence_number=sequence_number.get_and_increment(),
987-
)
988-
refusal_index = empty_text_part[0] + 1
989-
state.text_content_index_and_output = None
990979
refusal_message = "Response withheld by the provider's content filter."
991980
state.refusal_content_index_and_output = (
992981
refusal_index,
993982
ResponseOutputRefusal(refusal=refusal_message, type="refusal"),
994983
)
995-
# Announce the assistant message only if it wasn't already opened by a
996-
# prior text part.
997-
if empty_text_part is None:
998-
assistant_item = ResponseOutputMessage(
999-
id=FAKE_RESPONSES_ID,
1000-
content=[],
1001-
role="assistant",
1002-
type="message",
1003-
status="in_progress",
1004-
)
1005-
if state.provider_data:
1006-
assistant_item.provider_data = state.provider_data.copy() # type: ignore[attr-defined]
1007-
yield ResponseOutputItemAddedEvent(
1008-
item=assistant_item,
1009-
output_index=output_layout.assistant_message_output_index(state),
1010-
type="response.output_item.added",
1011-
sequence_number=sequence_number.get_and_increment(),
1012-
)
984+
assistant_item = ResponseOutputMessage(
985+
id=FAKE_RESPONSES_ID,
986+
content=[],
987+
role="assistant",
988+
type="message",
989+
status="in_progress",
990+
)
991+
if state.provider_data:
992+
assistant_item.provider_data = state.provider_data.copy() # type: ignore[attr-defined]
993+
yield ResponseOutputItemAddedEvent(
994+
item=assistant_item,
995+
output_index=output_layout.assistant_message_output_index(state),
996+
type="response.output_item.added",
997+
sequence_number=sequence_number.get_and_increment(),
998+
)
1013999
yield ResponseContentPartAddedEvent(
10141000
content_index=refusal_index,
10151001
item_id=FAKE_RESPONSES_ID,

‎tests/models/test_litellm_chatcompletions_stream.py‎

Lines changed: 21 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -428,10 +428,10 @@ async def test_stream_response_synthesizes_refusal_on_content_filter(monkeypatch
428428
ResponseOutputRefusal so the completed response carries an explicit refusal
429429
rather than an empty assistant turn.
430430
431-
Mirrors the real Bedrock chunk shape: an empty-string content delta (which
432-
opens a text content part) followed by a terminal content_filter chunk with
433-
no content. The synthesized refusal must replace that empty text part, not
434-
sit alongside it.
431+
Mirrors the real Bedrock chunk shape: an empty-string content delta followed
432+
by a terminal content_filter chunk with no content. The empty "" delta must
433+
not open a text content part; the synthesized refusal must be the only
434+
content part, at the same index in the stream and in response.completed.
435435
"""
436436
chunk1 = ChatCompletionChunk(
437437
id="chunk-id",
@@ -500,16 +500,19 @@ async def patched_fetch_response(self, *args, **kwargs):
500500
refusal_deltas = [e for e in output_events if e.type == "response.refusal.delta"]
501501
assert refusal_deltas and refusal_deltas[0].delta
502502

503-
# Event coherence: the assistant message is announced exactly once (not
504-
# re-announced by the synthesized refusal), and every content part that is
505-
# opened is also closed — the empty text part opened by the "" content delta
506-
# must be given a matching content_part.done, not left dangling.
503+
# Event coherence: the assistant message is announced exactly once, and every
504+
# content part that is opened is also closed.
507505
assert types.count("response.output_item.added") == 1
508506
assert types.count("response.content_part.added") == types.count("response.content_part.done")
509507

508+
# The empty "" content delta must NOT open a text content part: no text part
509+
# events and no output_text.delta are emitted at all.
510+
assert "response.output_text.delta" not in types
511+
added_parts = [e for e in output_events if e.type == "response.content_part.added"]
512+
assert len(added_parts) == 1
513+
assert isinstance(added_parts[0].part, ResponseOutputRefusal)
514+
510515
# The completed response contains exactly one content part: the refusal.
511-
# (The empty text part opened by the "" content delta must be dropped, not
512-
# left sitting alongside the refusal.)
513516
completed_event = output_events[-1]
514517
assert isinstance(completed_event, ResponseCompletedEvent)
515518
completed_resp = completed_event.response
@@ -518,8 +521,14 @@ async def patched_fetch_response(self, *args, **kwargs):
518521
refusal_part = completed_resp.output[0].content[0]
519522
assert isinstance(refusal_part, ResponseOutputRefusal)
520523
assert refusal_part.refusal
521-
# No spurious empty text delta leaked through as real content.
522-
assert not any(e.type == "response.output_text.delta" and e.delta for e in output_events)
524+
525+
# The refusal's streamed content_index matches its position in the completed
526+
# response (0), so raw-event replay and the final response stay aligned.
527+
assert added_parts[0].content_index == 0
528+
assert refusal_deltas[0].content_index == 0
529+
done_parts = [e for e in output_events if e.type == "response.content_part.done"]
530+
assert len(done_parts) == 1
531+
assert done_parts[0].content_index == 0
523532

524533

525534
@pytest.mark.allow_call_model_methods

0 commit comments

Comments
 (0)