Skip to content

Commit 9a899e1

Browse files
committed
fix(voice): mark the say() generation done before the realtime push
Mark the generation done once playout is over and move the realtime chat context update after the agent state update, so the scheduler can start the next queued speech while the update waits for its acknowledgement. The task still finishes after the update, so awaiting say() and a handoff keep waiting for it.
1 parent 6a27e28 commit 9a899e1

2 files changed

Lines changed: 59 additions & 20 deletions

File tree

‎livekit-agents/livekit/agents/voice/agent_activity.py‎

Lines changed: 24 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -3379,33 +3379,17 @@ def _on_first_frame(fut: asyncio.Future[float] | asyncio.Future[None]) -> None:
33793379
current_span.set_attribute(trace_types.ATTR_E2E_LATENCY, e2e_latency)
33803380
_record_user_turn_stages(current_span, _previous_user_metrics)
33813381

3382+
spoken_msg: llm.ChatMessage | None = None
33823383
if forwarded_text and add_to_chat_ctx:
3383-
msg = self._agent._chat_ctx.add_message(
3384+
spoken_msg = self._agent._chat_ctx.add_message(
33843385
role="assistant",
33853386
content=forwarded_text,
33863387
interrupted=speech_handle.interrupted,
33873388
created_at=started_speaking_at if started_speaking_at is not None else time.time(),
33883389
metrics=assistant_metrics,
33893390
)
3390-
speech_handle._item_added([msg])
3391-
self._session._conversation_item_added(msg)
3392-
3393-
if (
3394-
self._rt_session is not None
3395-
and isinstance(self.llm, llm.RealtimeModel)
3396-
and self.llm.capabilities.mutable_chat_context
3397-
):
3398-
# send the spoken text now, placed by when it started, so the model's history
3399-
# keeps the order it was said in even if the user spoke during playback
3400-
rt_chat_ctx = self._rt_session.chat_ctx.copy()
3401-
rt_chat_ctx.insert(msg)
3402-
try:
3403-
await self._rt_session.update_chat_ctx(rt_chat_ctx)
3404-
except llm.RealtimeError as e:
3405-
logger.warning(
3406-
"failed to add the spoken message to the realtime chat context",
3407-
extra={"error": str(e)},
3408-
)
3391+
speech_handle._item_added([spoken_msg])
3392+
self._session._conversation_item_added(spoken_msg)
34093393

34103394
if self._session.agent_state == "speaking":
34113395
self._session._update_agent_state(
@@ -3418,6 +3402,26 @@ def _on_first_frame(fut: asyncio.Future[float] | asyncio.Future[None]) -> None:
34183402
if audio_out is not None and not audio_out.first_frame_fut.done():
34193403
audio_out.first_frame_fut.cancel()
34203404

3405+
if (
3406+
spoken_msg is not None
3407+
and self._rt_session is not None
3408+
and isinstance(self.llm, llm.RealtimeModel)
3409+
and self.llm.capabilities.mutable_chat_context
3410+
):
3411+
# playout is over, so the next speech can start while the update is acknowledged
3412+
speech_handle._mark_generation_done()
3413+
# send the spoken text, placed by when it started, so the model's history keeps
3414+
# the order it was said in even if the user spoke during playback
3415+
rt_chat_ctx = self._rt_session.chat_ctx.copy()
3416+
rt_chat_ctx.insert(spoken_msg)
3417+
try:
3418+
await self._rt_session.update_chat_ctx(rt_chat_ctx)
3419+
except llm.RealtimeError as e:
3420+
logger.warning(
3421+
"failed to add the spoken message to the realtime chat context",
3422+
extra={"error": str(e)},
3423+
)
3424+
34213425
def _on_enter_ignored_tools(self, tool_ctx: llm.ToolContext) -> list[llm.Tool]:
34223426
"""Tools flagged IGNORE_ON_ENTER, when this reply runs inside on_enter."""
34233427
on_enter_data = _OnEnterContextVar.get(None)

‎tests/test_realtime_say_chat_ctx.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
from __future__ import annotations
88

9+
import asyncio
910
import time
1011

1112
import pytest
@@ -42,6 +43,40 @@ async def test_say_adds_spoken_message_to_realtime_chat_ctx(mutable_chat_context
4243
assert remote_msgs == []
4344

4445

46+
async def test_say_lets_next_speech_start_while_chat_ctx_update_is_pending() -> None:
47+
model = FakeRealtimeModel(capabilities=fake_capabilities(supports_say=False))
48+
agent = Agent(instructions="test")
49+
50+
async with AgentSession(llm=model) as session:
51+
await session.start(agent)
52+
rt_session = model.active_session
53+
release_update = asyncio.Event()
54+
apply_update = rt_session.update_chat_ctx
55+
56+
async def unacknowledged_update(chat_ctx: llm.ChatContext) -> None:
57+
await release_update.wait()
58+
await apply_update(chat_ctx)
59+
60+
rt_session.update_chat_ctx = unacknowledged_update # type: ignore[method-assign]
61+
62+
first = session.say("first")
63+
second = session.say("second")
64+
65+
async def second_spoken() -> None:
66+
while not any(
67+
isinstance(item, llm.ChatMessage) and item.text_content == "second"
68+
for item in agent.chat_ctx.items
69+
):
70+
await asyncio.sleep(0.01)
71+
72+
await asyncio.wait_for(second_spoken(), timeout=5)
73+
assert not first.done()
74+
75+
release_update.set()
76+
await first
77+
await second
78+
79+
4580
async def test_say_places_spoken_message_before_later_remote_items() -> None:
4681
model = FakeRealtimeModel(capabilities=fake_capabilities(supports_say=False))
4782
agent = Agent(instructions="test")

0 commit comments

Comments
 (0)