Skip to content

Commit 86a13db

Browse files
jbeckwith-oaijbeckwith-oai
authored
fix(realtime): preserve heard audio during later responses (#5193)
1 parent b7d22bd commit 86a13db

2 files changed

Lines changed: 141 additions & 2 deletions

File tree

‎src/agents/realtime/openai_realtime.py‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,10 @@ def __init__(self) -> None:
247247
def ongoing_response(self) -> bool:
248248
return self._ongoing_response
249249

250+
@property
251+
def ongoing_response_id(self) -> str | None:
252+
return self._ongoing_response_id
253+
250254
@property
251255
def response_control(self) -> Literal["free", "create_requested", "cancel_requested"]:
252256
return self._response_control
@@ -1017,6 +1021,17 @@ def _get_playback_state(self) -> RealtimePlaybackState:
10171021
"elapsed_ms": None,
10181022
}
10191023

1024+
def _ongoing_response_owns_audio(self, item_id: str, item_content_index: int) -> bool:
1025+
if not self._ongoing_response:
1026+
return False
1027+
response_id = self._response_create_sequencer.ongoing_response_id
1028+
audio_state = self._audio_state_tracker.get_state(item_id, item_content_index)
1029+
# Preserve interruption when ownership is unknown. With known identities, only
1030+
# the response that produced this item can extend its received audio length.
1031+
if response_id is None or audio_state is None or audio_state.response_id is None:
1032+
return True
1033+
return audio_state.response_id == response_id
1034+
10201035
def _get_audio_limits(self, item_id: str, item_content_index: int) -> tuple[float, int] | None:
10211036
audio_state = self._audio_state_tracker.get_state(item_id, item_content_index)
10221037
if audio_state is None:
@@ -1099,7 +1114,12 @@ async def _interrupt_audio_playback(
10991114
_, max_audio_ms = audio_limits
11001115
truncated_ms = max(int(elapsed_ms), 0)
11011116
if (
1102-
(self._ongoing_response and not event.playback_only)
1117+
(
1118+
not event.playback_only
1119+
and self._ongoing_response_owns_audio(
1120+
current_item_id, current_item_content_index
1121+
)
1122+
)
11031123
or max_audio_ms is None
11041124
or truncated_ms < max_audio_ms
11051125
):
@@ -1452,7 +1472,9 @@ async def _handle_ws_event(self, event: dict[str, Any]):
14521472
if (
14531473
max_audio_ms is not None
14541474
and truncated_ms >= max_audio_ms
1455-
and not self._ongoing_response
1475+
and not self._ongoing_response_owns_audio(
1476+
playback_item_id, playback_content_index
1477+
)
14561478
):
14571479
logger.debug(
14581480
"Skipping truncate because playback appears complete. Item id: %s, "

‎tests/realtime/test_openai_realtime.py‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1332,6 +1332,123 @@ async def test_speech_started_truncates_when_response_ongoing(self, model, monke
13321332
assert truncate_events[0].audio_end_ms == 1000
13331333

13341334

1335+
class TestAudioResponseOwnership(TestOpenAIRealtimeWebSocketModel):
1336+
@pytest.mark.asyncio
1337+
@pytest.mark.parametrize("interrupt", [False, True], ids=["vad", "session-interrupt"])
1338+
@pytest.mark.parametrize("use_playback_tracker", [False, True], ids=["clock", "tracker"])
1339+
@pytest.mark.parametrize(
1340+
("start_second_response", "elapsed_ms", "expected_truncates"),
1341+
[
1342+
(True, 5000, []),
1343+
(False, 5000, [("audio_1", 1000)]),
1344+
(True, 500, [("audio_1", 500)]),
1345+
],
1346+
ids=["heard-earlier-response", "same-response-streaming", "unplayed-earlier-audio"],
1347+
)
1348+
async def test_interrupt_truncates_only_unfinished_audio(
1349+
self,
1350+
model,
1351+
monkeypatch,
1352+
interrupt,
1353+
use_playback_tracker,
1354+
start_second_response,
1355+
elapsed_ms,
1356+
expected_truncates,
1357+
):
1358+
model._websocket = AsyncMock()
1359+
model._send_raw_message = AsyncMock()
1360+
model._audio_state_tracker.set_audio_format("pcm16")
1361+
listener = AsyncMock()
1362+
model.add_listener(listener)
1363+
1364+
def response(response_id, status):
1365+
return {
1366+
"id": response_id,
1367+
"object": "realtime.response",
1368+
"status": status,
1369+
"output": [],
1370+
}
1371+
1372+
monkeypatch.setattr(time, "monotonic", lambda: 100.0)
1373+
await model._handle_ws_event(
1374+
{
1375+
"type": "response.created",
1376+
"event_id": "e1",
1377+
"response": response("r1", "in_progress"),
1378+
}
1379+
)
1380+
await model._handle_ws_event(
1381+
{
1382+
"type": "response.output_audio.delta",
1383+
"event_id": "e2",
1384+
"response_id": "r1",
1385+
"item_id": "audio_1",
1386+
"output_index": 0,
1387+
"content_index": 0,
1388+
# One second of PCM16 audio at 24 kHz.
1389+
"delta": base64.b64encode(bytes(48_000)).decode(),
1390+
}
1391+
)
1392+
if start_second_response:
1393+
await model._handle_ws_event(
1394+
{
1395+
"type": "response.done",
1396+
"event_id": "e3",
1397+
"response": response("r1", "completed"),
1398+
}
1399+
)
1400+
await model._handle_ws_event(
1401+
{
1402+
"type": "response.created",
1403+
"event_id": "e4",
1404+
"response": response("r2", "in_progress"),
1405+
}
1406+
)
1407+
# A later response can generate tool arguments without producing audio.
1408+
await model._handle_ws_event(
1409+
{
1410+
"type": "response.function_call_arguments.delta",
1411+
"event_id": "e5",
1412+
"response_id": "r2",
1413+
"item_id": "tool_2",
1414+
"output_index": 0,
1415+
"call_id": "call_2",
1416+
"delta": '{"query":',
1417+
}
1418+
)
1419+
1420+
if use_playback_tracker:
1421+
model._playback_tracker = RealtimePlaybackTracker()
1422+
model._playback_tracker.on_play_ms("audio_1", 0, min(elapsed_ms, 1000))
1423+
monkeypatch.setattr(time, "monotonic", lambda: 100.0 + elapsed_ms / 1000)
1424+
if interrupt:
1425+
session = RealtimeSession(model=model, agent=RealtimeAgent(name="Test"), context=None)
1426+
await session.interrupt()
1427+
else:
1428+
await model._handle_ws_event(
1429+
{
1430+
"type": "input_audio_buffer.speech_started",
1431+
"event_id": "e6",
1432+
"item_id": "user_2",
1433+
"audio_start_ms": 0,
1434+
}
1435+
)
1436+
1437+
sent = [call.args[0] for call in model._send_raw_message.await_args_list]
1438+
assert [
1439+
(event.item_id, event.audio_end_ms)
1440+
for event in sent
1441+
if event.type == "conversation.item.truncate"
1442+
] == expected_truncates
1443+
assert sum(event.type == "response.cancel" for event in sent) == 1
1444+
emitted = [call.args[0] for call in listener.on_event.await_args_list]
1445+
assert not [event for event in emitted if event.type in {"error", "exception"}]
1446+
assert [event.item_id for event in emitted if event.type == "audio_interrupted"] == [
1447+
"audio_1"
1448+
]
1449+
assert model._get_playback_state()["current_item_id"] is None
1450+
1451+
13351452
class TestSendEventAndConfig(TestOpenAIRealtimeWebSocketModel):
13361453
@pytest.mark.asyncio
13371454
async def test_send_event_dispatch(self, model, monkeypatch):

0 commit comments

Comments
 (0)