Skip to content

Commit c42f3c6

Browse files
fix(voice): finish streamed transcription on end of input (#5195)
* fix(voice): finish streamed transcription on end of input * fix(voice): preserve EOF compatibility and inactivity timeout
1 parent 160ed62 commit c42f3c6

2 files changed

Lines changed: 344 additions & 4 deletions

File tree

‎src/agents/voice/models/openai_stt.py‎

Lines changed: 67 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
from dataclasses import dataclass
88
from time import monotonic
99
from typing import Any, cast
10+
from uuid import uuid4
1011

1112
from openai import AsyncOpenAI
1213

@@ -46,6 +47,10 @@ class WebsocketDoneSentinel:
4647
pass
4748

4849

50+
class _InputCompleteSentinel:
51+
pass
52+
53+
4954
class _ListenerError(Exception):
5055
pass
5156

@@ -136,9 +141,9 @@ def __init__(
136141
asyncio.Queue()
137142
)
138143
self._websocket: websockets.ClientConnection | None = None
139-
self._event_queue: asyncio.Queue[dict[str, Any] | ErrorSentinel | WebsocketDoneSentinel] = (
140-
asyncio.Queue()
141-
)
144+
self._event_queue: asyncio.Queue[
145+
dict[str, Any] | ErrorSentinel | WebsocketDoneSentinel | _InputCompleteSentinel
146+
] = asyncio.Queue()
142147
self._state_queue: asyncio.Queue[dict[str, Any] | ErrorSentinel] = asyncio.Queue()
143148
self._turn_audio_buffer: list[npt.NDArray[np.int16 | np.float32]] = []
144149
self._tracing_span: Span[TranscriptionSpanData] | None = None
@@ -151,6 +156,8 @@ def __init__(
151156
self._connection_task: asyncio.Task[Any] | None = None
152157
self._stored_exception: Exception | None = None
153158
self._completion_signaled = False
159+
self._final_commit_id: str | None = None
160+
self._closing = False
154161

155162
def _get_transcription_config(self) -> dict[str, Any]:
156163
transcription_config: dict[str, Any] = {"model": self._model}
@@ -215,6 +222,15 @@ async def _event_listener(self) -> None:
215222
event = json.loads(message)
216223

217224
if event.get("type") == "error":
225+
error = event.get("error", {})
226+
if (
227+
self._final_commit_id is not None
228+
and error.get("event_id") == self._final_commit_id
229+
and error.get("code") == "input_audio_buffer_commit_empty"
230+
):
231+
# VAD may already have committed the final buffer. Only this
232+
# specific response to our EOF commit is an expected no-op.
233+
continue
218234
raise STTWebsocketConnectionError(f"Error event: {event.get('error')}")
219235

220236
if event.get("type") in [
@@ -302,29 +318,74 @@ async def _setup_connection(self, ws: websockets.ClientConnection) -> None:
302318
raise
303319

304320
async def _handle_events(self) -> None:
321+
pending_transcripts: set[str] = set()
322+
input_cleared = False
305323
while True:
306324
try:
307325
event = await asyncio.wait_for(
308326
self._event_queue.get(), timeout=EVENT_INACTIVITY_TIMEOUT
309327
)
328+
if isinstance(event, _InputCompleteSentinel):
329+
self._final_commit_id = uuid4().hex
330+
assert self._websocket is not None, "Websocket not initialized"
331+
await self._websocket.send(
332+
json.dumps(
333+
{"type": "input_audio_buffer.commit", "event_id": self._final_commit_id}
334+
)
335+
)
336+
# The clear acknowledgement follows the final commit (or its
337+
# empty-buffer error), even when VAD wins the commit race. It
338+
# ends input without cancelling already committed transcripts.
339+
await self._websocket.send(json.dumps({"type": "input_audio_buffer.clear"}))
340+
continue
310341
if isinstance(event, WebsocketDoneSentinel):
342+
if (
343+
not self._closing
344+
and self._final_commit_id is not None
345+
and (not input_cleared or pending_transcripts)
346+
):
347+
raise STTWebsocketConnectionError(
348+
"Websocket closed before final transcription completed"
349+
)
311350
# processed all events and websocket is done
312351
break
313352
if isinstance(event, ErrorSentinel):
314353
raise STTWebsocketConnectionError("Error parsing events") from event.error
315354

316355
event_type = event.get("type", "unknown")
356+
if event_type == "input_audio_buffer.committed":
357+
pending_transcripts.add(event["item_id"])
358+
elif event_type == "input_audio_buffer.cleared":
359+
input_cleared = True
360+
elif event_type == "conversation.item.input_audio_transcription.failed":
361+
raise STTWebsocketConnectionError("Input audio transcription failed")
317362
if event_type in [
318363
"input_audio_transcription_completed", # legacy
319364
"conversation.item.input_audio_transcription.completed",
320365
]:
366+
if (
367+
event_type == "input_audio_transcription_completed"
368+
and "item_id" not in event
369+
):
370+
# Legacy sessions don't identify individual completed items;
371+
# each completion still settles one outstanding transcript.
372+
if pending_transcripts:
373+
pending_transcripts.pop()
374+
else:
375+
pending_transcripts.discard(event.get("item_id", ""))
321376
transcript = cast(str, event.get("transcript", ""))
322377
if len(transcript) > 0:
323378
self._end_turn(transcript)
324379
self._start_turn()
325380
await self._output_queue.put(transcript)
381+
if self._final_commit_id is not None and input_cleared and not pending_transcripts:
382+
break
326383
await asyncio.sleep(0) # yield control
327-
except asyncio.TimeoutError:
384+
except asyncio.TimeoutError as e:
385+
if self._final_commit_id is not None:
386+
error = STTWebsocketConnectionError("Timeout waiting for final transcription")
387+
await self._output_queue.put(ErrorSentinel(error))
388+
raise error from e
328389
# No new events for a while. Assume the session is done.
329390
break
330391
except Exception as e:
@@ -345,6 +406,7 @@ async def _stream_audio(
345406
while True:
346407
buffer = await audio_queue.get()
347408
if buffer is None:
409+
await self._event_queue.put(_InputCompleteSentinel())
348410
break
349411

350412
if self._trace_include_sensitive_audio_data:
@@ -510,6 +572,7 @@ async def transcribe_turns(self) -> AsyncIterator[str]:
510572
raise exception_to_raise
511573

512574
async def close(self) -> None:
575+
self._closing = True
513576
try:
514577
if self._websocket:
515578
await self._websocket.close()

0 commit comments

Comments
 (0)