|
13 | 13 | the suppressed segment. Otherwise those stale background words stay in |
14 | 14 | ``_current_transcript`` - where they count toward the ``min_words`` interruption |
15 | 15 | gate - and ``commit_user_turn`` appends them to the user turn. |
| 16 | +
|
| 17 | +The empty final carries no segment identity, so recognition only drops interim |
| 18 | +text that can belong to the suppressed segment: a speaker-less (Deepgram-style) |
| 19 | +interim is dropped, while an interim the adapter attributed to the primary |
| 20 | +speaker is another segment's in-progress words and survives for the commit. The |
| 21 | +empty final also refreshes the manual-commit freshness anchor: the wrapped STT |
| 22 | +did finalize, so a commit must not wait for yet another final. |
16 | 23 | """ |
17 | 24 |
|
18 | 25 | from __future__ import annotations |
|
23 | 30 |
|
24 | 31 | import pytest |
25 | 32 |
|
| 33 | +from livekit import rtc |
26 | 34 | from livekit.agents import stt |
27 | 35 | from livekit.agents.language import LanguageCode |
| 36 | +from livekit.agents.stt import MultiSpeakerAdapter, RecognizeStream |
| 37 | +from livekit.agents.stt.multi_speaker_adapter import PrimarySpeakerDetectionOptions |
| 38 | +from livekit.agents.types import ( |
| 39 | + DEFAULT_API_CONNECT_OPTIONS, |
| 40 | + NOT_GIVEN, |
| 41 | + APIConnectOptions, |
| 42 | + NotGivenOr, |
| 43 | +) |
| 44 | +from livekit.agents.utils import AudioBuffer |
28 | 45 | from livekit.agents.voice.audio_recognition import AudioRecognition |
29 | 46 |
|
30 | 47 | pytestmark = pytest.mark.unit |
31 | 48 |
|
| 49 | +_CONN = APIConnectOptions(max_retry=0, timeout=5.0) |
| 50 | + |
32 | 51 |
|
33 | 52 | def _make_recognition() -> AudioRecognition: |
34 | 53 | """Wire the attributes the transcript path of ``_on_stt_event`` touches.""" |
@@ -57,6 +76,7 @@ def _make_recognition() -> AudioRecognition: |
57 | 76 | ar._end_of_turn_task = None |
58 | 77 | ar._audio_transcript = "" |
59 | 78 | ar._audio_interim_transcript = "" |
| 79 | + ar._audio_interim_from_primary = False |
60 | 80 | ar._audio_preflight_transcript = "" |
61 | 81 | ar._final_transcript_confidence = [] |
62 | 82 | ar._final_transcript_received = asyncio.Event() |
@@ -153,12 +173,276 @@ async def test_empty_final_keeps_background_text_out_of_commit_user_turn() -> No |
153 | 173 | ar = _make_recognition() |
154 | 174 |
|
155 | 175 | await _feed_adapter_sequence(ar) |
156 | | - # the empty final just arrived, so the transcript is fresh and the commit |
157 | | - # does not need to flush the STT and wait for another final |
158 | | - ar._last_final_transcript_time = time.time() |
159 | 176 |
|
160 | 177 | committed = await ar._commit_user_turn( |
161 | 178 | audio_detached=False, transcript_timeout=0.01, stt_flush_duration=0.0 |
162 | 179 | ) |
163 | 180 |
|
164 | 181 | assert committed == "hello" |
| 182 | + |
| 183 | + |
| 184 | +async def test_commit_after_empty_final_does_not_wait_for_another_final() -> None: |
| 185 | + """An empty final satisfies the manual-commit freshness check. |
| 186 | +
|
| 187 | + The commit waits for a new final only when the last one is older than 0.5s, |
| 188 | + because it then assumes the STT still holds unfinalized audio. A suppressed |
| 189 | + background segment produces a real final, so once the empty final arrives |
| 190 | + the STT buffer is flushed: waiting for yet another final stalls the commit |
| 191 | + until ``transcript_timeout``. |
| 192 | + """ |
| 193 | + ar = _make_recognition() |
| 194 | + |
| 195 | + await ar._on_stt_event(_final_transcript("hello", speaker_id="S0")) |
| 196 | + await asyncio.sleep(0.55) # the last final is now outside the 0.5s window |
| 197 | + await ar._on_stt_event(_interim_transcript("background chatter")) |
| 198 | + await ar._on_stt_event(_empty_final_transcript()) |
| 199 | + |
| 200 | + started = time.monotonic() |
| 201 | + committed = await ar._commit_user_turn( |
| 202 | + audio_detached=False, transcript_timeout=1.0, stt_flush_duration=0.0 |
| 203 | + ) |
| 204 | + elapsed = time.monotonic() - started |
| 205 | + |
| 206 | + assert committed == "hello" |
| 207 | + assert elapsed < 0.5, f"commit waited {elapsed:.2f}s for a final that already arrived" |
| 208 | + |
| 209 | + |
| 210 | +# --- the empty final through the real MultiSpeakerAdapter ------------------- |
| 211 | +# |
| 212 | +# The tests above feed the event sequence directly. The adapter's empty final is |
| 213 | +# not addressed to a particular interim, so these drive the real adapter to pin |
| 214 | +# down which interim it is allowed to clear. |
| 215 | + |
| 216 | +_SAMPLE_RATE = 1000 |
| 217 | +_SAMPLES_PER_FRAME = 100 # 100 ms, matching frame_size_ms below |
| 218 | + |
| 219 | + |
| 220 | +def _tone_frame(amplitude: int) -> rtc.AudioFrame: |
| 221 | + """A mono int16 frame whose every sample equals ``amplitude``.""" |
| 222 | + return rtc.AudioFrame( |
| 223 | + data=amplitude.to_bytes(2, "little", signed=True) * _SAMPLES_PER_FRAME, |
| 224 | + sample_rate=_SAMPLE_RATE, |
| 225 | + num_channels=1, |
| 226 | + samples_per_channel=_SAMPLES_PER_FRAME, |
| 227 | + ) |
| 228 | + |
| 229 | + |
| 230 | +def _timed_transcript( |
| 231 | + event_type: stt.SpeechEventType, |
| 232 | + text: str, |
| 233 | + *, |
| 234 | + speaker_id: str | None, |
| 235 | + start_time: float, |
| 236 | + end_time: float, |
| 237 | +) -> stt.SpeechEvent: |
| 238 | + return stt.SpeechEvent( |
| 239 | + type=event_type, |
| 240 | + alternatives=[ |
| 241 | + stt.SpeechData( |
| 242 | + language=LanguageCode("en"), |
| 243 | + text=text, |
| 244 | + confidence=0.9, |
| 245 | + speaker_id=speaker_id, |
| 246 | + start_time=start_time, |
| 247 | + end_time=end_time, |
| 248 | + ) |
| 249 | + ], |
| 250 | + ) |
| 251 | + |
| 252 | + |
| 253 | +class _ScriptedDiarizingSTT(stt.STT): |
| 254 | + """Diarization-capable STT that replays one event sequence per stream. |
| 255 | +
|
| 256 | + The adapter runs RMS-based primary-speaker detection over the audio it |
| 257 | + forwards, so the scripted stream waits for ``expected_frames`` frames before |
| 258 | + emitting, matching the order a provider produces: audio first, then the |
| 259 | + results for it. |
| 260 | + """ |
| 261 | + |
| 262 | + def __init__(self, events: list[stt.SpeechEvent], *, expected_frames: int) -> None: |
| 263 | + super().__init__( |
| 264 | + capabilities=stt.STTCapabilities(streaming=True, interim_results=True, diarization=True) |
| 265 | + ) |
| 266 | + self._scripted_events = events |
| 267 | + self._expected_frames = expected_frames |
| 268 | + |
| 269 | + async def _recognize_impl( |
| 270 | + self, |
| 271 | + buffer: AudioBuffer, |
| 272 | + *, |
| 273 | + language: NotGivenOr[str] = NOT_GIVEN, |
| 274 | + conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS, |
| 275 | + ) -> stt.SpeechEvent: |
| 276 | + raise NotImplementedError("this fake only supports streaming") |
| 277 | + |
| 278 | + def stream( |
| 279 | + self, |
| 280 | + *, |
| 281 | + language: NotGivenOr[str] = NOT_GIVEN, |
| 282 | + conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS, |
| 283 | + ) -> RecognizeStream: |
| 284 | + return _ScriptedDiarizingStream( |
| 285 | + stt=self, |
| 286 | + events=self._scripted_events, |
| 287 | + expected_frames=self._expected_frames, |
| 288 | + conn_options=conn_options, |
| 289 | + ) |
| 290 | + |
| 291 | + |
| 292 | +class _ScriptedDiarizingStream(RecognizeStream): |
| 293 | + def __init__( |
| 294 | + self, |
| 295 | + *, |
| 296 | + stt: stt.STT, |
| 297 | + events: list[stt.SpeechEvent], |
| 298 | + expected_frames: int, |
| 299 | + conn_options: APIConnectOptions, |
| 300 | + ) -> None: |
| 301 | + super().__init__(stt=stt, conn_options=conn_options) |
| 302 | + self._scripted_events = events |
| 303 | + self._expected_frames = expected_frames |
| 304 | + |
| 305 | + async def _run(self) -> None: |
| 306 | + received = 0 |
| 307 | + while received < self._expected_frames: |
| 308 | + if isinstance(await self._input_ch.recv(), rtc.AudioFrame): |
| 309 | + received += 1 |
| 310 | + for event in self._scripted_events: |
| 311 | + self._event_ch.send_nowait(event) |
| 312 | + |
| 313 | + |
| 314 | +def _diarizing_adapter(events: list[stt.SpeechEvent], *, frames: int) -> MultiSpeakerAdapter: |
| 315 | + return MultiSpeakerAdapter( |
| 316 | + stt=_ScriptedDiarizingSTT(events, expected_frames=frames), |
| 317 | + suppress_background_speaker=True, |
| 318 | + primary_detection_options=PrimarySpeakerDetectionOptions( |
| 319 | + frame_size_ms=100, min_rms_samples=1 |
| 320 | + ), |
| 321 | + ) |
| 322 | + |
| 323 | + |
| 324 | +async def _adapter_events( |
| 325 | + adapter: MultiSpeakerAdapter, frames: list[rtc.AudioFrame], count: int |
| 326 | +) -> list[stt.SpeechEvent]: |
| 327 | + stream = adapter.stream(conn_options=_CONN) |
| 328 | + for frame in frames: |
| 329 | + stream.push_frame(frame) |
| 330 | + events: list[stt.SpeechEvent] = [] |
| 331 | + async for event in stream: |
| 332 | + events.append(event) |
| 333 | + if len(events) == count: |
| 334 | + break |
| 335 | + await stream.aclose() |
| 336 | + return events |
| 337 | + |
| 338 | + |
| 339 | +async def test_primary_interim_survives_a_suppressed_background_final() -> None: |
| 340 | + """A background final must not drop the primary speaker's interim. |
| 341 | +
|
| 342 | + Diarizing providers that attribute interim results (AssemblyAI's turns carry |
| 343 | + ``speaker_label``) let the adapter forward a primary interim. A quieter |
| 344 | + background final right after it is suppressed and replaced by the empty |
| 345 | + final, which carries no segment identity: clearing the primary interim here |
| 346 | + would make a manual commit lose speech that has not been finalized yet. |
| 347 | + """ |
| 348 | + events = [ |
| 349 | + _timed_transcript( |
| 350 | + stt.SpeechEventType.FINAL_TRANSCRIPT, |
| 351 | + "hello", |
| 352 | + speaker_id="S0", |
| 353 | + start_time=0.0, |
| 354 | + end_time=0.2, |
| 355 | + ), |
| 356 | + _timed_transcript( |
| 357 | + stt.SpeechEventType.INTERIM_TRANSCRIPT, |
| 358 | + "book a ticket", |
| 359 | + speaker_id="S0", |
| 360 | + start_time=0.2, |
| 361 | + end_time=0.3, |
| 362 | + ), |
| 363 | + # quiet background speaker: the detector keeps S0 as primary and the |
| 364 | + # adapter suppresses this final, emitting the empty final instead |
| 365 | + _timed_transcript( |
| 366 | + stt.SpeechEventType.FINAL_TRANSCRIPT, |
| 367 | + "background chatter", |
| 368 | + speaker_id="S1", |
| 369 | + start_time=0.3, |
| 370 | + end_time=0.4, |
| 371 | + ), |
| 372 | + ] |
| 373 | + adapter = _diarizing_adapter(events, frames=4) |
| 374 | + frames = [_tone_frame(3000), _tone_frame(3000), _tone_frame(50), _tone_frame(50)] |
| 375 | + |
| 376 | + emitted = await _adapter_events(adapter, frames, count=3) |
| 377 | + |
| 378 | + assert [event.type for event in emitted] == [ |
| 379 | + stt.SpeechEventType.FINAL_TRANSCRIPT, |
| 380 | + stt.SpeechEventType.INTERIM_TRANSCRIPT, |
| 381 | + stt.SpeechEventType.FINAL_TRANSCRIPT, |
| 382 | + ] |
| 383 | + assert emitted[0].alternatives[0].text == "hello" |
| 384 | + assert emitted[1].alternatives[0].text == "book a ticket" |
| 385 | + assert emitted[2].alternatives[0].text == "" # the adapter's synthesized clear |
| 386 | + |
| 387 | + ar = _make_recognition() |
| 388 | + for event in emitted: |
| 389 | + await ar._on_stt_event(event) |
| 390 | + |
| 391 | + assert ar._audio_interim_transcript == "book a ticket" |
| 392 | + committed = await ar._commit_user_turn( |
| 393 | + audio_detached=False, transcript_timeout=0.01, stt_flush_duration=0.0 |
| 394 | + ) |
| 395 | + assert committed == "hello book a ticket" |
| 396 | + |
| 397 | + |
| 398 | +async def test_speakerless_interim_is_still_cleared_by_the_empty_final() -> None: |
| 399 | + """Control: the Deepgram-style case this fix targets is still cleared. |
| 400 | +
|
| 401 | + Deepgram reports interims without a speaker id, so they bypass the adapter's |
| 402 | + suppression and reach recognition. The suppressed segment's final arrives as |
| 403 | + the empty final, and the buffered interim is that same segment's partial |
| 404 | + text: it must not stay in the user turn. |
| 405 | + """ |
| 406 | + events = [ |
| 407 | + _timed_transcript( |
| 408 | + stt.SpeechEventType.FINAL_TRANSCRIPT, |
| 409 | + "hello", |
| 410 | + speaker_id="S0", |
| 411 | + start_time=0.0, |
| 412 | + end_time=0.2, |
| 413 | + ), |
| 414 | + _timed_transcript( |
| 415 | + stt.SpeechEventType.INTERIM_TRANSCRIPT, |
| 416 | + "background chatter", |
| 417 | + speaker_id=None, |
| 418 | + start_time=0.2, |
| 419 | + end_time=0.3, |
| 420 | + ), |
| 421 | + _timed_transcript( |
| 422 | + stt.SpeechEventType.FINAL_TRANSCRIPT, |
| 423 | + "background chatter", |
| 424 | + speaker_id="S1", |
| 425 | + start_time=0.3, |
| 426 | + end_time=0.4, |
| 427 | + ), |
| 428 | + ] |
| 429 | + adapter = _diarizing_adapter(events, frames=4) |
| 430 | + frames = [_tone_frame(3000), _tone_frame(3000), _tone_frame(50), _tone_frame(50)] |
| 431 | + |
| 432 | + emitted = await _adapter_events(adapter, frames, count=3) |
| 433 | + assert [event.alternatives[0].text for event in emitted] == [ |
| 434 | + "hello", |
| 435 | + "background chatter", |
| 436 | + "", |
| 437 | + ] |
| 438 | + |
| 439 | + ar = _make_recognition() |
| 440 | + for event in emitted: |
| 441 | + await ar._on_stt_event(event) |
| 442 | + |
| 443 | + assert ar._audio_interim_transcript == "" |
| 444 | + assert ar._current_transcript == "hello" |
| 445 | + committed = await ar._commit_user_turn( |
| 446 | + audio_detached=False, transcript_timeout=0.01, stt_flush_duration=0.0 |
| 447 | + ) |
| 448 | + assert committed == "hello" |
0 commit comments