Skip to content

Commit b81bc83

Browse files
committed
fix(voice): keep primary interim and refresh commit anchor on empty final
MultiSpeakerAdapter signals a suppressed background segment with an empty FINAL_TRANSCRIPT. It carries no segment identity, so the previous clear dropped whatever interim was buffered. For providers that attribute interims the adapter forwards only the primary speaker's, and that interim belongs to a different, still-open segment: clearing it made a manual commit omit those words. Scope the clear to interim state that can belong to the suppressed segment: a speaker-less interim (Deepgram) is still dropped, a primary- attributed interim survives until its own final. The adapter contract is unchanged. The empty final is also a real final from the wrapped STT. Refresh _last_final_transcript_time so commit_user_turn does not clear the completion signal and wait for another final until transcript_timeout. Tests drive the real adapter: interleaved primary interim plus suppressed background final (kept), speaker-less interim plus suppressed final (cleared, control), and a commit after an empty final whose previous final is older than the 0.5s window (returns immediately).
1 parent 46142b3 commit b81bc83

2 files changed

Lines changed: 307 additions & 5 deletions

File tree

‎livekit-agents/livekit/agents/voice/audio_recognition.py‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -290,6 +290,9 @@ def __init__(
290290
self._final_transcript_confidence: list[float] = []
291291
self._audio_transcript = ""
292292
self._audio_interim_transcript = ""
293+
# set when a MultiSpeakerAdapter attributed the buffered interim to the
294+
# primary speaker: a suppressed background segment cannot own that text
295+
self._audio_interim_from_primary = False
293296
# used for STTs that support preflight mode, so it could start preemptive generation earlier
294297
self._audio_preflight_transcript = ""
295298
self._last_language: LanguageCode | None = None
@@ -1016,6 +1019,7 @@ def _detach_turn_detector(self) -> _StreamingTurnDetectorStream | None:
10161019
def _clear_user_turn(self) -> None:
10171020
self._audio_transcript = ""
10181021
self._audio_interim_transcript = ""
1022+
self._audio_interim_from_primary = False
10191023
self._audio_preflight_transcript = ""
10201024
self._final_transcript_confidence = []
10211025
self._last_final_transcript_time = None
@@ -1108,6 +1112,7 @@ async def _commit_user_turn() -> None:
11081112

11091113
transcript = self._audio_transcript
11101114
self._audio_interim_transcript = ""
1115+
self._audio_interim_from_primary = False
11111116
chat_ctx = self._hooks.retrieve_chat_ctx().copy()
11121117
self._run_eou_detection(
11131118
chat_ctx,
@@ -1243,8 +1248,18 @@ def _process_stt_event(self, ev: stt.SpeechEvent) -> None:
12431248
# The transport cannot carry a wire-level clear, so drop the
12441249
# buffered text here: it must not stay in _current_transcript
12451250
# (min_words) or be appended by commit_user_turn.
1246-
self._audio_interim_transcript = ""
1247-
self._audio_preflight_transcript = ""
1251+
#
1252+
# The empty final does not identify a segment, so only drop
1253+
# interim text that can belong to the suppressed speaker. An
1254+
# interim the adapter attributed to the primary speaker is a
1255+
# different segment's in-progress words and must survive for
1256+
# commit_user_turn.
1257+
if not self._audio_interim_from_primary:
1258+
self._audio_interim_transcript = ""
1259+
self._audio_preflight_transcript = ""
1260+
# the suppressed segment still finalized in the wrapped STT, so
1261+
# the manual-commit freshness check must treat this as a final
1262+
self._last_final_transcript_time = time.time()
12481263
return
12491264

12501265
self._hooks.on_final_transcript(
@@ -1270,6 +1285,7 @@ def _process_stt_event(self, ev: stt.SpeechEvent) -> None:
12701285
self._final_transcript_confidence.append(confidence)
12711286
transcript_changed = self._audio_transcript != self._audio_preflight_transcript
12721287
self._audio_interim_transcript = ""
1288+
self._audio_interim_from_primary = False
12731289
self._audio_preflight_transcript = ""
12741290

12751291
if use_stt_speaking_time:
@@ -1326,6 +1342,7 @@ def _process_stt_event(self, ev: stt.SpeechEvent) -> None:
13261342
# preflight transcript includes all pre-committed transcripts (including final transcript from the previous STT run)
13271343
self._audio_preflight_transcript = (self._audio_transcript + " " + transcript).lstrip()
13281344
self._audio_interim_transcript = transcript
1345+
self._audio_interim_from_primary = ev.alternatives[0].is_primary_speaker is True
13291346

13301347
if use_stt_speaking_time:
13311348
self._last_speaking_time = stt_last_speaking_time
@@ -1348,6 +1365,7 @@ def _process_stt_event(self, ev: stt.SpeechEvent) -> None:
13481365
else None,
13491366
)
13501367
self._audio_interim_transcript = ev.alternatives[0].text
1368+
self._audio_interim_from_primary = ev.alternatives[0].is_primary_speaker is True
13511369

13521370
elif ev.type == stt.SpeechEventType.END_OF_SPEECH and self._turn_detection_mode == "stt":
13531371
with tracer.use_span(self._ensure_user_turn_span()):

‎tests/test_multi_speaker_interim_clear.py‎

Lines changed: 287 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,13 @@
1313
the suppressed segment. Otherwise those stale background words stay in
1414
``_current_transcript`` - where they count toward the ``min_words`` interruption
1515
gate - and ``commit_user_turn`` appends them to the user turn.
16+
17+
The empty final carries no segment identity, so recognition only drops interim
18+
text that can belong to the suppressed segment: a speaker-less (Deepgram-style)
19+
interim is dropped, while an interim the adapter attributed to the primary
20+
speaker is another segment's in-progress words and survives for the commit. The
21+
empty final also refreshes the manual-commit freshness anchor: the wrapped STT
22+
did finalize, so a commit must not wait for yet another final.
1623
"""
1724

1825
from __future__ import annotations
@@ -23,12 +30,24 @@
2330

2431
import pytest
2532

33+
from livekit import rtc
2634
from livekit.agents import stt
2735
from livekit.agents.language import LanguageCode
36+
from livekit.agents.stt import MultiSpeakerAdapter, RecognizeStream
37+
from livekit.agents.stt.multi_speaker_adapter import PrimarySpeakerDetectionOptions
38+
from livekit.agents.types import (
39+
DEFAULT_API_CONNECT_OPTIONS,
40+
NOT_GIVEN,
41+
APIConnectOptions,
42+
NotGivenOr,
43+
)
44+
from livekit.agents.utils import AudioBuffer
2845
from livekit.agents.voice.audio_recognition import AudioRecognition
2946

3047
pytestmark = pytest.mark.unit
3148

49+
_CONN = APIConnectOptions(max_retry=0, timeout=5.0)
50+
3251

3352
def _make_recognition() -> AudioRecognition:
3453
"""Wire the attributes the transcript path of ``_on_stt_event`` touches."""
@@ -57,6 +76,7 @@ def _make_recognition() -> AudioRecognition:
5776
ar._end_of_turn_task = None
5877
ar._audio_transcript = ""
5978
ar._audio_interim_transcript = ""
79+
ar._audio_interim_from_primary = False
6080
ar._audio_preflight_transcript = ""
6181
ar._final_transcript_confidence = []
6282
ar._final_transcript_received = asyncio.Event()
@@ -153,12 +173,276 @@ async def test_empty_final_keeps_background_text_out_of_commit_user_turn() -> No
153173
ar = _make_recognition()
154174

155175
await _feed_adapter_sequence(ar)
156-
# the empty final just arrived, so the transcript is fresh and the commit
157-
# does not need to flush the STT and wait for another final
158-
ar._last_final_transcript_time = time.time()
159176

160177
committed = await ar._commit_user_turn(
161178
audio_detached=False, transcript_timeout=0.01, stt_flush_duration=0.0
162179
)
163180

164181
assert committed == "hello"
182+
183+
184+
async def test_commit_after_empty_final_does_not_wait_for_another_final() -> None:
185+
"""An empty final satisfies the manual-commit freshness check.
186+
187+
The commit waits for a new final only when the last one is older than 0.5s,
188+
because it then assumes the STT still holds unfinalized audio. A suppressed
189+
background segment produces a real final, so once the empty final arrives
190+
the STT buffer is flushed: waiting for yet another final stalls the commit
191+
until ``transcript_timeout``.
192+
"""
193+
ar = _make_recognition()
194+
195+
await ar._on_stt_event(_final_transcript("hello", speaker_id="S0"))
196+
await asyncio.sleep(0.55) # the last final is now outside the 0.5s window
197+
await ar._on_stt_event(_interim_transcript("background chatter"))
198+
await ar._on_stt_event(_empty_final_transcript())
199+
200+
started = time.monotonic()
201+
committed = await ar._commit_user_turn(
202+
audio_detached=False, transcript_timeout=1.0, stt_flush_duration=0.0
203+
)
204+
elapsed = time.monotonic() - started
205+
206+
assert committed == "hello"
207+
assert elapsed < 0.5, f"commit waited {elapsed:.2f}s for a final that already arrived"
208+
209+
210+
# --- the empty final through the real MultiSpeakerAdapter -------------------
211+
#
212+
# The tests above feed the event sequence directly. The adapter's empty final is
213+
# not addressed to a particular interim, so these drive the real adapter to pin
214+
# down which interim it is allowed to clear.
215+
216+
_SAMPLE_RATE = 1000
217+
_SAMPLES_PER_FRAME = 100 # 100 ms, matching frame_size_ms below
218+
219+
220+
def _tone_frame(amplitude: int) -> rtc.AudioFrame:
221+
"""A mono int16 frame whose every sample equals ``amplitude``."""
222+
return rtc.AudioFrame(
223+
data=amplitude.to_bytes(2, "little", signed=True) * _SAMPLES_PER_FRAME,
224+
sample_rate=_SAMPLE_RATE,
225+
num_channels=1,
226+
samples_per_channel=_SAMPLES_PER_FRAME,
227+
)
228+
229+
230+
def _timed_transcript(
231+
event_type: stt.SpeechEventType,
232+
text: str,
233+
*,
234+
speaker_id: str | None,
235+
start_time: float,
236+
end_time: float,
237+
) -> stt.SpeechEvent:
238+
return stt.SpeechEvent(
239+
type=event_type,
240+
alternatives=[
241+
stt.SpeechData(
242+
language=LanguageCode("en"),
243+
text=text,
244+
confidence=0.9,
245+
speaker_id=speaker_id,
246+
start_time=start_time,
247+
end_time=end_time,
248+
)
249+
],
250+
)
251+
252+
253+
class _ScriptedDiarizingSTT(stt.STT):
254+
"""Diarization-capable STT that replays one event sequence per stream.
255+
256+
The adapter runs RMS-based primary-speaker detection over the audio it
257+
forwards, so the scripted stream waits for ``expected_frames`` frames before
258+
emitting, matching the order a provider produces: audio first, then the
259+
results for it.
260+
"""
261+
262+
def __init__(self, events: list[stt.SpeechEvent], *, expected_frames: int) -> None:
263+
super().__init__(
264+
capabilities=stt.STTCapabilities(streaming=True, interim_results=True, diarization=True)
265+
)
266+
self._scripted_events = events
267+
self._expected_frames = expected_frames
268+
269+
async def _recognize_impl(
270+
self,
271+
buffer: AudioBuffer,
272+
*,
273+
language: NotGivenOr[str] = NOT_GIVEN,
274+
conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
275+
) -> stt.SpeechEvent:
276+
raise NotImplementedError("this fake only supports streaming")
277+
278+
def stream(
279+
self,
280+
*,
281+
language: NotGivenOr[str] = NOT_GIVEN,
282+
conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
283+
) -> RecognizeStream:
284+
return _ScriptedDiarizingStream(
285+
stt=self,
286+
events=self._scripted_events,
287+
expected_frames=self._expected_frames,
288+
conn_options=conn_options,
289+
)
290+
291+
292+
class _ScriptedDiarizingStream(RecognizeStream):
293+
def __init__(
294+
self,
295+
*,
296+
stt: stt.STT,
297+
events: list[stt.SpeechEvent],
298+
expected_frames: int,
299+
conn_options: APIConnectOptions,
300+
) -> None:
301+
super().__init__(stt=stt, conn_options=conn_options)
302+
self._scripted_events = events
303+
self._expected_frames = expected_frames
304+
305+
async def _run(self) -> None:
306+
received = 0
307+
while received < self._expected_frames:
308+
if isinstance(await self._input_ch.recv(), rtc.AudioFrame):
309+
received += 1
310+
for event in self._scripted_events:
311+
self._event_ch.send_nowait(event)
312+
313+
314+
def _diarizing_adapter(events: list[stt.SpeechEvent], *, frames: int) -> MultiSpeakerAdapter:
315+
return MultiSpeakerAdapter(
316+
stt=_ScriptedDiarizingSTT(events, expected_frames=frames),
317+
suppress_background_speaker=True,
318+
primary_detection_options=PrimarySpeakerDetectionOptions(
319+
frame_size_ms=100, min_rms_samples=1
320+
),
321+
)
322+
323+
324+
async def _adapter_events(
325+
adapter: MultiSpeakerAdapter, frames: list[rtc.AudioFrame], count: int
326+
) -> list[stt.SpeechEvent]:
327+
stream = adapter.stream(conn_options=_CONN)
328+
for frame in frames:
329+
stream.push_frame(frame)
330+
events: list[stt.SpeechEvent] = []
331+
async for event in stream:
332+
events.append(event)
333+
if len(events) == count:
334+
break
335+
await stream.aclose()
336+
return events
337+
338+
339+
async def test_primary_interim_survives_a_suppressed_background_final() -> None:
340+
"""A background final must not drop the primary speaker's interim.
341+
342+
Diarizing providers that attribute interim results (AssemblyAI's turns carry
343+
``speaker_label``) let the adapter forward a primary interim. A quieter
344+
background final right after it is suppressed and replaced by the empty
345+
final, which carries no segment identity: clearing the primary interim here
346+
would make a manual commit lose speech that has not been finalized yet.
347+
"""
348+
events = [
349+
_timed_transcript(
350+
stt.SpeechEventType.FINAL_TRANSCRIPT,
351+
"hello",
352+
speaker_id="S0",
353+
start_time=0.0,
354+
end_time=0.2,
355+
),
356+
_timed_transcript(
357+
stt.SpeechEventType.INTERIM_TRANSCRIPT,
358+
"book a ticket",
359+
speaker_id="S0",
360+
start_time=0.2,
361+
end_time=0.3,
362+
),
363+
# quiet background speaker: the detector keeps S0 as primary and the
364+
# adapter suppresses this final, emitting the empty final instead
365+
_timed_transcript(
366+
stt.SpeechEventType.FINAL_TRANSCRIPT,
367+
"background chatter",
368+
speaker_id="S1",
369+
start_time=0.3,
370+
end_time=0.4,
371+
),
372+
]
373+
adapter = _diarizing_adapter(events, frames=4)
374+
frames = [_tone_frame(3000), _tone_frame(3000), _tone_frame(50), _tone_frame(50)]
375+
376+
emitted = await _adapter_events(adapter, frames, count=3)
377+
378+
assert [event.type for event in emitted] == [
379+
stt.SpeechEventType.FINAL_TRANSCRIPT,
380+
stt.SpeechEventType.INTERIM_TRANSCRIPT,
381+
stt.SpeechEventType.FINAL_TRANSCRIPT,
382+
]
383+
assert emitted[0].alternatives[0].text == "hello"
384+
assert emitted[1].alternatives[0].text == "book a ticket"
385+
assert emitted[2].alternatives[0].text == "" # the adapter's synthesized clear
386+
387+
ar = _make_recognition()
388+
for event in emitted:
389+
await ar._on_stt_event(event)
390+
391+
assert ar._audio_interim_transcript == "book a ticket"
392+
committed = await ar._commit_user_turn(
393+
audio_detached=False, transcript_timeout=0.01, stt_flush_duration=0.0
394+
)
395+
assert committed == "hello book a ticket"
396+
397+
398+
async def test_speakerless_interim_is_still_cleared_by_the_empty_final() -> None:
399+
"""Control: the Deepgram-style case this fix targets is still cleared.
400+
401+
Deepgram reports interims without a speaker id, so they bypass the adapter's
402+
suppression and reach recognition. The suppressed segment's final arrives as
403+
the empty final, and the buffered interim is that same segment's partial
404+
text: it must not stay in the user turn.
405+
"""
406+
events = [
407+
_timed_transcript(
408+
stt.SpeechEventType.FINAL_TRANSCRIPT,
409+
"hello",
410+
speaker_id="S0",
411+
start_time=0.0,
412+
end_time=0.2,
413+
),
414+
_timed_transcript(
415+
stt.SpeechEventType.INTERIM_TRANSCRIPT,
416+
"background chatter",
417+
speaker_id=None,
418+
start_time=0.2,
419+
end_time=0.3,
420+
),
421+
_timed_transcript(
422+
stt.SpeechEventType.FINAL_TRANSCRIPT,
423+
"background chatter",
424+
speaker_id="S1",
425+
start_time=0.3,
426+
end_time=0.4,
427+
),
428+
]
429+
adapter = _diarizing_adapter(events, frames=4)
430+
frames = [_tone_frame(3000), _tone_frame(3000), _tone_frame(50), _tone_frame(50)]
431+
432+
emitted = await _adapter_events(adapter, frames, count=3)
433+
assert [event.alternatives[0].text for event in emitted] == [
434+
"hello",
435+
"background chatter",
436+
"",
437+
]
438+
439+
ar = _make_recognition()
440+
for event in emitted:
441+
await ar._on_stt_event(event)
442+
443+
assert ar._audio_interim_transcript == ""
444+
assert ar._current_transcript == "hello"
445+
committed = await ar._commit_user_turn(
446+
audio_detached=False, transcript_timeout=0.01, stt_flush_duration=0.0
447+
)
448+
assert committed == "hello"

0 commit comments

Comments
 (0)