Skip to content

Commit 881ce50

Browse files
perf(huddle): stream Pocket TTS decoder chunks
Signed-off-by: John Tennant <[email protected]>
1 parent 6186fc2 commit 881ce50

16 files changed

Lines changed: 933 additions & 467 deletions

File tree

‎crates/buzz-voice/src/lib.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33
pub mod pocket;
44

55
pub use pocket::{
6-
load_text_to_speech, load_voice_style, PocketTts, VoiceStyle, DEFAULT_VOICE, SAMPLE_RATE,
7-
VOICE_FILE_EXT,
6+
load_text_to_speech, load_voice_style, PocketTts, SynthesisOutcome, VoiceStyle, DEFAULT_VOICE,
7+
SAMPLE_RATE, VOICE_FILE_EXT,
88
};
99

1010
/// One immutable artifact required by the April Pocket bundle.

‎crates/buzz-voice/src/pocket.rs‎

Lines changed: 137 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ mod pocket_april;
2323
#[path = "pocket_models.rs"]
2424
mod pocket_models;
2525

26-
use pocket_april::{prepare_april_prompt, AprilPocketTts};
26+
use pocket_april::{prepare_april_prompt, AprilPocketTts, AprilSynthesisOutcome};
2727
pub(crate) use pocket_models::{
2828
april_model_info, PocketModelArtifact, APRIL_BUNDLE_ID, APRIL_MODEL_ID, APRIL_MODEL_REVISION,
2929
};
@@ -68,6 +68,16 @@ pub struct PocketTts {
6868
inner: Mutex<AprilPocketTts>,
6969
}
7070

71+
/// Result of a callback-driven Pocket synthesis request.
72+
#[derive(Debug, Clone, PartialEq)]
73+
pub enum SynthesisOutcome {
74+
/// Synthesis finished and contains the same PCM exposed cumulatively to
75+
/// the callback.
76+
Complete(Vec<f32>),
77+
/// The callback requested cancellation before synthesis completed.
78+
Interrupted,
79+
}
80+
7181
/// Load Buzz Desktop's pinned April INT8 model.
7282
pub fn load_text_to_speech(model_dir: &str) -> Result<PocketTts, String> {
7383
let dir = PathBuf::from(model_dir);
@@ -104,27 +114,63 @@ impl PocketTts {
104114
/// Pocket detects language from text and this model uses one synthesis
105115
/// step, so `_lang` and `_steps` intentionally do not affect output.
106116
pub fn synth_chunk(
117+
&self,
118+
text: &str,
119+
lang: &str,
120+
style: &VoiceStyle,
121+
steps: usize,
122+
) -> Result<Vec<f32>, String> {
123+
match self.synth_chunk_streaming(text, lang, style, steps, |_, _| true)? {
124+
SynthesisOutcome::Complete(samples) => Ok(samples),
125+
SynthesisOutcome::Interrupted => Ok(Vec::new()),
126+
}
127+
}
128+
129+
/// Synthesize text while reporting cumulative PCM as decoder blocks finish.
130+
///
131+
/// Callback sample buffers contain all PCM produced for this call so far.
132+
/// Their lengths never decrease, but equal lengths are allowed while the
133+
/// engine advances between internal model-safe text chunks. Returning
134+
/// `false` interrupts synthesis before the next decoder block.
135+
pub fn synth_chunk_streaming<F>(
107136
&self,
108137
text: &str,
109138
_lang: &str,
110139
style: &VoiceStyle,
111140
_steps: usize,
112-
) -> Result<Vec<f32>, String> {
141+
mut callback: F,
142+
) -> Result<SynthesisOutcome, String>
143+
where
144+
F: FnMut(&[f32], f32) -> bool,
145+
{
113146
let Some(prepared) = prepare_april_prompt(text) else {
114-
return Ok(Vec::new());
147+
return Ok(SynthesisOutcome::Complete(Vec::new()));
115148
};
116149
let mut engine = self
117150
.inner
118151
.lock()
119152
.map_err(|_| "Pocket TTS engine lock poisoned".to_string())?;
120153
let chunks = engine.split_prompt(&prepared)?;
154+
let chunk_count = chunks.len();
121155
let mut samples = Vec::new();
122-
for chunk in chunks {
156+
for (chunk_index, chunk) in chunks.into_iter().enumerate() {
157+
if chunk_index > 0 && !callback(&samples, chunk_index as f32 / chunk_count as f32) {
158+
return Ok(SynthesisOutcome::Interrupted);
159+
}
123160
let prepared = prepare_april_prompt(&chunk)
124161
.ok_or_else(|| "Pocket TTS prompt chunk became empty".to_string())?;
125-
samples.extend(engine.synth_chunk(&prepared, style)?);
162+
let outcome = engine.synth_chunk_streaming(&prepared, style, |block, progress| {
163+
samples.extend_from_slice(block);
164+
callback(
165+
&samples,
166+
(chunk_index as f32 + progress) / chunk_count as f32,
167+
)
168+
})?;
169+
if matches!(outcome, AprilSynthesisOutcome::Interrupted) {
170+
return Ok(SynthesisOutcome::Interrupted);
171+
}
126172
}
127-
Ok(samples)
173+
Ok(SynthesisOutcome::Complete(samples))
128174
}
129175
}
130176

@@ -163,4 +209,89 @@ mod tests {
163209
assert!(samples.iter().all(|sample| sample.is_finite()));
164210
assert!(samples.iter().any(|sample| sample.abs() > 1.0e-6));
165211
}
212+
213+
#[test]
214+
#[ignore = "requires BUZZ_POCKET_TEST_MODEL_DIR"]
215+
fn production_streaming_callbacks_are_cumulative_across_model_chunks() {
216+
let dir = std::env::var("BUZZ_POCKET_TEST_MODEL_DIR")
217+
.expect("set BUZZ_POCKET_TEST_MODEL_DIR to an April INT8 model directory");
218+
let engine = load_text_to_speech(&dir).expect("load April INT8 engine");
219+
let style = load_voice_style(&Path::new(&dir).join("reference_sample.wav"))
220+
.expect("load reference voice");
221+
engine
222+
.synth_chunk("Warm up.", "en", &style, 1)
223+
.expect("warm production engine");
224+
let text = "And sometimes, when I am certain the reader is rested, I will engage him with a sentence of considerable length, a sentence that burns with energy and builds with all the impetus of a crescendo, the roll of the drums, the crash of the cymbals–sounds that say listen to this, it is important.";
225+
let mut reconstructed = Vec::new();
226+
let mut previous_len = 0;
227+
let mut saw_equal_repeat = false;
228+
let mut callback_count = 0;
229+
let mut first_callback = None;
230+
let started = std::time::Instant::now();
231+
232+
let outcome = engine
233+
.synth_chunk_streaming(text, "en", &style, 1, |cumulative, _| {
234+
callback_count += 1;
235+
first_callback.get_or_insert_with(|| started.elapsed());
236+
assert!(cumulative.len() >= previous_len);
237+
saw_equal_repeat |= cumulative.len() == previous_len;
238+
reconstructed.extend_from_slice(&cumulative[previous_len..]);
239+
previous_len = cumulative.len();
240+
true
241+
})
242+
.expect("stream through the production API");
243+
let SynthesisOutcome::Complete(samples) = outcome else {
244+
panic!("uninterrupted synthesis must complete");
245+
};
246+
let total = started.elapsed();
247+
let first_callback = first_callback.expect("decoder must produce a callback");
248+
let audio_duration =
249+
std::time::Duration::from_secs_f64(samples.len() as f64 / SAMPLE_RATE as f64);
250+
eprintln!(
251+
"first_callback_ms={:.1} total_ms={:.1} audio_seconds={:.3} rtf={:.3} callbacks={callback_count}",
252+
first_callback.as_secs_f64() * 1000.0,
253+
total.as_secs_f64() * 1000.0,
254+
audio_duration.as_secs_f64(),
255+
total.as_secs_f64() / audio_duration.as_secs_f64(),
256+
);
257+
258+
assert!(saw_equal_repeat);
259+
assert_eq!(reconstructed, samples);
260+
}
261+
262+
#[test]
263+
#[ignore = "requires BUZZ_POCKET_TEST_MODEL_DIR"]
264+
fn production_streaming_callback_interrupts_after_first_decoder_block() {
265+
let dir = std::env::var("BUZZ_POCKET_TEST_MODEL_DIR")
266+
.expect("set BUZZ_POCKET_TEST_MODEL_DIR to an April INT8 model directory");
267+
let engine = load_text_to_speech(&dir).expect("load April INT8 engine");
268+
let style = load_voice_style(&Path::new(&dir).join("reference_sample.wav"))
269+
.expect("load reference voice");
270+
engine
271+
.synth_chunk("Warm up.", "en", &style, 1)
272+
.expect("warm production engine");
273+
let mut callback_at = None;
274+
let started = std::time::Instant::now();
275+
276+
let outcome = engine
277+
.synth_chunk_streaming(
278+
"This sentence is long enough to require more than one decoder block.",
279+
"en",
280+
&style,
281+
1,
282+
|_, _| {
283+
callback_at = Some(started.elapsed());
284+
false
285+
},
286+
)
287+
.expect("interrupt production streaming");
288+
let callback_at = callback_at.expect("decoder must produce a callback");
289+
let cancellation_latency = started.elapsed().saturating_sub(callback_at);
290+
eprintln!(
291+
"callback_to_cancel_return_ms={:.1}",
292+
cancellation_latency.as_secs_f64() * 1000.0
293+
);
294+
295+
assert!(matches!(outcome, SynthesisOutcome::Interrupted));
296+
}
166297
}

‎crates/buzz-voice/src/pocket_april.rs‎

Lines changed: 28 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,11 @@ pub(crate) struct AprilPocketTts {
108108
cached_voice: Option<CachedVoice>,
109109
}
110110

111+
pub(crate) enum AprilSynthesisOutcome {
112+
Complete,
113+
Interrupted,
114+
}
115+
111116
#[derive(Debug, Clone, PartialEq)]
112117
pub(crate) struct AprilPreparedPrompt {
113118
pub(crate) text: String,
@@ -304,11 +309,15 @@ impl AprilPocketTts {
304309
.collect()
305310
}
306311

307-
pub(crate) fn synth_chunk(
312+
pub(crate) fn synth_chunk_streaming<F>(
308313
&mut self,
309314
prepared: &AprilPreparedPrompt,
310315
style: &VoiceStyle,
311-
) -> Result<Vec<f32>, String> {
316+
callback: F,
317+
) -> Result<AprilSynthesisOutcome, String>
318+
where
319+
F: FnMut(&[f32], f32) -> bool,
320+
{
312321
let voice_embeddings = self.voice_embeddings(style)?;
313322
let mut flow_state = self.condition_voice(&voice_embeddings)?;
314323
let token_ids = self
@@ -321,7 +330,7 @@ impl AprilPocketTts {
321330
.map(i64::from)
322331
.collect::<Vec<_>>();
323332
if token_ids.is_empty() {
324-
return Ok(Vec::new());
333+
return Ok(AprilSynthesisOutcome::Complete);
325334
}
326335
if token_ids.len() > self.bundle.max_token_per_chunk {
327336
return Err(format!(
@@ -337,7 +346,7 @@ impl AprilPocketTts {
337346
let max_frames = estimate_max_frames(token_count, self.bundle.frame_rate);
338347
let latents =
339348
self.generate_latents(max_frames, prepared.frames_after_eos, &mut flow_state)?;
340-
self.decode_latents(&latents)
349+
self.decode_latents(&latents, callback)
341350
}
342351

343352
fn prepared_token_count(&self, text: &str) -> Result<usize, String> {
@@ -597,9 +606,16 @@ impl AprilPocketTts {
597606
Ok(latents)
598607
}
599608

600-
fn decode_latents(&mut self, latents: &[f32]) -> Result<Vec<f32>, String> {
609+
fn decode_latents<F>(
610+
&mut self,
611+
latents: &[f32],
612+
mut callback: F,
613+
) -> Result<AprilSynthesisOutcome, String>
614+
where
615+
F: FnMut(&[f32], f32) -> bool,
616+
{
601617
if latents.is_empty() {
602-
return Ok(Vec::new());
618+
return Ok(AprilSynthesisOutcome::Complete);
603619
}
604620
if !latents.len().is_multiple_of(self.bundle.latent_dim) {
605621
return Err(format!(
@@ -610,7 +626,6 @@ impl AprilPocketTts {
610626
}
611627
let frame_count = latents.len() / self.bundle.latent_dim;
612628
let mut state = initialize_state(&self.bundle.mimi_state_manifest)?;
613-
let mut audio = Vec::new();
614629

615630
for start in (0..frame_count).step_by(DECODER_CHUNK_FRAMES) {
616631
let end = (start + DECODER_CHUNK_FRAMES).min(frame_count);
@@ -630,11 +645,14 @@ impl AprilPocketTts {
630645
let samples = outputs[0]
631646
.try_extract_tensor::<f32>()
632647
.map_err(ort_error("extract Mimi audio"))?
633-
.1;
634-
audio.extend_from_slice(samples);
648+
.1
649+
.to_vec();
635650
replace_state_from_outputs(&mut state, &mut outputs)?;
651+
if !callback(&samples, end as f32 / frame_count as f32) {
652+
return Ok(AprilSynthesisOutcome::Interrupted);
653+
}
636654
}
637-
Ok(audio)
655+
Ok(AprilSynthesisOutcome::Complete)
638656
}
639657
}
640658

‎desktop/src-tauri/src/huddle/mod.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,9 @@
1717
//! `tts_starting` sentinels prevent TOCTOU races during the ~200ms window.
1818
//! 3. **`agent_pubkeys` has its own inner `Arc<Mutex>`** — the transcription task
1919
//! clones the `Arc` and reads at post time without the outer lock.
20-
//! 4. **Atomics for cross-thread signaling** — `tts_active`, `tts_cancel`,
21-
//! `ptt_active`, `session_generation` are shared with pipeline worker threads.
20+
//! 4. **Atomics for cross-thread signaling** — `tts_active`,
21+
//! `tts_synthesizing`, `tts_cancel`, `ptt_active`, `session_generation` are
22+
//! shared with pipeline worker threads.
2223
//! 5. **Pipeline teardown extracts handles before dropping** — `teardown_huddle`
2324
//! takes `stt_pipeline`/`tts_pipeline` out of the lock, then calls `shutdown()`
2425
//! and drops them outside the lock (thread joins can block ~200ms).

‎desktop/src-tauri/src/huddle/pipeline.rs‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,15 @@ pub(crate) async fn maybe_start_stt_pipeline(
113113
// transcription task's next POST sees a stale generation and exits.
114114
// Take the old pipeline OUT of the lock before dropping — Drop joins
115115
// the worker thread (~200ms) and must not block under the mutex.
116-
let (tts_active, tts_cancel, agent_pubkeys_arc, session_gen, ptt_active_for_stt, old_stt) = {
116+
let (
117+
tts_active,
118+
tts_synthesizing,
119+
tts_cancel,
120+
agent_pubkeys_arc,
121+
session_gen,
122+
ptt_active_for_stt,
123+
old_stt,
124+
) = {
117125
let mut hs = state.huddle()?;
118126
// Invalidate any existing transcription task before replacing the pipeline.
119127
if hs.stt_pipeline.is_some() {
@@ -130,6 +138,7 @@ pub(crate) async fn maybe_start_stt_pipeline(
130138
};
131139
(
132140
Arc::clone(&hs.tts_active),
141+
Arc::clone(&hs.tts_synthesizing),
133142
Some(Arc::clone(&hs.tts_cancel)),
134143
Arc::clone(&hs.agent_pubkeys),
135144
Arc::clone(&hs.session_generation),
@@ -141,7 +150,13 @@ pub(crate) async fn maybe_start_stt_pipeline(
141150
drop(old_stt);
142151

143152
let constructed = tokio::task::spawn_blocking(move || {
144-
stt::SttPipeline::new(model_dir, tts_active, tts_cancel, ptt_active_for_stt)
153+
stt::SttPipeline::new(
154+
model_dir,
155+
tts_active,
156+
tts_synthesizing,
157+
tts_cancel,
158+
ptt_active_for_stt,
159+
)
145160
})
146161
.await;
147162
let (pipeline, text_rx) = match constructed {
@@ -198,7 +213,7 @@ pub(crate) async fn maybe_start_tts_pipeline(state: &AppState) -> Result<bool, S
198213
// Atomically check preconditions and claim the construction slot.
199214
// The sentinel prevents a second caller from starting construction
200215
// while we're building outside the lock.
201-
let (tts_active, tts_cancel, tts_starting) = {
216+
let (tts_active, tts_synthesizing, tts_cancel, tts_starting) = {
202217
let hs = state.huddle()?;
203218
if hs.tts_pipeline.is_some() {
204219
return Ok(false);
@@ -211,6 +226,7 @@ pub(crate) async fn maybe_start_tts_pipeline(state: &AppState) -> Result<bool, S
211226
}
212227
(
213228
Arc::clone(&hs.tts_active),
229+
Arc::clone(&hs.tts_synthesizing),
214230
Arc::clone(&hs.tts_cancel),
215231
Arc::clone(&hs.tts_starting),
216232
)
@@ -238,6 +254,7 @@ pub(crate) async fn maybe_start_tts_pipeline(state: &AppState) -> Result<bool, S
238254
tts::TtsPipeline::new_with_voice(
239255
model_dir,
240256
tts_active,
257+
tts_synthesizing,
241258
tts_cancel,
242259
&initial_voice,
243260
output_device,

‎desktop/src-tauri/src/huddle/playout.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ pub(crate) async fn run_playout_recv_loop(
137137
app_handle: Option<tauri::AppHandle>,
138138
initial_peers: Vec<(u8, String)>,
139139
tts_active: Arc<AtomicBool>,
140+
tts_synthesizing: Arc<AtomicBool>,
140141
tts_cancel: Arc<AtomicBool>,
141142
) {
142143
use rodio::buffer::SamplesBuffer;
@@ -256,7 +257,10 @@ pub(crate) async fn run_playout_recv_loop(
256257
}
257258

258259
// TTS interrupt frame counter — reset on TTS rising edge.
259-
let tts_now = tts_active.load(Ordering::Acquire);
260+
let tts_now = super::tts::is_tts_interruptible(
261+
tts_active.as_ref(),
262+
tts_synthesizing.as_ref(),
263+
);
260264
if tts_now && !tts_was_active {
261265
frame_counts.clear();
262266
last_frame_reset = tokio::time::Instant::now();

0 commit comments

Comments
 (0)