Skip to content

Commit cf32dac

Browse files
author
npub1qyvc0c5kl4gqv2fd97fsk46tu378sqgy35vc83rvgfwne90sel7s0ed67d
committed
refactor(voice): extract reusable Pocket primitives + add Pocket voice settings
Combined landing of #2467 (extract buzz-voice crate) and #3208 (Pocket voice settings), replayed onto main after #3266 and #3180 merged. The original stacked PRs could not land as-is: the repo is squash-only with delete-branch-on-merge, so #2467 was auto-closed when its base branch was deleted, and squash merges severed the ancestry GitHub uses for conflict detection. Content is byte-identical to the blessed jt/buzz-voice-refactor branch (93029c5, tree 6729e0e) except for the three-file interaction with #3180 (huddle/mod.rs, huddle/state.rs, e2eBridge.ts), resolved here as the union of both sides: - huddle/mod.rs: keep #3180's pipeline re-exports (check_pipeline_hotstart, start_auto_enabled_transcription, PostConnectOutcome) alongside #3208's agent_tts_routing imports and await_inflight_tts_start. - huddle/state.rs: reset_preserving_generation preserves both #3180's huddle_generation and #3208's tts_enabled; both test sets merged into the single tests module (fixes E0428 from textual union). - e2eBridge.ts: both switch arms kept — #3180's huddle mock commands and #3208's TTS settings/voice-registry commands; no duplicate case labels. Verified at this tree: cargo test 2047+3/0 (14 ignored), clippy -D warnings clean, cargo fmt clean, tsc clean, pnpm test 3885/0, lint clean. Co-authored-by: npub1qyvc0c5kl4gqv2fd97fsk46tu378sqgy35vc83rvgfwne90sel7s0ed67d <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> Signed-off-by: npub1qyvc0c5kl4gqv2fd97fsk46tu378sqgy35vc83rvgfwne90sel7s0ed67d <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz>
1 parent 4632c55 commit cf32dac

63 files changed

Lines changed: 5017 additions & 696 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Cargo.lock‎

Lines changed: 538 additions & 64 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ members = [
2626
"crates/buzz-pair-relay",
2727
"crates/buzz-relay-mesh",
2828
"crates/buzz-dev-mcp",
29+
"crates/buzz-voice",
2930
"examples/countdown-bot",
3031
]
3132
exclude = ["desktop/src-tauri"]

‎Justfile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -276,6 +276,7 @@ test-unit:
276276
#!/usr/bin/env bash
277277
if command -v cargo-nextest &>/dev/null; then
278278
cargo nextest run -p buzz-core -p buzz-auth --lib
279+
cargo nextest run -p buzz-voice --lib
279280
cargo nextest run -p buzz-cli
280281
# buzz-db migrator/lint tests: pure SQL-parsing unit tests (no infra).
281282
# They guard the embedded-migrator invariant (exactly the consolidated

‎crates/buzz-voice/Cargo.toml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
[package]
2+
name = "buzz-voice"
3+
version.workspace = true
4+
edition.workspace = true
5+
rust-version.workspace = true
6+
license.workspace = true
7+
repository.workspace = true
8+
description = "Reusable local voice primitives for Buzz"
9+
10+
[dependencies]
11+
ort = { version = "=2.0.0-rc.12", default-features = false, features = ["api-24", "ndarray", "std"] }
12+
ort-sys = { version = "=2.0.0-rc.12", features = ["disable-linking"] }
13+
rand = "0.10"
14+
sentencepiece-model = "0.1"
15+
serde = { version = "1", features = ["derive"] }
16+
serde_json = "1"
17+
sherpa-onnx = "1.12"
18+
tokenizers = { version = "0.22", default-features = false, features = ["fancy-regex"] }

‎crates/buzz-voice/src/lib.rs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
//! Reusable local voice primitives for Buzz.
2+
3+
pub mod pocket;
4+
5+
pub use pocket::{
6+
april_model_info, load_text_to_speech, load_voice_style, PocketModelInfo, PocketTts,
7+
VoiceStyle, DEFAULT_VOICE, SAMPLE_RATE, VOICE_FILE_EXT,
8+
};
9+
10+
/// One immutable artifact required by the April Pocket bundle.
11+
///
12+
/// `filename` is the bundle-relative file name, `sha256` pins its contents,
13+
/// `size_bytes` supports download progress and validation, and `quantized`
14+
/// identifies the INT8 components.
15+
pub type PocketModelArtifact = pocket::PocketModelArtifact;
16+
17+
/// Language bundle selected from the pinned export.
18+
pub const APRIL_BUNDLE_ID: &str = pocket::APRIL_BUNDLE_ID;
19+
/// Pinned upstream export repository.
20+
pub const APRIL_MODEL_ID: &str = pocket::APRIL_MODEL_ID;
21+
/// Pinned revision containing the April bundle.
22+
pub const APRIL_MODEL_REVISION: &str = pocket::APRIL_MODEL_REVISION;

‎crates/buzz-voice/src/pocket.rs‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
//! April 2026 Pocket TTS engine for Buzz Desktop.
2+
//!
3+
//! The `english_2026-04` bundle uses SentencePiece tokenization, a learned
4+
//! voice BOS embedding, recurrent FlowLM state, and stateful Mimi decoding.
5+
//! Buzz selects the upstream three-graph INT8 variant while retaining the
6+
//! full-precision Mimi encoder and text conditioner specified by that variant.
7+
//!
8+
//! ## Attribution
9+
//!
10+
//! - Pocket TTS and Mimi: Kyutai, CC-BY-4.0.
11+
//! - ONNX export: KevinAHM/pocket-tts-onnx, CC-BY-4.0.
12+
//! - Reference voice: Kyutai's Mary preset (VCTK p333), CC-BY-4.0.
13+
//!
14+
//! `huddle::models` writes the complete attribution beside the cached bytes.
15+
16+
use std::path::{Path, PathBuf};
17+
use std::sync::Mutex;
18+
19+
use sherpa_onnx::Wave;
20+
21+
#[path = "pocket_april.rs"]
22+
mod pocket_april;
23+
#[path = "pocket_models.rs"]
24+
mod pocket_models;
25+
26+
use pocket_april::{prepare_april_prompt, AprilPocketTts};
27+
pub use pocket_models::{
28+
april_model_info, PocketModelArtifact, PocketModelInfo, APRIL_BUNDLE_ID, APRIL_MODEL_ID,
29+
APRIL_MODEL_REVISION,
30+
};
31+
32+
/// Pocket TTS emits 24 kHz mono PCM.
33+
pub const SAMPLE_RATE: u32 = 24_000;
34+
35+
/// Bundled reference voice name without its extension.
36+
pub const DEFAULT_VOICE: &str = "reference_sample";
37+
38+
/// Pocket voice files are reference WAVs.
39+
pub const VOICE_FILE_EXT: &str = "wav";
40+
41+
const TTS_NUM_THREADS: usize = 1;
42+
43+
/// Loaded reference voice samples and their original sample rate.
44+
#[derive(Debug, Clone)]
45+
pub struct VoiceStyle {
46+
samples: Vec<f32>,
47+
sample_rate: i32,
48+
}
49+
50+
/// Load a Pocket reference voice WAV from disk.
51+
pub fn load_voice_style(path: &Path) -> Result<VoiceStyle, String> {
52+
let path_str = path
53+
.to_str()
54+
.ok_or_else(|| format!("voice path is not valid UTF-8: {}", path.display()))?;
55+
let wave = Wave::read(path_str)
56+
.ok_or_else(|| format!("could not read voice WAV at {}", path.display()))?;
57+
let samples = wave.samples().to_vec();
58+
if samples.is_empty() {
59+
return Err(format!("voice WAV is empty: {}", path.display()));
60+
}
61+
Ok(VoiceStyle {
62+
samples,
63+
sample_rate: wave.sample_rate(),
64+
})
65+
}
66+
67+
/// Resident April INT8 Pocket TTS engine.
68+
pub struct PocketTts {
69+
inner: Mutex<AprilPocketTts>,
70+
}
71+
72+
/// Load Buzz Desktop's pinned April INT8 model.
73+
pub fn load_text_to_speech(model_dir: &str) -> Result<PocketTts, String> {
74+
let dir = PathBuf::from(model_dir);
75+
for artifact in april_model_info().artifacts {
76+
let path = dir.join(artifact.filename);
77+
if !path.is_file() {
78+
return Err(format!(
79+
"incomplete Pocket TTS {} INT8 bundle: missing {}",
80+
APRIL_BUNDLE_ID,
81+
path.display()
82+
));
83+
}
84+
}
85+
Ok(PocketTts {
86+
inner: Mutex::new(AprilPocketTts::load(&dir, TTS_NUM_THREADS)?),
87+
})
88+
}
89+
90+
impl PocketTts {
91+
/// Split text into synthesis units that satisfy the bundle's exact
92+
/// 50-token input limit.
93+
pub fn split_text_into_chunks(&self, text: &str) -> Result<Vec<String>, String> {
94+
let Some(prepared) = prepare_april_prompt(text) else {
95+
return Ok(Vec::new());
96+
};
97+
self.inner
98+
.lock()
99+
.map_err(|_| "Pocket TTS engine lock poisoned".to_string())?
100+
.split_prompt(&prepared)
101+
}
102+
103+
/// Synthesize text with the supplied reference voice.
104+
///
105+
/// Pocket detects language from text and this model uses one synthesis
106+
/// step, so `_lang` and `_steps` intentionally do not affect output.
107+
pub fn synth_chunk(
108+
&self,
109+
text: &str,
110+
_lang: &str,
111+
style: &VoiceStyle,
112+
_steps: usize,
113+
) -> Result<Vec<f32>, String> {
114+
let Some(prepared) = prepare_april_prompt(text) else {
115+
return Ok(Vec::new());
116+
};
117+
let mut engine = self
118+
.inner
119+
.lock()
120+
.map_err(|_| "Pocket TTS engine lock poisoned".to_string())?;
121+
let chunks = engine.split_prompt(&prepared)?;
122+
let mut samples = Vec::new();
123+
for chunk in chunks {
124+
let prepared = prepare_april_prompt(&chunk)
125+
.ok_or_else(|| "Pocket TTS prompt chunk became empty".to_string())?;
126+
samples.extend(engine.synth_chunk(&prepared, style)?);
127+
}
128+
Ok(samples)
129+
}
130+
}
131+
132+
#[cfg(test)]
133+
mod tests {
134+
use super::*;
135+
136+
#[test]
137+
fn desktop_model_is_april_int8_only() {
138+
let info = april_model_info();
139+
assert_eq!(info.max_token_per_chunk, 50);
140+
assert_eq!(info.sample_rate, SAMPLE_RATE);
141+
assert!(info
142+
.artifacts
143+
.iter()
144+
.any(|artifact| artifact.filename == "flow_lm_main_int8.onnx"));
145+
assert!(!info
146+
.artifacts
147+
.iter()
148+
.any(|artifact| artifact.filename == "flow_lm_main.onnx"));
149+
}
150+
151+
#[test]
152+
#[ignore = "requires BUZZ_POCKET_TEST_MODEL_DIR"]
153+
fn production_api_emits_non_silent_april_int8_pcm() {
154+
let dir = std::env::var("BUZZ_POCKET_TEST_MODEL_DIR")
155+
.expect("set BUZZ_POCKET_TEST_MODEL_DIR to an April INT8 model directory");
156+
let engine = load_text_to_speech(&dir).expect("load April INT8 engine");
157+
let style = load_voice_style(&Path::new(&dir).join("reference_sample.wav"))
158+
.expect("load reference voice");
159+
let samples = engine
160+
.synth_chunk("Bright birds begin beside the bay.", "en", &style, 1)
161+
.expect("synthesize through the production API");
162+
163+
assert!(!samples.is_empty());
164+
assert!(samples.iter().all(|sample| sample.is_finite()));
165+
assert!(samples.iter().any(|sample| sample.abs() > 1.0e-6));
166+
}
167+
}
File renamed without changes.

desktop/src-tauri/src/huddle/pocket_models.rs renamed to crates/buzz-voice/src/pocket_models.rs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,19 @@ pub struct PocketModelArtifact {
2424
/// Capabilities of Buzz Desktop's sole Pocket model.
2525
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2626
pub struct PocketModelInfo {
27+
/// Language bundle selected from the pinned export.
2728
pub bundle_id: &'static str,
29+
/// Upstream model repository.
2830
pub source_model_id: &'static str,
31+
/// Pinned upstream model revision.
2932
pub revision: &'static str,
33+
/// PCM output sample rate.
3034
pub sample_rate: u32,
35+
/// Maximum input size declared by the bundle.
3136
pub max_token_per_chunk: usize,
37+
/// Immutable files required by the runtime.
3238
pub artifacts: &'static [PocketModelArtifact],
39+
/// Components quantized in the selected bundle.
3340
pub quantized_components: &'static [&'static str],
3441
}
3542

‎desktop/playwright.config.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ export default defineConfig({
4949
"**/activity-scope-label-screenshots.spec.ts",
5050
"**/welcome-agent-modal-screenshots.spec.ts",
5151
"**/local-archive-screenshots.spec.ts",
52+
"**/voice-settings.spec.ts",
5253
"**/agent-readiness-screenshots.spec.ts",
5354
"**/agent-error-state-screenshots.spec.ts",
5455
"**/edit-agent.spec.ts",

‎desktop/src-tauri/Cargo.lock‎

Lines changed: 15 additions & 5 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)