Skip to content

Commit 15667de

Browse files
npub1jmc9dt2lyvzu3h0kxlwxt5zg4fxp9476awyxw6gwxn72g6cw7exqs64whmtlongwell-block
andcommitted
perf(voice): stop token counting at the first oversized boundary
Each boundary scan in split_at_natural_boundaries tokenized every word and clause boundary through end-of-text, skipping candidates that already exceeded the limit instead of stopping at them. Because token_count re-encodes the prefix text[start..end], tokenizer input grew superlinearly in prompt length, and that work is paid before the first chunk reaches synthesis -- taxing the time-to-first-audio this segmentation is meant to improve. A 1.9 KB prompt at the 2000-char MAX_TTS_TEXT_LEN cap tokenized 2.36 MB, and doubling a prompt multiplied tokenizer input by ~5.5x. Prepared token counts are monotonic in prefix length, so the first candidate that overflows proves no longer candidate can fit. Break out of the scan there. Chunk boundaries are unchanged: verified output-identical against the previous behavior across 5040 cases (21 corpora x max_tokens 1..=60 x both split policies x two tokenizer shapes), while saving 1,073,280 tokenizer calls and 568 MB of tokenized input. Co-authored-by: Tyler Longwell <[email protected]> Signed-off-by: Tyler Longwell <[email protected]>
1 parent 4e6d2df commit 15667de

1 file changed

Lines changed: 42 additions & 1 deletion

File tree

‎crates/buzz-voice/src/pocket_april.rs‎

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -672,9 +672,17 @@ where
672672
.chars()
673673
.next()
674674
.is_some_and(is_closing_punctuation);
675-
if (!at_word_end && !at_clause_end) || token_count(&text[start..end])? > max_tokens {
675+
if !at_word_end && !at_clause_end {
676676
continue;
677677
}
678+
// Prepared token counts are monotonic in prefix length, so once a
679+
// candidate overflows the limit no longer candidate can fit. Stop
680+
// scanning instead of tokenizing every remaining boundary: that
681+
// kept this loop superlinear in prompt length, and the cost landed
682+
// before the first chunk reached synthesis.
683+
if token_count(&text[start..end])? > max_tokens {
684+
break;
685+
}
678686

679687
word_end = Some(end);
680688
match natural_boundary(&text[start..end], end == text.len()) {
@@ -1078,6 +1086,39 @@ mod tests {
10781086
assert_eq!(chunks.concat(), text);
10791087
}
10801088

1089+
#[test]
1090+
fn natural_split_stops_counting_tokens_past_the_limit() {
1091+
// Each boundary scan must stop at the first overflowing candidate
1092+
// rather than tokenizing every remaining boundary. Scanning to
1093+
// end-of-text makes tokenizer input grow superlinearly in prompt
1094+
// length, and that cost is paid before the first chunk reaches
1095+
// synthesis, taxing time-to-first-audio on long prompts.
1096+
let sentence = "The relay finished its migration and the channel list refreshed. ";
1097+
let tokenized_bytes = |repeats: usize| -> usize {
1098+
let text = sentence.repeat(repeats).trim_end().to_string();
1099+
let total = std::cell::Cell::new(0_usize);
1100+
let chunks = split_at_natural_boundaries(&text, 50, true, |chunk| {
1101+
total.set(total.get() + chunk.len());
1102+
whitespace_token_count(chunk)
1103+
})
1104+
.expect("split repeated sentences");
1105+
assert_eq!(chunks.concat(), text);
1106+
assert!(chunks.len() > 1);
1107+
total.get()
1108+
};
1109+
1110+
// Doubling the prompt must not multiply tokenizer work superlinearly.
1111+
// Bounded scans grow ~2x here; scanning to end-of-text grows ~5.5x.
1112+
let single = tokenized_bytes(12);
1113+
let double = tokenized_bytes(24);
1114+
assert!(
1115+
double < single * 3,
1116+
"doubling the prompt grew tokenizer input from {single} to {double} bytes \
1117+
({:.1}x); bounded scans stay near 2x",
1118+
double as f64 / single as f64,
1119+
);
1120+
}
1121+
10811122
#[test]
10821123
fn normal_noise_has_requested_length() {
10831124
let mut rng = rand::rng();

0 commit comments

Comments
 (0)