Skip to content

Commit d2aae10

Browse files
remskylionel-rowe
andauthored
fix(chunking): replace regex sentence splitter with UAX #29 segmentation (#308, adapts #415) (#520)
Co-authored-by: lionel-rowe <[email protected]>
1 parent 8307b0e commit d2aae10

5 files changed

Lines changed: 201 additions & 20 deletions

File tree

‎api/src/services/text_processing/text_processor.py‎

Lines changed: 6 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
from typing import AsyncGenerator, Dict, Iterator, List, Optional, Tuple
77

88
from loguru import logger
9+
from unicode_segmentation_rs import unicode_sentences
910

1011
from ...core.config import settings
1112
from ...structures.schemas import VOICE_NAME_BODY, NormalizationOptions, clamp_rate
@@ -101,26 +102,12 @@ def get_sentence_info(
101102
text: str, lang_code: str = "a"
102103
) -> Iterator[Tuple[str, List[int], int]]:
103104
"""Yield (sentence, tokens, token_count) per sentence, phonemizing lazily."""
104-
# Detect Chinese text
105-
is_chinese = lang_code.startswith("z") or re.search(r"[\u4e00-\u9fff]", text)
106-
if is_chinese:
107-
# Split using Chinese punctuation
108-
sentences = re.split(r"([,。!?;])+", text)
109-
else:
110-
sentences = re.split(r"([.!?;:])(?=\s|$)", text)
111-
112-
for i in range(0, len(sentences), 2):
113-
sentence = sentences[i].strip()
114-
punct = sentences[i + 1] if i + 1 < len(sentences) else ""
105+
for sentence in unicode_sentences(text):
106+
sentence = sentence.strip()
115107
if not sentence:
116108
continue
117-
full = sentence + punct
118-
# Strip the full sentence to remove any leading/trailing spaces before processing
119-
full = full.strip()
120-
if not full: # Skip if empty after stripping
121-
continue
122-
tokens = process_text_chunk(full)
123-
yield full, tokens, len(tokens)
109+
tokens = process_text_chunk(sentence)
110+
yield sentence, tokens, len(tokens)
124111

125112

126113
def split_by_voice(text: str, default_voice: str) -> List[Tuple[str, float, str]]:
@@ -267,7 +254,7 @@ async def smart_split(
267254
current_count = 0
268255

269256
# Split long sentence on commas
270-
clauses = re.split(r"([,])", sentence)
257+
clauses = re.split(r"([,;:,、;:])", sentence)
271258
clause_chunk = []
272259
clause_tokens = []
273260
clause_count = 0

‎api/tests/test_text_processor.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,29 @@ def test_get_sentence_info():
4747
assert count > 0
4848

4949

50+
def test_get_sentence_info_abbreviations():
51+
"""Abbreviations, decimals, and ellipses do not force bogus sentence breaks."""
52+
text = (
53+
"This, that, the other thing, etc. Another sentence... A, b, c, etc., and "
54+
"more. D, e, f, etc. and more. One, i. e. two. Three, i. e., four. Five, "
55+
"i.e. six. You have 4.2 messages. Property access: `a.b.c`."
56+
)
57+
58+
sentences = [s for s, _, _ in get_sentence_info(text)]
59+
60+
assert sentences == [
61+
"This, that, the other thing, etc.",
62+
"Another sentence...",
63+
"A, b, c, etc., and more.",
64+
"D, e, f, etc. and more.",
65+
"One, i. e. two.",
66+
"Three, i. e., four.",
67+
"Five, i.e. six.",
68+
"You have 4.2 messages.",
69+
"Property access: `a.b.c`.",
70+
]
71+
72+
5073
def test_get_sentence_info_is_lazy(monkeypatch):
5174
"""Sentences are phonemized as they are pulled, not all up front."""
5275
calls = []

‎examples/ab_split_test.py‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
"""A/B listen test for the sentence splitter change (PR #415 adaptation).
2+
3+
Run once against the old image with --label old, rebuild, run again with
4+
--label new. Files land in examples/ab_split/<label>/ with matching names,
5+
so any player sorted by name gives direct A/B pairs.
6+
7+
uv run ab_split_test.py --label old
8+
"""
9+
10+
import argparse
11+
from pathlib import Path
12+
13+
import requests
14+
15+
URL = "http://localhost:8880/v1/audio/speech"
16+
17+
CASES = [
18+
(
19+
"01_abbrev_mid",
20+
"af_heart",
21+
"This, that, the other thing, etc. was on the list. We need bread, milk, eggs, etc. before the store closes.",
22+
),
23+
(
24+
"02_abbrev_end",
25+
"af_heart",
26+
"Bring pencils, paper, erasers, etc. Then we can start the exam.",
27+
),
28+
(
29+
"03_ie_eg",
30+
"af_heart",
31+
"Use the primary color, i.e. red, for warnings. Some fruits, e.g. apples and pears, keep well.",
32+
),
33+
(
34+
"04_decimals",
35+
"af_heart",
36+
"You have 4.2 messages on average. The rate rose 3.5 percent, reaching 12.8 million this year.",
37+
),
38+
(
39+
"05_semicolons",
40+
"af_heart",
41+
"The plan was simple; the execution was not. We packed three things: rope, water, and a map.",
42+
),
43+
(
44+
"06_titles",
45+
"af_heart",
46+
"Dr. Smith met Mr. Jones at 5 p.m. on Main St. to discuss the results.",
47+
),
48+
(
49+
"07_dash_continuation",
50+
"af_heart",
51+
"The Marbles. --- In the heart of a vibrant city in Spain stood an ancient temple, a masterpiece of design.",
52+
),
53+
(
54+
"08_runon_clauses",
55+
"af_heart",
56+
"When the committee finally assembled after weeks of delays, cancellations, rescheduled flights, "
57+
"missing paperwork, and two separate hotel mixups, the chairman, visibly exhausted but determined "
58+
"to proceed, opened the meeting with a lengthy statement about procedure, precedent, patience, "
59+
"and the pressing need to conclude before the holidays, which everyone privately doubted was possible.",
60+
),
61+
(
62+
"09_tags",
63+
"af_heart",
64+
"First a normal sentence. [pause:1.5s] Then after a pause, [voice:am_adam] a different voice finishes the thought.",
65+
),
66+
(
67+
"10_zh_commas",
68+
"zf_xiaobei",
69+
"他走进房间,放下行李,看了看窗外,然后叹了口气。天色渐暗,远处的灯光一盏盏亮起,街道安静得出奇。",
70+
),
71+
(
72+
"11_zh_terminators",
73+
"zf_xiaobei",
74+
"这是第一句。这真的可能吗?当然可能!我们明天就出发。",
75+
),
76+
]
77+
78+
NEWLINE_CASE = (
79+
"12_newlines_nonorm",
80+
"af_heart",
81+
"Chapter One\n\nThe rain had not stopped for three days. Rivers crept up their banks.",
82+
)
83+
84+
85+
def synth(label_dir: Path, name: str, voice: str, text: str, normalize: bool = True):
86+
payload = {
87+
"model": "kokoro",
88+
"input": text,
89+
"voice": voice,
90+
"response_format": "mp3",
91+
"stream": False,
92+
}
93+
if not normalize:
94+
payload["normalization_options"] = {"normalize": False}
95+
r = requests.post(URL, json=payload, timeout=120)
96+
r.raise_for_status()
97+
out = label_dir / f"{name}.mp3"
98+
out.write_bytes(r.content)
99+
print(f" {out.name} ({len(r.content) / 1024:.0f} KB)")
100+
101+
102+
def main():
103+
parser = argparse.ArgumentParser()
104+
parser.add_argument("--label", required=True, choices=["old", "new"])
105+
args = parser.parse_args()
106+
107+
label_dir = Path(__file__).parent / "ab_split" / args.label
108+
label_dir.mkdir(parents=True, exist_ok=True)
109+
print(f"Writing {label_dir}")
110+
111+
for name, voice, text in CASES:
112+
synth(label_dir, name, voice, text)
113+
name, voice, text = NEWLINE_CASE
114+
synth(label_dir, name, voice, text, normalize=False)
115+
116+
117+
if __name__ == "__main__":
118+
main()

‎pyproject.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ dependencies = [
2020
# Audio processing
2121
"soundfile==0.13.0",
2222
"regex>=2025.10.22",
23+
"unicode-segmentation-rs>=0.3.3",
2324
# Utilities
2425
"aiofiles==23.2.1",
2526
"tqdm==4.67.1",

0 commit comments

Comments
 (0)