Skip to content

Commit 794a27b

Browse files
committed
feat(normalizer): caps_normalization option, spoken phone numbers, hertz plurals, drop the 'S pass, emoji tag flags and joiners, web toggles (remsky#353)
1 parent 4d1c6f4 commit 794a27b

14 files changed

Lines changed: 103 additions & 67 deletions

File tree

‎.claude/skills/integration-tests/SKILL.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,4 +36,4 @@ Or the full compose stack (builds a CPU server + test-client, self-contained):
3636
- `test_tts_roundtrip.py` - synth + Whisper transcribe across 9 languages, WER/CER thresholds
3737
- `test_voices_endpoint.py` - voice listing shape, legacy compat, nova mapping
3838

39-
17 tests total. The roundtrip tests need ~30s (Whisper inference is the bottleneck).
39+
19 tests total. The roundtrip tests need ~30s (Whisper inference is the bottleneck).

‎CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ Per-PR attribution and contributor credits are published automatically on the co
77
## [Unreleased]
88
### Added
99
- `normalization_options.remove_emoji` drops emoji before synthesis instead of reading them by name, any language (#353). Off by default.
10+
- `normalization_options.caps_normalization` reads all-caps headers and names (`ARNE SAKNUSSEMM`) as words instead of letter by letter. On by default. Short acronyms (`FBI`, `US GDP`) are still spelled.
1011

1112
### Changed
1213
- Text normalization refactored towards multi-language support: `Normalizer` base class w/ neutral passes, `EnglishNormalizer` implements the rest, registry keyed by lang code. Adding a language is a subclass + table test, see CONTRIBUTING.md.
@@ -20,6 +21,9 @@ Per-PR attribution and contributor credits are published automatically on the co
2021
- Digits glued to letters (`MP3`, `B2B`, `v1.0`, `1.5x`) pass through as written. `COVID-19` is no longer COVID minus nineteen.
2122
- `.5` reads as zero point five, `1980S` as nineteen eighty S.
2223
- Long digit runs no longer stall or 500 the request.
24+
- Phone numbers read as spoken digits, `555-123-4567` as five five five, one two three, four five six seven.
25+
- `3.5 GHz` reads gigahertz and `1 min` reads one minute with `unit_normalization` on.
26+
- Plural and possessive acronyms (`DVDs`, `DVD's`) are no longer rewritten to `DVD'S`, which was spelled out as dee-vee-dee-ess.
2327
- Times with seconds keep their am/pm (`12:30:15 pm`).
2428
- Blank input, or emoji-only input with `remove_emoji`, is a 400 on the streaming path too, not an empty 200.
2529
- `--` and `---` read as a dash. Previously the words on either side fused and word timestamps stopped for the rest of the chunk (#249).

‎api/src/services/text_processing/normalization/base.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ def pass_order(
2323
self, options: NormalizationOptions
2424
) -> tuple[Callable[[str], str] | None, ...]:
2525
return (
26+
self.caps_words if options.caps_normalization else None,
2627
self.contractions,
2728
self.emails if options.email_normalization else None,
2829
self.urls if options.url_normalization else None,
@@ -39,7 +40,6 @@ def pass_order(
3940
self.common_words,
4041
self.numbers,
4142
self.symbols if options.replace_remaining_symbols else None,
42-
self.possessives,
4343
self.acronyms,
4444
)
4545

@@ -65,6 +65,9 @@ def whitespace(self, text: str) -> str:
6565
return re.sub(r" +", " ", text)
6666

6767
# language hooks, no-op until overridden
68+
def caps_words(self, text: str) -> str:
69+
return text
70+
6871
def contractions(self, text: str) -> str:
6972
return text
7073

@@ -98,8 +101,5 @@ def numbers(self, text: str) -> str:
98101
def symbols(self, text: str) -> str:
99102
return text
100103

101-
def possessives(self, text: str) -> str:
102-
return text
103-
104104
def acronyms(self, text: str) -> str:
105105
return text

‎api/src/services/text_processing/normalization/english.py‎

Lines changed: 28 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,17 @@
77
import inflect
88

99
from .base import Normalizer
10-
from .english_data import MONEY_UNITS, SYMBOL_REPLACEMENTS, VALID_TLDS, VALID_UNITS
10+
from .english_data import (
11+
INFLECT_NOUNS,
12+
MONEY_UNITS,
13+
SYMBOL_REPLACEMENTS,
14+
VALID_TLDS,
15+
VALID_UNITS,
16+
)
1117

1218
INFLECT_ENGINE = inflect.engine()
19+
for singular, plural in INFLECT_NOUNS.items():
20+
INFLECT_ENGINE.defnoun(singular, plural)
1321

1422

1523
def conditional_int(number: float, threshold: float = 0.00001):
@@ -53,6 +61,21 @@ def wrapped(m: re.Match[str]) -> str:
5361
class EnglishNormalizer(Normalizer):
5462
lang_codes = ("a", "b", "en-us", "en-gb")
5563

64+
CAPS_RUN_PATTERN = re.compile(r"\b[A-Z]+(?:[^A-Za-z]+[A-Z]+)+\b")
65+
CAPS_WORD_PATTERN = re.compile(r"\b[A-Z]{4,}\b")
66+
LONG_CAPS_PATTERN = re.compile(r"\b[A-Z]{6,}\b")
67+
ROMAN_PATTERN = re.compile(r"[IVXLCDM]+")
68+
69+
def caps_words(self, text: str) -> str:
70+
def lower(m: re.Match[str]) -> str:
71+
word = m.group()
72+
return word if self.ROMAN_PATTERN.fullmatch(word) else word.lower()
73+
74+
text = self.CAPS_RUN_PATTERN.sub(
75+
lambda run: self.CAPS_WORD_PATTERN.sub(lower, run.group()), text
76+
)
77+
return self.LONG_CAPS_PATTERN.sub(lower, text)
78+
5679
def contractions(self, text: str) -> str:
5780
# Expand the "'re" contractions that espeak mis-phonemizes with a spurious
5881
# /ɹeɪ/ ("-ray") ending, e.g. "how're" -> /haʊɹeɪ/ which sounds like "harry".
@@ -175,28 +198,16 @@ def optional_plurals(self, text: str) -> str:
175198
return re.sub(r"\(s\)", "s", text)
176199

177200
PHONE_PATTERN = re.compile(
178-
r"(\+?\d{1,2})?([ .-]?)(\(?\d{3}\)?)[\s.-](\d{3})[\s.-](\d{4})"
201+
r"(?:(\+?\d{1,2})[ .-]?)?(\(?\d{3}\)?)[\s.-](\d{3})[\s.-](\d{4})"
179202
)
180203

181204
def phone_numbers(self, text: str) -> str:
182205
def handle(p: re.Match[str]) -> str:
183-
p = list(p.groups())
184-
185-
country_code = ""
186-
if p[0] is not None:
187-
p[0] = p[0].replace("+", "")
188-
country_code += INFLECT_ENGINE.number_to_words(p[0])
189-
190-
area_code = INFLECT_ENGINE.number_to_words(
191-
p[2].replace("(", "").replace(")", ""), group=1, comma=""
206+
groups = [re.sub(r"\D", "", g) for g in p.groups() if g]
207+
return ", ".join(
208+
" ".join(INFLECT_ENGINE.number_to_words(d) for d in g) for g in groups
192209
)
193210

194-
telephone_prefix = INFLECT_ENGINE.number_to_words(p[3], group=1, comma="")
195-
196-
line_number = INFLECT_ENGINE.number_to_words(p[4], group=1, comma="")
197-
198-
return ",".join([country_code, area_code, telephone_prefix, line_number])
199-
200211
return self.PHONE_PATTERN.sub(handle, text)
201212

202213
TIME_PATTERN = re.compile(
@@ -350,10 +361,6 @@ def symbols(self, text: str) -> str:
350361
text = text.replace(symbol, replacement)
351362
return text
352363

353-
def possessives(self, text: str) -> str:
354-
text = re.sub(r"(?<=[BCDFGHJ-NP-TV-Z])'?s\b", "'S", text)
355-
return re.sub(r"(?<=X')S\b", "s", text)
356-
357364
def acronyms(self, text: str) -> str:
358365
text = re.sub(
359366
# {2,12} not {2,}: real acronyms are short; unbounded backtracks O(n^2) on floods.

‎api/src/services/text_processing/normalization/english_data.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@
5757
"mg": "milligram", # Mass
5858
"s": "second",
5959
"ms": "millisecond",
60-
"min": "minutes",
60+
"min": "minute",
6161
"h": "hour", # Time
6262
"l": "liter",
6363
"ml": "mililiter",
@@ -116,6 +116,10 @@
116116
"px": "pixel", # CSS units
117117
}
118118

119+
INFLECT_NOUNS = {
120+
"(.+)hertz": "$1hertz",
121+
}
122+
119123
SYMBOL_REPLACEMENTS = {
120124
"~": " ",
121125
"@": " at ",

‎api/src/services/text_processing/text_processor.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,9 @@
2323
# Pattern to find pause tags like [pause:0.5s]
2424
PAUSE_TAG_PATTERN = re.compile(r"\[pause:(\d+(?:\.\d+)?)s\]", re.IGNORECASE)
2525
EMOJI_PATTERN = regex.compile(
26-
r" ?(?:[0-9#*]\uFE0F?\u20E3|[\p{Extended_Pictographic}\p{Regional_Indicator}"
27-
r"\p{Emoji_Modifier}\u200D\uFE0F\u20E3]+) ?"
26+
r" ?(?:[0-9#*]\uFE0F?\u20E3|[\p{Extended_Pictographic}\p{Regional_Indicator}]"
27+
r"[\p{Extended_Pictographic}\p{Regional_Indicator}\p{Emoji_Modifier}"
28+
r"\u200D\uFE0F\u20E3\U000E0020-\U000E007F]*) ?"
2829
)
2930
PARAGRAPH_PATTERN = re.compile(r"(?:\r?\n[^\S\r\n]*){2,}")
3031
LINE_PATTERN = re.compile(r"\r?\n[^\S\r\n]*")

‎api/src/structures/schemas.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,10 @@ class NormalizationOptions(BaseModel):
123123
default=True,
124124
description="Changes phone numbers so they can be properly pronouced by kokoro",
125125
)
126+
caps_normalization: bool = Field(
127+
default=True,
128+
description="Reads all-caps headers and names as words instead of spelling them out, short acronyms like FBI stay spelled",
129+
)
126130
replace_remaining_symbols: bool = Field(
127131
default=True,
128132
description="Replaces the remaining symbols after normalization with their words",

‎api/tests/integration/test_tts_roundtrip.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ class Case:
4646
CASES: list[Case] = [
4747
Case("af_heart", "en", "The quick brown fox jumps over the lazy dog."),
4848
Case("bf_emma", "en", "The rain in Spain falls mainly on the plain."),
49+
Case("am_michael", "en", "GANDALF AND FRODO LEAVE MORDOR."),
4950
Case("ef_dora", "es", "El sol brilla en el cielo azul."),
5051
Case("ff_siwis", "fr", "Le soleil brille dans le ciel bleu."),
5152
Case("if_sara", "it", "Il gatto dorme sul tappeto rosso."),

‎api/tests/test_normalizer.py‎

Lines changed: 30 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,20 @@
184184
"Blinked? No. Smiled ever so slightly? Yes.",
185185
"Blinked? No. Smiled ever so slightly? Yes.",
186186
),
187+
# phone numbers (pr #179)
188+
("Call 555-123-4567", "Call five five five, one two three, four five six seven"),
189+
("Call (555) 123-4567", "Call five five five, one two three, four five six seven"),
190+
(
191+
"Call +1 555 123 4567",
192+
"Call one, five five five, one two three, four five six seven",
193+
),
194+
(
195+
"Dial 555.123.4567 now",
196+
"Dial five five five, one two three, four five six seven now",
197+
),
198+
# plural and possessive acronyms (pr #493)
199+
("the DVD's case", "the DVD's case"),
200+
("two CDs and DVDs", "two CDs and DVDs"),
187201
# punctuation and whitespace
188202
("你好,世界", "你好, 世界"),
189203
("你好。世界!好吗?再见", "你好. 世界! 好吗? 再见"),
@@ -231,6 +245,15 @@
231245
"Therefore the genre was hardcore before more.",
232246
"Therefore the genre was hardcore before more.",
233247
),
248+
# all-caps names and headers
249+
("ARNE SAKNUSSEMM", "arne saknussemm"),
250+
("CHAPTER IV. ARNE SAKNUSSEMM", "chapter IV. arne saknussemm"),
251+
("LIDENBROCK.", "lidenbrock."),
252+
("A TALE OF TWO CITIES", "A tale OF TWO cities"),
253+
("the FBI and CIA agreed", "the FBI and CIA agreed"),
254+
("US GDP grew", "US GDP grew"),
255+
("an HDMI cable", "an HDMI cable"),
256+
("NASA JPL", "nasa JPL"),
234257
]
235258

236259
UNITS = NormalizationOptions(unit_normalization=True)
@@ -242,6 +265,10 @@
242265
("The file is 5 MB", "The file is five megabytes", UNITS),
243266
("Download 5MB", "Download five megabytes", UNITS),
244267
("Set to 10 Mb", "Set to ten megabits", UNITS),
268+
("Runs at 3.5 GHz", "Runs at three point five gigahertz", UNITS),
269+
("Tuned to 1 Hz", "Tuned to one hertz", UNITS),
270+
("Wait 5 min", "Wait five minutes", UNITS),
271+
("Wait 1 min", "Wait one minute", UNITS),
245272
# pr #526
246273
("It is 72 F outside", "It is seventy-two degrees fahrenheit outside", UNITS),
247274
("It is 72 °F outside", "It is seventy-two degrees fahrenheit outside", UNITS),
@@ -266,28 +293,11 @@
266293
NormalizationOptions(optional_pluralization_normalization=False),
267294
),
268295
("a & b @ c", "a & b @ c", NormalizationOptions(replace_remaining_symbols=False)),
269-
]
270-
271-
KNOWN_BUGS = [
272-
("Runs at 3.5 GHz", "Runs at three point five gigahertz", UNITS),
273-
# phone numbers (pr #179)
274296
(
275-
"Call 555-123-4567",
276-
"Call five five five, one two three, four five six seven",
277-
NormalizationOptions(),
278-
),
279-
(
280-
"Call (555) 123-4567",
281-
"Call five five five, one two three, four five six seven",
282-
NormalizationOptions(),
283-
),
284-
(
285-
"Call +1 555 123 4567",
286-
"Call one, five five five, one two three, four five six seven",
287-
NormalizationOptions(),
297+
"ARNE SAKNUSSEMM",
298+
"ARNE SAKNUSSEMM",
299+
NormalizationOptions(caps_normalization=False),
288300
),
289-
# pr #493
290-
("the DVD's case", "the DVD's case", NormalizationOptions()),
291301
]
292302

293303

@@ -301,11 +311,3 @@ def test_normalize(text, expected):
301311
)
302312
def test_normalize_with_options(text, expected, options):
303313
assert normalize_text(text, options) == expected
304-
305-
306-
@pytest.mark.xfail(strict=True, reason="known bug, move the row to CASES once fixed")
307-
@pytest.mark.parametrize(
308-
"text, expected, options", KNOWN_BUGS, ids=[text for text, _, _ in KNOWN_BUGS]
309-
)
310-
def test_known_bugs(text, expected, options):
311-
assert normalize_text(text, options) == expected

‎api/tests/test_normalizer_redos.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,14 @@ def test_pattern_floods_scale_linearly():
8989
assert large / max(small, 0.05) < 9.0, unit
9090

9191

92+
def test_caps_floods_scale_linearly():
93+
"""Caps runs, long caps words and roman-numeral floods through the caps pass."""
94+
for unit in ("AB ", "ABCD ", "ABCDEF\n", "I", "A.B "):
95+
small = _elapsed(unit * 5_000)
96+
large = _elapsed(unit * 20_000)
97+
assert large / max(small, 0.05) < 9.0, unit
98+
99+
92100
def test_huge_digit_run_does_not_raise():
93101
"""float() overflows to inf on 310+ digits, which used to crash the request."""
94102
assert normalize_text("9" * 400, OPTS) == "9" * 400

0 commit comments

Comments
 (0)