|
7 | 7 | import inflect |
8 | 8 |
|
9 | 9 | from .base import Normalizer |
10 | | -from .english_data import MONEY_UNITS, SYMBOL_REPLACEMENTS, VALID_TLDS, VALID_UNITS |
| 10 | +from .english_data import ( |
| 11 | + INFLECT_NOUNS, |
| 12 | + MONEY_UNITS, |
| 13 | + SYMBOL_REPLACEMENTS, |
| 14 | + VALID_TLDS, |
| 15 | + VALID_UNITS, |
| 16 | +) |
11 | 17 |
|
12 | 18 | INFLECT_ENGINE = inflect.engine() |
| 19 | +for singular, plural in INFLECT_NOUNS.items(): |
| 20 | + INFLECT_ENGINE.defnoun(singular, plural) |
13 | 21 |
|
14 | 22 |
|
15 | 23 | def conditional_int(number: float, threshold: float = 0.00001): |
@@ -53,6 +61,21 @@ def wrapped(m: re.Match[str]) -> str: |
53 | 61 | class EnglishNormalizer(Normalizer): |
54 | 62 | lang_codes = ("a", "b", "en-us", "en-gb") |
55 | 63 |
|
| 64 | + CAPS_RUN_PATTERN = re.compile(r"\b[A-Z]+(?:[^A-Za-z]+[A-Z]+)+\b") |
| 65 | + CAPS_WORD_PATTERN = re.compile(r"\b[A-Z]{4,}\b") |
| 66 | + LONG_CAPS_PATTERN = re.compile(r"\b[A-Z]{6,}\b") |
| 67 | + ROMAN_PATTERN = re.compile(r"[IVXLCDM]+") |
| 68 | + |
| 69 | + def caps_words(self, text: str) -> str: |
| 70 | + def lower(m: re.Match[str]) -> str: |
| 71 | + word = m.group() |
| 72 | + return word if self.ROMAN_PATTERN.fullmatch(word) else word.lower() |
| 73 | + |
| 74 | + text = self.CAPS_RUN_PATTERN.sub( |
| 75 | + lambda run: self.CAPS_WORD_PATTERN.sub(lower, run.group()), text |
| 76 | + ) |
| 77 | + return self.LONG_CAPS_PATTERN.sub(lower, text) |
| 78 | + |
56 | 79 | def contractions(self, text: str) -> str: |
57 | 80 | # Expand the "'re" contractions that espeak mis-phonemizes with a spurious |
58 | 81 | # /ɹeɪ/ ("-ray") ending, e.g. "how're" -> /haʊɹeɪ/ which sounds like "harry". |
@@ -175,28 +198,16 @@ def optional_plurals(self, text: str) -> str: |
175 | 198 | return re.sub(r"\(s\)", "s", text) |
176 | 199 |
|
177 | 200 | PHONE_PATTERN = re.compile( |
178 | | - r"(\+?\d{1,2})?([ .-]?)(\(?\d{3}\)?)[\s.-](\d{3})[\s.-](\d{4})" |
| 201 | + r"(?:(\+?\d{1,2})[ .-]?)?(\(?\d{3}\)?)[\s.-](\d{3})[\s.-](\d{4})" |
179 | 202 | ) |
180 | 203 |
|
181 | 204 | def phone_numbers(self, text: str) -> str: |
182 | 205 | def handle(p: re.Match[str]) -> str: |
183 | | - p = list(p.groups()) |
184 | | - |
185 | | - country_code = "" |
186 | | - if p[0] is not None: |
187 | | - p[0] = p[0].replace("+", "") |
188 | | - country_code += INFLECT_ENGINE.number_to_words(p[0]) |
189 | | - |
190 | | - area_code = INFLECT_ENGINE.number_to_words( |
191 | | - p[2].replace("(", "").replace(")", ""), group=1, comma="" |
| 206 | + groups = [re.sub(r"\D", "", g) for g in p.groups() if g] |
| 207 | + return ", ".join( |
| 208 | + " ".join(INFLECT_ENGINE.number_to_words(d) for d in g) for g in groups |
192 | 209 | ) |
193 | 210 |
|
194 | | - telephone_prefix = INFLECT_ENGINE.number_to_words(p[3], group=1, comma="") |
195 | | - |
196 | | - line_number = INFLECT_ENGINE.number_to_words(p[4], group=1, comma="") |
197 | | - |
198 | | - return ",".join([country_code, area_code, telephone_prefix, line_number]) |
199 | | - |
200 | 211 | return self.PHONE_PATTERN.sub(handle, text) |
201 | 212 |
|
202 | 213 | TIME_PATTERN = re.compile( |
@@ -345,14 +356,11 @@ def handle(n: re.Match[str]) -> str: |
345 | 356 | return self.NUMBER_PATTERN.sub(handle, text) |
346 | 357 |
|
347 | 358 | def symbols(self, text: str) -> str: |
| 359 | + text = re.sub(r" ?-{2,} ?", " — ", text) |
348 | 360 | for symbol, replacement in SYMBOL_REPLACEMENTS.items(): |
349 | 361 | text = text.replace(symbol, replacement) |
350 | 362 | return text |
351 | 363 |
|
352 | | - def possessives(self, text: str) -> str: |
353 | | - text = re.sub(r"(?<=[BCDFGHJ-NP-TV-Z])'?s\b", "'S", text) |
354 | | - return re.sub(r"(?<=X')S\b", "s", text) |
355 | | - |
356 | 364 | def acronyms(self, text: str) -> str: |
357 | 365 | text = re.sub( |
358 | 366 | # {2,12} not {2,}: real acronyms are short; unbounded backtracks O(n^2) on floods. |
|
0 commit comments