Skip to content

Commit 158e4e8

Browse files
committed
fix(normalizer): read comma-grouped numbers as thousands, not years (#259)
1 parent b6253e1 commit 158e4e8

4 files changed

Lines changed: 63 additions & 10 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ Per-PR attribution and contributor credits are published automatically on the co
1919
- `DEFAULT_VOICE_CODE` now applies on the speech endpoints and `/dev/generate_from_phonemes` (#514, #515 by @Christian-Sidak, #516).
2020
- Entrypoint and `start-*.sh` honour `HOST` / `PORT`, so `HOST=::` binds IPv6-only (#408, reported by @felixls).
2121
- Blank lines now end a chunk, so headings, bylines, etc no longer run into the following paragraph. Single newlines still join. With `normalize: false` the text reaches the model verbatim, newlines included (#519).
22+
- Comma-grouped numbers like `3,497` are read as thousands, not as a year (#259). Long digit runs no longer stall or 500 the request.
2223

2324
## [v0.8.1] - 2026-08-24
2425
### Fixed

‎api/src/services/text_processing/normalizer.py‎

Lines changed: 21 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -184,12 +184,12 @@
184184
MONEY_PATTERN = re.compile(
185185
r"(-?)(["
186186
+ "".join(MONEY_UNITS.keys())
187-
+ r"])(\d+(?:\.\d+)?)((?: hundred| thousand| (?:[bm]|tr|quadr)illion|k|m|b|t)*)\b",
187+
+ r"])(\d{1,3}(?:,\d{3})+(?:\.\d+)?|\d+(?:\.\d+)?(?!,\d))((?: hundred| thousand| (?:[bm]|tr|quadr)illion|k|m|b|t)*)\b",
188188
re.IGNORECASE,
189189
)
190190

191191
NUMBER_PATTERN = re.compile(
192-
r"(-?)(\d+(?:\.\d+)?)((?: hundred| thousand| (?:[bm]|tr|quadr)illion|k|m|b)*)\b",
192+
r"(-?)(\d{1,3}(?:,\d{3})+(?:\.\d+)?|\d+(?:\.\d+)?(?!,\d))((?: hundred| thousand| (?:[bm]|tr|quadr)illion|k|m|b)*)\b",
193193
re.IGNORECASE,
194194
)
195195

@@ -242,12 +242,14 @@ def split_four_digit(number: float):
242242

243243

244244
def handle_numbers(n: re.Match[str]) -> str:
245-
number = n.group(2)
245+
raw = n.group(2)
246246

247247
try:
248-
number = float(number)
248+
number = float(raw.replace(",", ""))
249249
except:
250250
return n.group()
251+
if not math.isfinite(number):
252+
return n.group()
251253

252254
if n.group(1) == "-":
253255
number *= -1
@@ -259,7 +261,8 @@ def handle_numbers(n: re.Match[str]) -> str:
259261
multiplier = f" {multiplier}"
260262
else:
261263
if (
262-
number % 1 == 0
264+
"," not in raw
265+
and number % 1 == 0
263266
and len(str(number)) == 4
264267
and number > 1500
265268
and number % 1000 > 9
@@ -274,12 +277,12 @@ def handle_money(m: re.Match[str]) -> str:
274277

275278
bill, coin = MONEY_UNITS[m.group(2)]
276279

277-
number = m.group(3)
278-
279280
try:
280-
number = float(number)
281+
number = float(m.group(3).replace(",", ""))
281282
except:
282283
return m.group()
284+
if not math.isfinite(number):
285+
return m.group()
283286

284287
if m.group(1) == "-":
285288
number *= -1
@@ -500,11 +503,19 @@ def normalize_text(text: str, normalization_options: NormalizationOptions) -> st
500503
text = re.sub(r"(?i)\b(y)eah?\b", r"\1e'a", text)
501504

502505
# Handle numbers and money BEFORE replacing special characters
503-
text = re.sub(r"(?<=\d),(?=\d)", "", text)
506+
text = re.sub(
507+
r"(?<!\d)\d+(?:,\d+)+",
508+
lambda m: (
509+
m.group()
510+
if re.fullmatch(r"\d{1,3}(?:,\d{3})+", m.group())
511+
else m.group().replace(",", "")
512+
),
513+
text,
514+
)
504515
text = re.sub(r"(?<=\d)-(?=\d)", " to ", text)
505516

506517
# version-like sequences (2.0.1, 10.3.2) before NUMBER_PATTERN eats the first decimal
507-
text = re.sub(r"\d+(?:\.\d+){2,}", handle_version, text)
518+
text = re.sub(r"(?<!\d)\d+(?:\.\d+){2,}", handle_version, text)
508519

509520
text = MONEY_PATTERN.sub(
510521
handle_money,

‎api/tests/test_normalizer.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -289,6 +289,32 @@ def test_number():
289289
== "The book has three hundred and forty-two pages."
290290
)
291291

292+
293+
def test_comma_grouped_number_is_not_a_year():
294+
"""Thousands separators rule out the year reading (#259)."""
295+
assert (
296+
normalize_text(
297+
"3,497 head of cattle", normalization_options=NormalizationOptions()
298+
)
299+
== "three thousand, four hundred and ninety-seven head of cattle"
300+
)
301+
assert (
302+
normalize_text("in 3497", normalization_options=NormalizationOptions())
303+
== "in thirty-four ninety-seven"
304+
)
305+
assert (
306+
normalize_text("$3,497.50", normalization_options=NormalizationOptions())
307+
== "three thousand, four hundred and ninety-seven dollars and fifty cents"
308+
)
309+
assert (
310+
normalize_text("the 3,497th item", normalization_options=NormalizationOptions())
311+
== "the 3,497th item"
312+
)
313+
assert (
314+
normalize_text("12,34", normalization_options=NormalizationOptions())
315+
== "one thousand, two hundred and thirty-four"
316+
)
317+
292318
assert (
293319
normalize_text(
294320
"He made -50 sales today.", normalization_options=NormalizationOptions()

‎api/tests/test_normalizer_redos.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,21 @@ def test_paragraph_split_is_fast():
6363
assert time.monotonic() - start < BUDGET_S
6464

6565

66+
def test_number_floods_scale_linearly():
67+
"""Comma runs and bare digit runs: the version regex went quadratic on the latter."""
68+
for unit in ("1", "1,", "123,", "123,4", "$1,"):
69+
small = _elapsed(unit * 5_000)
70+
large = _elapsed(unit * 20_000)
71+
assert large / max(small, 0.05) < 9.0, unit
72+
73+
74+
def test_huge_digit_run_does_not_raise():
75+
"""float() overflows to inf on 310+ digits, which used to crash the request."""
76+
assert normalize_text("9" * 400, OPTS) == "9" * 400
77+
assert normalize_text("$" + "9" * 400, OPTS).startswith(" dollar ")
78+
assert normalize_text("123," * 400, OPTS)
79+
80+
6681
def test_decimal_normalization_preserved():
6782
"""the lookbehind must not break normal decimal rendering."""
6883
out = normalize_text("costs 3.14 dollars", OPTS)

0 commit comments

Comments
 (0)