Skip to content

Commit fbbaa75

Browse files
fix: preserve paragraph breaks as sentence boundaries in normalizer (#525)
The normalizer replaced all newlines with spaces unconditionally, merging paragraph-separated sections (headings, chapter titles, body text) into one continuous phrase with no prosodic pause. Convert double-newline paragraph breaks to a period before the newline-to-space pass: if the preceding non-whitespace character is already sentence-ending punctuation the period is omitted to avoid duplication. Single newlines continue to become spaces. Add tests covering the exact failure case from the issue report, no- double-punctuation, single newline, and multiple consecutive blank lines. Fixes #519 Signed-off-by: Christian-Sidak <[email protected]>
1 parent 34d694c commit fbbaa75

2 files changed

Lines changed: 63 additions & 1 deletion

File tree

‎api/src/services/text_processing/normalizer.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -485,7 +485,13 @@ def normalize_text(text: str, normalization_options: NormalizationOptions) -> st
485485
text = re.sub(r"(?<=\n) +(?=\n)", "", text)
486486

487487
# Handle special characters that might cause audio artifacts first
488-
# Replace newlines with spaces (or pauses if needed)
488+
# Paragraph breaks (\n\n) are converted to a period so the TTS engine
489+
# treats them as sentence boundaries and produces a natural pause.
490+
# Only insert a period when the preceding non-whitespace character is not
491+
# already sentence-ending punctuation, avoiding double punctuation.
492+
text = re.sub(r"([^\.\!\?])\n\n+", r"\1. ", text)
493+
text = re.sub(r"([\.\!\?])\n\n+", r"\1 ", text)
494+
# Single newlines become spaces.
489495
text = text.replace("\n", " ")
490496
text = text.replace("\r", " ")
491497

‎api/tests/test_normalizer.py‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -394,3 +394,59 @@ def test_re_contraction_expansion_without_apostrophe():
394394
)
395395
== "Therefore the genre was hardcore before more."
396396
)
397+
398+
399+
def test_paragraph_breaks_preserved_as_sentence_boundary():
400+
"""Paragraph breaks (\\n\\n) must become a sentence-ending period so the
401+
TTS engine inserts a natural pause between sections. Without this fix the
402+
normalizer merged heading and body text into one continuous phrase."""
403+
# Bare heading followed by body text
404+
assert (
405+
normalize_text(
406+
"Steven Erikson\n\nFive riders drew rein in the pass.",
407+
normalization_options=NormalizationOptions(),
408+
)
409+
== "Steven Erikson. Five riders drew rein in the pass."
410+
)
411+
412+
413+
def test_paragraph_break_no_double_punctuation():
414+
"""When the paragraph already ends with sentence punctuation, no extra
415+
period should be inserted."""
416+
assert (
417+
normalize_text(
418+
"End of chapter.\n\nNew chapter begins.",
419+
normalization_options=NormalizationOptions(),
420+
)
421+
== "End of chapter. New chapter begins."
422+
)
423+
assert (
424+
normalize_text(
425+
"Really?\n\nYes.",
426+
normalization_options=NormalizationOptions(),
427+
)
428+
== "Really? Yes."
429+
)
430+
431+
432+
def test_single_newline_becomes_space():
433+
"""A single newline (soft line break) is still collapsed to a space."""
434+
assert (
435+
normalize_text(
436+
"Line one\nLine two",
437+
normalization_options=NormalizationOptions(),
438+
)
439+
== "Line one Line two"
440+
)
441+
442+
443+
def test_multiple_blank_lines_treated_as_paragraph_break():
444+
"""Three or more consecutive newlines should be treated like a single
445+
paragraph break, not produce extra periods."""
446+
assert (
447+
normalize_text(
448+
"Heading\n\n\nBody text.",
449+
normalization_options=NormalizationOptions(),
450+
)
451+
== "Heading. Body text."
452+
)

0 commit comments

Comments
 (0)