Skip to content

Commit c0acb91

Browse files
mlevinmailclaude
andauthored
fix: word timestamps for non-English (espeak) voices in captioned_speech (#484)
KPipeline only attaches timed tokens for English (misaki) voices, so /dev/captioned_speech silently returned timestamps: null for Spanish, French, Italian, Hindi and Portuguese. The model does predict a duration for every phoneme in every language (pred_dur) - the audio is rendered from exactly those durations - but the espeak path discarded them. Derive word timestamps for espeak pipelines from pred_dur directly: pred_dur[0] is BOS and pred_dur[1+i] covers phonemes[i], so summing per-character durations and splitting at the phoneme string's spaces yields exact word times. Words espeak expands into several spoken words (e.g. "1863") are reconciled by re-phonemizing per word; if counts still disagree, no timestamps are emitted (previous behavior). English is untouched, and ja/zh (non-espeak G2Ps, no space-separated word groups) deliberately keep the previous behavior. Fixes #331 Co-authored-by: Claude Fable 5 <[email protected]>
1 parent 24ac58e commit c0acb91

2 files changed

Lines changed: 149 additions & 0 deletions

File tree

‎api/src/inference/kokoro_v1.py‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,69 @@
1414
from ..structures.schemas import WordTimestamp
1515
from .base import AudioChunk, BaseModelBackend
1616

17+
_ESPEAK_TS_SCALE = 2.0 / 80.0 # pred_dur unit -> seconds (matches KPipeline.join_timestamps)
18+
19+
20+
def _espeak_word_timestamps(graphemes, phonemes, pred_dur, g2p=None):
21+
"""Derive word timestamps for espeak-based (non-English) pipelines.
22+
23+
KPipeline attaches timed tokens only for English (misaki G2P) voices, but
24+
the model predicts a duration for every phoneme in every language — the
25+
audio is rendered from exactly those durations. pred_dur[0] is BOS and
26+
pred_dur[1 + i] covers phonemes[i], so summing per-character durations and
27+
splitting at the phoneme string's spaces yields exact word times.
28+
29+
Words espeak expands into several spoken words (numbers, some
30+
abbreviations) are reconciled by re-phonemizing per word via `g2p`; if the
31+
counts still disagree, returns None so the caller emits no timestamps and
32+
clients can fall back to their own handling.
33+
"""
34+
try:
35+
if pred_dur is None or not phonemes or len(pred_dur) < len(phonemes) + 1:
36+
return None
37+
words = graphemes.split()
38+
if not words:
39+
return None
40+
groups = [] # [start, end) seconds per space-separated phoneme group
41+
t = float(pred_dur[0]) * _ESPEAK_TS_SCALE
42+
start = None
43+
for i, ch in enumerate(phonemes):
44+
d = float(pred_dur[1 + i]) * _ESPEAK_TS_SCALE
45+
if ch.isspace():
46+
if start is not None:
47+
groups.append((start, t))
48+
start = None
49+
elif start is None:
50+
start = t
51+
t += d
52+
if start is not None:
53+
groups.append((start, t))
54+
55+
if len(groups) != len(words):
56+
if g2p is None:
57+
return None
58+
counts = []
59+
for w in words:
60+
ps = g2p(w)
61+
if isinstance(ps, tuple):
62+
ps = ps[0]
63+
counts.append(max(len((ps or "").split()), 1))
64+
if sum(counts) != len(groups):
65+
return None
66+
merged, gi = [], 0
67+
for c in counts:
68+
merged.append((groups[gi][0], groups[gi + c - 1][1]))
69+
gi += c
70+
groups = merged
71+
72+
return [
73+
WordTimestamp(word=w, start_time=round(s, 3), end_time=round(e, 3))
74+
for w, (s, e) in zip(words, groups)
75+
]
76+
except Exception as e:
77+
logger.warning(f"espeak timestamp mapping failed: {e}")
78+
return None
79+
1780

1881
class KokoroV1(BaseModelBackend):
1982
"""Kokoro backend with controlled resource management."""
@@ -326,6 +389,29 @@ async def generate(
326389
logger.error(
327390
f"Failed to process timestamps for chunk: {e}"
328391
)
392+
elif (
393+
return_timestamps
394+
and result.phonemes
395+
and type(getattr(pipeline, "g2p", None)).__name__
396+
== "EspeakG2P"
397+
):
398+
# espeak pipelines (es/fr/it/hi/pt) yield no timed
399+
# tokens; derive word times from the model's own
400+
# phoneme durations instead. Espeak-only: other
401+
# non-English G2Ps (ja/zh) have no space-separated
402+
# word groups to map.
403+
pred_dur = getattr(result, "pred_dur", None)
404+
if (
405+
pred_dur is None
406+
and getattr(result, "output", None) is not None
407+
):
408+
pred_dur = getattr(result.output, "pred_dur", None)
409+
word_timestamps = _espeak_word_timestamps(
410+
result.graphemes,
411+
result.phonemes,
412+
pred_dur,
413+
g2p=getattr(pipeline, "g2p", None),
414+
)
329415

330416
yield AudioChunk(
331417
result.audio.numpy(), word_timestamps=word_timestamps

‎api/tests/test_kokoro_v1.py‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,3 +158,66 @@ async def test_generate_uses_correct_pipeline(kokoro_backend):
158158
voice_arg = Path(mock_pipeline.call_args[1]["voice"])
159159
assert voice_arg.name == "temp_voice_ef_voice"
160160
assert voice_arg.parent == Path(tempfile.gettempdir())
161+
162+
163+
def test_espeak_word_timestamps_basic():
164+
"""Word times derived from pred_dur for an espeak (non-English) result."""
165+
from api.src.inference.kokoro_v1 import _espeak_word_timestamps
166+
167+
# phonemes "ab cd": BOS=8, a=4, b=4, space=8, c=4, d=4, EOS=0
168+
# scale is 2/80 s per unit -> BOS 0.2s, each char 0.1s, space 0.2s
169+
pred_dur = torch.tensor([8, 4, 4, 8, 4, 4, 0])
170+
ts = _espeak_word_timestamps("hola mundo", "ab cd", pred_dur)
171+
172+
assert [t.word for t in ts] == ["hola", "mundo"]
173+
assert ts[0].start_time == pytest.approx(0.2)
174+
assert ts[0].end_time == pytest.approx(0.4)
175+
assert ts[1].start_time == pytest.approx(0.6)
176+
assert ts[1].end_time == pytest.approx(0.8)
177+
178+
179+
def test_espeak_word_timestamps_expansion():
180+
"""Words espeak expands (numbers) are merged back via per-word g2p."""
181+
from api.src.inference.kokoro_v1 import _espeak_word_timestamps
182+
183+
# Three text words but four phoneme groups ("1863" speaks as two words).
184+
phonemes = "a bb cc d"
185+
pred_dur = torch.tensor([0, 4] + [8, 4, 4] + [8, 4, 4] + [8, 4] + [0])
186+
187+
def g2p(word):
188+
return ("bb cc" if word == "1863" else "x", None)
189+
190+
ts = _espeak_word_timestamps("en 1863 el", phonemes, pred_dur, g2p=g2p)
191+
192+
assert [t.word for t in ts] == ["en", "1863", "el"]
193+
# "1863" spans both expanded groups.
194+
assert ts[1].start_time == pytest.approx(0.3)
195+
assert ts[1].end_time == pytest.approx(0.9)
196+
# Whole-string timing stays monotonic and inside the clip.
197+
assert ts[0].end_time <= ts[1].start_time <= ts[2].start_time
198+
199+
200+
def test_espeak_word_timestamps_unreconcilable():
201+
"""Group/word count mismatch that g2p can't explain yields None."""
202+
from api.src.inference.kokoro_v1 import _espeak_word_timestamps
203+
204+
phonemes = "a b c"
205+
pred_dur = torch.tensor([0, 4, 8, 4, 8, 4, 0])
206+
207+
# g2p says every word is a single group: 2 != 3 -> give up.
208+
ts = _espeak_word_timestamps("uno dos", phonemes, pred_dur, g2p=lambda w: (w, None))
209+
assert ts is None
210+
211+
# No g2p available -> also None.
212+
assert _espeak_word_timestamps("uno dos", phonemes, pred_dur) is None
213+
214+
215+
def test_espeak_word_timestamps_degenerate_inputs():
216+
"""Missing durations or empty text return None instead of raising."""
217+
from api.src.inference.kokoro_v1 import _espeak_word_timestamps
218+
219+
assert _espeak_word_timestamps("hola", "abc", None) is None
220+
assert _espeak_word_timestamps("hola", "", torch.tensor([0, 0])) is None
221+
assert _espeak_word_timestamps("", "abc", torch.tensor([0, 4, 4, 4, 0])) is None
222+
# pred_dur too short for the phoneme string.
223+
assert _espeak_word_timestamps("hola", "abcdef", torch.tensor([0, 4])) is None

0 commit comments

Comments
 (0)