Repository navigation
Expand file tree
/
Copy pathkokoro_v1.py
More file actions
540 lines (475 loc) · 21 KB
/
Copy pathkokoro_v1.py
File metadata and controls
540 lines (475 loc) · 21 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
"""Clean Kokoro implementation with controlled resource management."""
import asyncio
import os
import threading
from collections import OrderedDict
from typing import AsyncGenerator, Dict, Optional, Tuple, Union
import aiofiles.os
import numpy as np
import torch
from kokoro import KModel, KPipeline
from loguru import logger
from ..core import paths
from ..core.config import settings
from ..core.model_config import model_config
from ..structures.schemas import WordTimestamp
from .base import AudioChunk, BaseModelBackend
# Distinct voices and mixes held in memory; each is about half a megabyte
VOICE_CACHE_SIZE = 64
_ESPEAK_TS_SCALE = (
2.0 / 80.0
) # pred_dur unit -> seconds (matches KPipeline.join_timestamps)
def _espeak_word_timestamps(graphemes, phonemes, pred_dur, g2p=None):
"""Derive word timestamps for espeak-based (non-English) pipelines.
KPipeline attaches timed tokens only for English (misaki G2P) voices, but
the model predicts a duration for every phoneme in every language (the
audio is rendered from exactly those durations). pred_dur[0] is BOS and
pred_dur[1 + i] covers phonemes[i], so summing per-character durations and
splitting at the phoneme string's spaces yields exact word times.
Words espeak expands into several spoken words (numbers, some
abbreviations) are reconciled by re-phonemizing per word via `g2p`; if the
counts still disagree, returns None so the caller emits no timestamps and
clients can fall back to their own handling.
"""
try:
if pred_dur is None or not phonemes or len(pred_dur) < len(phonemes) + 1:
return None
words = graphemes.split()
if not words:
return None
durations = [dur * _ESPEAK_TS_SCALE for dur in pred_dur.tolist()]
groups = [] # [start, end) seconds per space-separated phoneme group
now = durations[0]
start = None
for i, ch in enumerate(phonemes):
if ch.isspace():
if start is not None:
groups.append((start, now))
start = None
elif start is None:
start = now
now += durations[1 + i]
if start is not None:
groups.append((start, now))
if len(groups) != len(words):
if g2p is None:
return None
counts = []
for word in words:
phones = g2p(word)
if isinstance(phones, tuple):
phones = phones[0]
counts.append(max(len((phones or "").split()), 1))
if sum(counts) != len(groups):
return None
merged, group_idx = [], 0
for count in counts:
merged.append((groups[group_idx][0], groups[group_idx + count - 1][1]))
group_idx += count
groups = merged
return [
WordTimestamp(word=word, start_time=round(start, 3), end_time=round(end, 3))
for word, (start, end) in zip(words, groups)
]
except Exception as e:
logger.warning(f"espeak timestamp mapping failed: {e}")
return None
def _configure_rocm_backend() -> None:
"""Disable MIOpen on ROCm.
MIOpen compiles a kernel per unseen tensor shape, and Kokoro's decoder
length varies with the input text, so nearly every request pays that
compile: 2433ms per generation against 268ms without, on gfx1100.
warmup_miopen.py does not help, as shape is not a function of phoneme
count.
ENABLE_MIOPEN=true keeps MIOpen. CUDA is unaffected.
"""
if not torch.version.hip:
return
if settings.enable_miopen:
logger.info("MIOpen left enabled via ENABLE_MIOPEN")
return
torch.backends.cudnn.enabled = False
logger.info("ROCm detected: MIOpen disabled to avoid per-shape kernel compilation")
class KokoroV1(BaseModelBackend):
"""Kokoro backend with controlled resource management."""
def __init__(self):
"""Initialize backend with environment-based configuration."""
super().__init__()
# Strictly respect settings.use_gpu
self._device = settings.get_device()
self._model: Optional[KModel] = None
self._pipelines: Dict[str, KPipeline] = {} # Store pipelines by lang_code
self._voice_cache: OrderedDict[str, torch.Tensor] = (
OrderedDict()
) # LRU of voice tensors by path
self._inference_lock = threading.Lock() # one model call at a time
def _next_locked(self, results, done):
with self._inference_lock:
return next(results, done)
async def _iterate_off_loop(self, results):
"""Step a pipeline generator in a worker thread so the event loop keeps serving."""
done = object()
while (
result := await asyncio.to_thread(self._next_locked, results, done)
) is not done:
yield result
async def _get_voice_tensor(self, voice_path: str) -> torch.Tensor:
"""Load voice tensor with in-memory caching to avoid repeated file I/O.
Args:
voice_path: Path to the .pt voice file
Returns:
Voice tensor on the target device
"""
cache_key = f"{voice_path}:{self._device}"
if cache_key in self._voice_cache:
self._voice_cache.move_to_end(cache_key)
else:
self._voice_cache[cache_key] = await paths.load_voice_tensor(
voice_path, device=self._device
)
logger.debug(f"Cached voice tensor from {voice_path}")
while len(self._voice_cache) > VOICE_CACHE_SIZE:
self._voice_cache.popitem(last=False)
return self._voice_cache[cache_key]
def forget_voice(self, voice_path: str) -> None:
"""Drop a pack's cached tensor."""
for key in [k for k in self._voice_cache if k.startswith(f"{voice_path}:")]:
del self._voice_cache[key]
async def load_model(self, path: str) -> None:
"""Load pre-baked model.
Args:
path: Path to model file
Raises:
RuntimeError: If model loading fails
"""
try:
# Get verified model path
model_path = await paths.get_model_path(path)
config_path = os.path.join(os.path.dirname(model_path), "config.json")
if not await aiofiles.os.path.exists(config_path):
raise RuntimeError(f"Config file not found: {config_path}")
logger.info(f"Loading Kokoro model on {self._device}")
logger.info(f"Config path: {config_path}")
logger.info(f"Model path: {model_path}")
# Load model and let KModel handle device mapping
self._model = KModel(
repo_id=settings.model_repo_id, config=config_path, model=model_path
).eval()
# For MPS, manually move ISTFT layers to CPU while keeping rest on MPS
if self._device == "mps":
logger.info(
"Moving model to MPS device with CPU fallback for unsupported operations"
)
self._model = self._model.to(torch.device("mps"))
elif self._device == "cuda":
# ROCm reports device "cuda", so this is the ROCm path too.
_configure_rocm_backend()
self._model = self._model.cuda()
else:
self._model = self._model.cpu()
except FileNotFoundError:
raise
except Exception as e:
raise RuntimeError(f"Failed to load Kokoro model: {e}")
def _get_pipeline(self, lang_code: str) -> KPipeline:
"""Get or create pipeline for language code.
Args:
lang_code: Language code to use
Returns:
KPipeline instance for the language
"""
if not self._model:
raise RuntimeError("Model not loaded")
if lang_code not in self._pipelines:
logger.info(f"Creating new pipeline for language code: {lang_code}")
self._pipelines[lang_code] = KPipeline(
lang_code=lang_code,
repo_id=settings.model_repo_id,
model=self._model,
device=self._device,
)
return self._pipelines[lang_code]
async def generate_from_tokens(
self,
tokens: str,
voice: Union[str, Tuple[str, Union[torch.Tensor, str]]],
speed: float = 1.0,
lang_code: Optional[str] = None,
_retry_oom: bool = True,
) -> AsyncGenerator[np.ndarray, None]:
"""Generate audio from phoneme tokens.
Args:
tokens: Input phoneme tokens to synthesize
voice: Either a voice path string or a tuple of (voice_name, voice_tensor/path)
speed: Speed multiplier
lang_code: Optional language code override
Yields:
Generated audio chunks
Raises:
RuntimeError: If generation fails
"""
if not self.is_loaded:
raise RuntimeError("Model not loaded")
yielded = False
try:
# Memory management for GPU
if self._device in ("cuda", "mps"):
if self._check_memory():
self._clear_memory()
# Handle voice input
voice_path: str
voice_name: str
if isinstance(voice, tuple):
voice_name, voice_data = voice
if isinstance(voice_data, str):
voice_path = voice_data
else:
# Save tensor to temporary file
import tempfile
temp_dir = tempfile.gettempdir()
voice_path = os.path.join(temp_dir, f"{voice_name}.pt")
# Save tensor with CPU mapping for portability
torch.save(voice_data.cpu(), voice_path)
else:
voice_path = voice
voice_name = os.path.splitext(os.path.basename(voice_path))[0]
# Load voice tensor with caching to avoid repeated file I/O
voice_tensor = await self._get_voice_tensor(voice_path)
# KPipeline uses a tensor as-is only when it is a CPU FloatTensor; a path
# would be cached in pipeline.voices for the life of the process
voice_pack = voice_tensor.cpu()
# Use provided lang_code, settings voice code override, or first letter of voice name
if lang_code: # api is given priority
pipeline_lang_code = lang_code
elif settings.default_voice_code: # settings is next priority
pipeline_lang_code = settings.default_voice_code
else: # voice name is default/fallback
pipeline_lang_code = voice_name[0].lower()
pipeline = self._get_pipeline(pipeline_lang_code)
logger.debug(
f"Generating audio from tokens with lang_code '{pipeline_lang_code}': '{tokens[:100]}{'...' if len(tokens) > 100 else ''}'"
)
async for result in self._iterate_off_loop(
pipeline.generate_from_tokens(
tokens=tokens, voice=voice_pack, speed=speed, model=self._model
)
):
if result.audio is not None:
logger.debug(f"Got audio chunk with shape: {result.audio.shape}")
yielded = True
yield result.audio.numpy()
else:
logger.warning("No audio in chunk")
except Exception as e:
logger.error(f"Generation failed: {e}")
# Retry once, and only before any audio went out, so a chunk is never repeated
if (
_retry_oom
and not yielded
and self._device == "cuda"
and model_config.pytorch_gpu.retry_on_oom
and "out of memory" in str(e).lower()
):
self._clear_memory()
async for chunk in self.generate_from_tokens(
tokens, voice, speed, lang_code, _retry_oom=False
):
yield chunk
return
raise
async def generate(
self,
text: str,
voice: Union[str, Tuple[str, Union[torch.Tensor, str]]],
speed: float = 1.0,
lang_code: Optional[str] = None,
return_timestamps: Optional[bool] = False,
_retry_oom: bool = True,
) -> AsyncGenerator[AudioChunk, None]:
"""Generate audio using model.
Args:
text: Input text to synthesize
voice: Either a voice path string or a tuple of (voice_name, voice_tensor/path)
speed: Speed multiplier
lang_code: Optional language code override
Yields:
Generated audio chunks
Raises:
RuntimeError: If generation fails
"""
if not self.is_loaded:
raise RuntimeError("Model not loaded")
yielded = False
try:
# Memory management for GPU
if self._device in ("cuda", "mps"):
if self._check_memory():
self._clear_memory()
# Handle voice input
voice_path: str
voice_name: str
if isinstance(voice, tuple):
voice_name, voice_data = voice
if isinstance(voice_data, str):
voice_path = voice_data
else:
# Save tensor to temporary file
import tempfile
temp_dir = tempfile.gettempdir()
voice_path = os.path.join(temp_dir, f"{voice_name}.pt")
# Save tensor with CPU mapping for portability
torch.save(voice_data.cpu(), voice_path)
else:
voice_path = voice
voice_name = os.path.splitext(os.path.basename(voice_path))[0]
# Load voice tensor with caching to avoid repeated file I/O
voice_tensor = await self._get_voice_tensor(voice_path)
# KPipeline uses a tensor as-is only when it is a CPU FloatTensor; a path
# would be cached in pipeline.voices for the life of the process
voice_pack = voice_tensor.cpu()
# Use provided lang_code, settings voice code override, or first letter of voice name
pipeline_lang_code = (
lang_code
if lang_code
else (
settings.default_voice_code
if settings.default_voice_code
else voice_name[0].lower()
)
)
pipeline = self._get_pipeline(pipeline_lang_code)
logger.debug(
f"Generating audio for text with lang_code '{pipeline_lang_code}': '{text[:100]}{'...' if len(text) > 100 else ''}'"
)
async for result in self._iterate_off_loop(
pipeline(text, voice=voice_pack, speed=speed, model=self._model)
):
if result.audio is not None:
logger.debug(f"Got audio chunk with shape: {result.audio.shape}")
word_timestamps = None
if (
return_timestamps
and hasattr(result, "tokens")
and result.tokens
):
word_timestamps = []
current_offset = 0.0
logger.debug(
f"Processing chunk timestamps with {len(result.tokens)} tokens"
)
if result.pred_dur is not None:
try:
# Add timestamps with offset
for token in result.tokens:
if not all(
hasattr(token, attr)
for attr in [
"text",
"start_ts",
"end_ts",
]
):
continue
if not token.text or not token.text.strip():
continue
if token.start_ts is None or token.end_ts is None:
continue
start_time = float(token.start_ts) + current_offset
end_time = float(token.end_ts) + current_offset
word_timestamps.append(
WordTimestamp(
word=str(token.text).strip(),
start_time=start_time,
end_time=end_time,
)
)
logger.debug(
f"Added timestamp for word '{token.text}': {start_time:.3f}s - {end_time:.3f}s"
)
except Exception as e:
logger.error(
f"Failed to process timestamps for chunk: {e}"
)
elif (
return_timestamps
and result.phonemes
and type(getattr(pipeline, "g2p", None)).__name__ == "EspeakG2P"
):
# espeak pipelines (es/fr/it/hi/pt) yield no timed
# tokens; derive word times from the model's own
# phoneme durations instead. Espeak-only: other
# non-English G2Ps (ja/zh) have no space-separated
# word groups to map.
pred_dur = getattr(result, "pred_dur", None)
if (
pred_dur is None
and getattr(result, "output", None) is not None
):
pred_dur = getattr(result.output, "pred_dur", None)
word_timestamps = _espeak_word_timestamps(
result.graphemes,
result.phonemes,
pred_dur,
g2p=getattr(pipeline, "g2p", None),
)
yielded = True
yield AudioChunk(
result.audio.numpy(), word_timestamps=word_timestamps
)
else:
logger.warning("No audio in chunk")
except Exception as e:
logger.error(f"Generation failed: {e}")
if (
_retry_oom
and not yielded
and self._device == "cuda"
and model_config.pytorch_gpu.retry_on_oom
and "out of memory" in str(e).lower()
):
self._clear_memory()
async for chunk in self.generate(
text, voice, speed, lang_code, return_timestamps, _retry_oom=False
):
yield chunk
return
raise
def _check_memory(self) -> bool:
"""Check if memory usage is above threshold."""
if self._device == "cuda":
memory_gb = torch.cuda.memory_allocated() / 1e9
return memory_gb > model_config.pytorch_gpu.memory_threshold
if self._device == "mps":
# Counts the allocator's cache, which is what grows.
memory_gb = torch.mps.driver_allocated_memory() / 1e9
return memory_gb > model_config.pytorch_gpu.memory_threshold
return False
def _clear_memory(self) -> None:
"""Clear device memory."""
if self._device == "cuda":
torch.cuda.empty_cache()
torch.cuda.synchronize()
elif self._device == "mps":
# Empty cache if available (future-proofing)
if hasattr(torch.mps, "empty_cache"):
torch.mps.empty_cache()
def unload(self) -> None:
"""Unload model and free resources."""
if self._model is not None:
del self._model
self._model = None
for pipeline in self._pipelines.values():
del pipeline
self._pipelines.clear()
self._voice_cache.clear()
if torch.cuda.is_available():
torch.cuda.empty_cache()
torch.cuda.synchronize()
@property
def is_loaded(self) -> bool:
"""Check if model is loaded."""
return self._model is not None
@property
def device(self) -> str:
"""Get device model is running on."""
return self._device