feat: Kokoro voice blending — blend builder UI + weighted tensor synthesis
Add voice blend support to TTS pipeline and settings UI. Users can mix 2–5 Kokoro voices with per-voice weight sliders; the blended style tensor replaces the single voice when enabled. Settings persist as JSON and auto- load on synthesis when no explicit voice is supplied in the request. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -126,8 +126,27 @@ async def synthesise_speech():
|
||||
except (TypeError, ValueError):
|
||||
speed = 1.0
|
||||
|
||||
voice_blend = data.get("voice_blend")
|
||||
if not isinstance(voice_blend, list) or len(voice_blend) < 2:
|
||||
voice_blend = None
|
||||
|
||||
# Auto-load blend from user settings when no explicit voice/blend provided
|
||||
if voice_blend is None and "voice" not in data and "voice_blend" not in data:
|
||||
from fabledassistant.services.settings import get_setting
|
||||
from fabledassistant.auth import get_current_user_id
|
||||
import json as _json
|
||||
try:
|
||||
uid = get_current_user_id()
|
||||
stored = await get_setting(uid, "voice_tts_blend", "")
|
||||
if stored:
|
||||
parsed = _json.loads(stored)
|
||||
if isinstance(parsed, list) and len(parsed) >= 2:
|
||||
voice_blend = parsed
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
wav_bytes = await synthesise(text, voice=voice, speed=speed)
|
||||
wav_bytes = await synthesise(text, voice=voice, speed=speed, voice_blend=voice_blend)
|
||||
except Exception:
|
||||
logger.exception("TTS synthesis failed")
|
||||
return jsonify({"error": "Synthesis failed"}), 500
|
||||
|
||||
@@ -73,20 +73,45 @@ def list_voices() -> list[dict]:
|
||||
return _VOICES
|
||||
|
||||
|
||||
async def synthesise(text: str, voice: str = "af_heart", speed: float = 1.0) -> bytes:
|
||||
"""Synthesise text to WAV bytes (24kHz, 16-bit mono). Runs in executor."""
|
||||
async def synthesise(
|
||||
text: str,
|
||||
voice: str = "af_heart",
|
||||
speed: float = 1.0,
|
||||
voice_blend: list[dict] | None = None,
|
||||
) -> bytes:
|
||||
"""Synthesise text to WAV bytes (24kHz, 16-bit mono). Runs in executor.
|
||||
|
||||
voice_blend is a list of {"voice": str, "weight": float} dicts.
|
||||
When provided with 2+ entries the voice style tensors are merged as a
|
||||
weighted average before synthesis. Weights are normalised automatically.
|
||||
"""
|
||||
if _pipeline is None:
|
||||
raise RuntimeError("TTS pipeline not loaded")
|
||||
|
||||
speed = max(0.7, min(1.3, speed))
|
||||
|
||||
def _build_voice_param():
|
||||
"""Return either a blended style tensor or a single voice ID string."""
|
||||
if not voice_blend or len(voice_blend) < 2:
|
||||
return voice_blend[0]["voice"] if voice_blend else voice
|
||||
import numpy as np
|
||||
total_w = sum(max(0.0, e.get("weight", 1.0)) for e in voice_blend) or 1.0
|
||||
blended = None
|
||||
for entry in voice_blend:
|
||||
vid = entry.get("voice", "af_heart")
|
||||
w = max(0.0, entry.get("weight", 1.0)) / total_w
|
||||
vt = _pipeline.load_voice(vid) # type: ignore[union-attr]
|
||||
blended = vt * w if blended is None else blended + vt * w
|
||||
return blended
|
||||
|
||||
def _run() -> bytes:
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
|
||||
voice_param = _build_voice_param()
|
||||
t0 = time.monotonic()
|
||||
audio_chunks: list = []
|
||||
for _, _, audio in _pipeline(text, voice=voice, speed=speed): # type: ignore[misc]
|
||||
for _, _, audio in _pipeline(text, voice=voice_param, speed=speed): # type: ignore[misc]
|
||||
if audio is not None:
|
||||
audio_chunks.append(audio)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user