feat: Kokoro voice blending — blend builder UI + weighted tensor synthesis

Add voice blend support to TTS pipeline and settings UI. Users can mix
2–5 Kokoro voices with per-voice weight sliders; the blended style tensor
replaces the single voice when enabled. Settings persist as JSON and auto-
load on synthesis when no explicit voice is supplied in the request.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-30 19:10:22 -04:00
parent 98b3cdb593
commit 1460863e82
4 changed files with 237 additions and 7 deletions
+20 -1
View File
@@ -126,8 +126,27 @@ async def synthesise_speech():
except (TypeError, ValueError):
speed = 1.0
voice_blend = data.get("voice_blend")
if not isinstance(voice_blend, list) or len(voice_blend) < 2:
voice_blend = None
# Auto-load blend from user settings when no explicit voice/blend provided
if voice_blend is None and "voice" not in data and "voice_blend" not in data:
from fabledassistant.services.settings import get_setting
from fabledassistant.auth import get_current_user_id
import json as _json
try:
uid = get_current_user_id()
stored = await get_setting(uid, "voice_tts_blend", "")
if stored:
parsed = _json.loads(stored)
if isinstance(parsed, list) and len(parsed) >= 2:
voice_blend = parsed
except Exception:
pass
try:
wav_bytes = await synthesise(text, voice=voice, speed=speed)
wav_bytes = await synthesise(text, voice=voice, speed=speed, voice_blend=voice_blend)
except Exception:
logger.exception("TTS synthesis failed")
return jsonify({"error": "Synthesis failed"}), 500
+28 -3
View File
@@ -73,20 +73,45 @@ def list_voices() -> list[dict]:
return _VOICES
async def synthesise(text: str, voice: str = "af_heart", speed: float = 1.0) -> bytes:
"""Synthesise text to WAV bytes (24kHz, 16-bit mono). Runs in executor."""
async def synthesise(
text: str,
voice: str = "af_heart",
speed: float = 1.0,
voice_blend: list[dict] | None = None,
) -> bytes:
"""Synthesise text to WAV bytes (24kHz, 16-bit mono). Runs in executor.
voice_blend is a list of {"voice": str, "weight": float} dicts.
When provided with 2+ entries the voice style tensors are merged as a
weighted average before synthesis. Weights are normalised automatically.
"""
if _pipeline is None:
raise RuntimeError("TTS pipeline not loaded")
speed = max(0.7, min(1.3, speed))
def _build_voice_param():
"""Return either a blended style tensor or a single voice ID string."""
if not voice_blend or len(voice_blend) < 2:
return voice_blend[0]["voice"] if voice_blend else voice
import numpy as np
total_w = sum(max(0.0, e.get("weight", 1.0)) for e in voice_blend) or 1.0
blended = None
for entry in voice_blend:
vid = entry.get("voice", "af_heart")
w = max(0.0, entry.get("weight", 1.0)) / total_w
vt = _pipeline.load_voice(vid) # type: ignore[union-attr]
blended = vt * w if blended is None else blended + vt * w
return blended
def _run() -> bytes:
import numpy as np
import soundfile as sf
voice_param = _build_voice_param()
t0 = time.monotonic()
audio_chunks: list = []
for _, _, audio in _pipeline(text, voice=voice, speed=speed): # type: ignore[misc]
for _, _, audio in _pipeline(text, voice=voice_param, speed=speed): # type: ignore[misc]
if audio is not None:
audio_chunks.append(audio)