From 1460863e828c956bdd4393a07f496c8c9733b3d9 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 30 Mar 2026 19:10:22 -0400 Subject: [PATCH] =?UTF-8?q?feat:=20Kokoro=20voice=20blending=20=E2=80=94?= =?UTF-8?q?=20blend=20builder=20UI=20+=20weighted=20tensor=20synthesis?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add voice blend support to TTS pipeline and settings UI. Users can mix 2–5 Kokoro voices with per-voice weight sliders; the blended style tensor replaces the single voice when enabled. Settings persist as JSON and auto- load on synthesis when no explicit voice is supplied in the request. Co-Authored-By: Claude Sonnet 4.6 --- frontend/src/api/client.ts | 16 ++- frontend/src/views/SettingsView.vue | 176 +++++++++++++++++++++++++++- src/fabledassistant/routes/voice.py | 21 +++- src/fabledassistant/services/tts.py | 31 ++++- 4 files changed, 237 insertions(+), 7 deletions(-) diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 411ad58..d9ffc25 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -639,6 +639,11 @@ export interface VoiceEntry { label: string } +export interface VoiceBlendEntry { + voice: string + weight: number +} + export const getVoiceStatus = () => apiGet('/api/voice/status') export const getVoiceList = () => @@ -658,12 +663,19 @@ export async function transcribeAudio(blob: Blob): Promise<{ transcript: string; export async function synthesiseSpeech( text: string, voice?: string, - speed?: number + speed?: number, + voiceBlend?: VoiceBlendEntry[] ): Promise { + const body: Record = { text, speed: speed ?? 1.0 } + if (voiceBlend && voiceBlend.length >= 2) { + body.voice_blend = voiceBlend + } else { + body.voice = voice ?? 'af_heart' + } const res = await fetch('/api/voice/synthesise', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ text, voice: voice ?? 'af_heart', speed: speed ?? 1.0 }), + body: JSON.stringify(body), }) if (!res.ok) { const err = await res.json().catch(() => ({ error: `HTTP ${res.status}` })) diff --git a/frontend/src/views/SettingsView.vue b/frontend/src/views/SettingsView.vue index 018166b..e4db167 100644 --- a/frontend/src/views/SettingsView.vue +++ b/frontend/src/views/SettingsView.vue @@ -3,7 +3,7 @@ import { ref, watch, onMounted } from "vue"; import { useSettingsStore } from "@/stores/settings"; import { useAuthStore } from "@/stores/auth"; import { useToastStore } from "@/stores/toast"; -import { apiGet, apiPost, apiPut, apiDelete, listGroups, createGroup, deleteGroup, listGroupMembers, addGroupMember, removeGroupMember, searchUsers, getBriefingConfig, saveBriefingConfig, getBriefingFeeds, createBriefingFeed, deleteBriefingFeed, refreshBriefingFeeds, geocodeAddress, getFableMcpInfo, listApiKeys, createApiKey as apiCreateApiKey, revokeApiKey as apiRevokeApiKey, getVoiceStatus, getVoiceList, getProfile, updateProfile, consolidateProfile, clearProfileObservations, type ApiKeyEntry, type GroupEntry, type GroupMember, type UserSearchResult, type BriefingConfig, type BriefingFeed, type VoiceStatusResult, type VoiceEntry, type UserProfile } from "@/api/client"; +import { apiGet, apiPost, apiPut, apiDelete, listGroups, createGroup, deleteGroup, listGroupMembers, addGroupMember, removeGroupMember, searchUsers, getBriefingConfig, saveBriefingConfig, getBriefingFeeds, createBriefingFeed, deleteBriefingFeed, refreshBriefingFeeds, geocodeAddress, getFableMcpInfo, listApiKeys, createApiKey as apiCreateApiKey, revokeApiKey as apiRevokeApiKey, getVoiceStatus, getVoiceList, synthesiseSpeech, getProfile, updateProfile, consolidateProfile, clearProfileObservations, type ApiKeyEntry, type GroupEntry, type GroupMember, type UserSearchResult, type BriefingConfig, type BriefingFeed, type VoiceStatusResult, type VoiceEntry, type VoiceBlendEntry, type UserProfile } from "@/api/client"; import { usePushStore } from "@/stores/push"; import type { User } from "@/types/auth"; import PaginationBar from "@/components/PaginationBar.vue"; @@ -514,6 +514,45 @@ const savingVoice = ref(false); const voiceSaved = ref(false); const voiceTabLoaded = ref(false); +// Voice blend +const blendEnabled = ref(false); +const voiceBlend = ref([ + { voice: "af_heart", weight: 1.0 }, + { voice: "af_bella", weight: 1.0 }, +]); +const blendPreviewText = ref("Hello! I'm your assistant. How can I help you today?"); +const blendPreviewing = ref(false); +let _blendAudioUrl = ""; + +function addBlendSlot() { + voiceBlend.value.push({ voice: "af_heart", weight: 1.0 }); +} + +function removeBlendSlot(idx: number) { + if (voiceBlend.value.length > 2) voiceBlend.value.splice(idx, 1); +} + +async function previewBlend() { + if (blendPreviewing.value || !blendPreviewText.value.trim()) return; + blendPreviewing.value = true; + try { + const blob = await synthesiseSpeech( + blendPreviewText.value, + undefined, + voiceTtsSpeed.value, + voiceBlend.value, + ); + if (_blendAudioUrl) URL.revokeObjectURL(_blendAudioUrl); + _blendAudioUrl = URL.createObjectURL(blob); + const audio = new Audio(_blendAudioUrl); + audio.play(); + } catch { + toastStore.show("Preview failed — TTS may not be ready", "error"); + } finally { + blendPreviewing.value = false; + } +} + async function loadVoiceTab() { if (voiceTabLoaded.value) return; voiceTabLoaded.value = true; @@ -538,6 +577,7 @@ async function saveVoiceSettings() { voice_tts_voice: voiceTtsVoice.value, voice_tts_speed: String(voiceTtsSpeed.value), voice_speech_style: voiceSpeechStyle.value, + voice_tts_blend: blendEnabled.value ? JSON.stringify(voiceBlend.value) : "", }); voiceSaved.value = true; setTimeout(() => { voiceSaved.value = false; }, 2000); @@ -656,6 +696,15 @@ onMounted(async () => { if (allSettings.voice_tts_voice) voiceTtsVoice.value = allSettings.voice_tts_voice; if (allSettings.voice_tts_speed) voiceTtsSpeed.value = parseFloat(allSettings.voice_tts_speed); if (allSettings.voice_speech_style) voiceSpeechStyle.value = allSettings.voice_speech_style; + if (allSettings.voice_tts_blend) { + try { + const parsed = JSON.parse(allSettings.voice_tts_blend); + if (Array.isArray(parsed) && parsed.length >= 2) { + voiceBlend.value = parsed; + blendEnabled.value = true; + } + } catch { /* ignore malformed */ } + } // Load CalDAV settings try { @@ -2207,6 +2256,71 @@ function formatUserDate(iso: string): string { +
+

Voice Blend

+

+ Mix two or more voice styles by weight. The resulting blended voice is used instead of + the single voice above when enabled. +

+ +
+ +
+ + +
+ @@ -4046,6 +4160,66 @@ FABLE_API_KEY=<your-api-key> .voice-admin-spinner span:nth-child(2) { animation-delay: 0.2s; } .voice-admin-spinner span:nth-child(3) { animation-delay: 0.4s; } +/* ── Voice blend ────────────────────────────────────────────────────────── */ +.blend-slots { + display: flex; + flex-direction: column; + gap: 0.75rem; + margin-bottom: 0.5rem; +} +.blend-slot { + background: color-mix(in srgb, var(--color-surface) 60%, transparent); + border: 1px solid var(--color-border); + border-radius: var(--radius-md); + padding: 0.75rem 1rem; + display: flex; + flex-direction: column; + gap: 0.5rem; +} +.blend-voice-select { + width: 100%; +} +.blend-weight-row { + display: flex; + align-items: center; + gap: 0.75rem; +} +.blend-weight-slider { + flex: 1; +} +.blend-weight-label { + font-size: 0.85rem; + font-variant-numeric: tabular-nums; + min-width: 2.5rem; + text-align: right; + color: var(--color-text-secondary); +} +.btn-remove-slot { + background: none; + border: 1px solid var(--color-border); + border-radius: var(--radius-sm); + color: var(--color-text-muted); + cursor: pointer; + font-size: 0.75rem; + padding: 0.2rem 0.45rem; + line-height: 1; + transition: color 0.15s, border-color 0.15s; +} +.btn-remove-slot:hover { + color: var(--color-danger, #e05555); + border-color: var(--color-danger, #e05555); +} +.blend-actions { + margin-top: 0.25rem; +} +.toggle-label { + display: flex; + align-items: center; + gap: 0.5rem; + cursor: pointer; + font-size: 0.9rem; +} + /* ── Profile tab ─────────────────────────────────────────────────────────── */ .day-picker { display: flex; diff --git a/src/fabledassistant/routes/voice.py b/src/fabledassistant/routes/voice.py index 377f6a6..1cce243 100644 --- a/src/fabledassistant/routes/voice.py +++ b/src/fabledassistant/routes/voice.py @@ -126,8 +126,27 @@ async def synthesise_speech(): except (TypeError, ValueError): speed = 1.0 + voice_blend = data.get("voice_blend") + if not isinstance(voice_blend, list) or len(voice_blend) < 2: + voice_blend = None + + # Auto-load blend from user settings when no explicit voice/blend provided + if voice_blend is None and "voice" not in data and "voice_blend" not in data: + from fabledassistant.services.settings import get_setting + from fabledassistant.auth import get_current_user_id + import json as _json + try: + uid = get_current_user_id() + stored = await get_setting(uid, "voice_tts_blend", "") + if stored: + parsed = _json.loads(stored) + if isinstance(parsed, list) and len(parsed) >= 2: + voice_blend = parsed + except Exception: + pass + try: - wav_bytes = await synthesise(text, voice=voice, speed=speed) + wav_bytes = await synthesise(text, voice=voice, speed=speed, voice_blend=voice_blend) except Exception: logger.exception("TTS synthesis failed") return jsonify({"error": "Synthesis failed"}), 500 diff --git a/src/fabledassistant/services/tts.py b/src/fabledassistant/services/tts.py index cf5495c..dcbb40e 100644 --- a/src/fabledassistant/services/tts.py +++ b/src/fabledassistant/services/tts.py @@ -73,20 +73,45 @@ def list_voices() -> list[dict]: return _VOICES -async def synthesise(text: str, voice: str = "af_heart", speed: float = 1.0) -> bytes: - """Synthesise text to WAV bytes (24kHz, 16-bit mono). Runs in executor.""" +async def synthesise( + text: str, + voice: str = "af_heart", + speed: float = 1.0, + voice_blend: list[dict] | None = None, +) -> bytes: + """Synthesise text to WAV bytes (24kHz, 16-bit mono). Runs in executor. + + voice_blend is a list of {"voice": str, "weight": float} dicts. + When provided with 2+ entries the voice style tensors are merged as a + weighted average before synthesis. Weights are normalised automatically. + """ if _pipeline is None: raise RuntimeError("TTS pipeline not loaded") speed = max(0.7, min(1.3, speed)) + def _build_voice_param(): + """Return either a blended style tensor or a single voice ID string.""" + if not voice_blend or len(voice_blend) < 2: + return voice_blend[0]["voice"] if voice_blend else voice + import numpy as np + total_w = sum(max(0.0, e.get("weight", 1.0)) for e in voice_blend) or 1.0 + blended = None + for entry in voice_blend: + vid = entry.get("voice", "af_heart") + w = max(0.0, entry.get("weight", 1.0)) / total_w + vt = _pipeline.load_voice(vid) # type: ignore[union-attr] + blended = vt * w if blended is None else blended + vt * w + return blended + def _run() -> bytes: import numpy as np import soundfile as sf + voice_param = _build_voice_param() t0 = time.monotonic() audio_chunks: list = [] - for _, _, audio in _pipeline(text, voice=voice, speed=speed): # type: ignore[misc] + for _, _, audio in _pipeline(text, voice=voice_param, speed=speed): # type: ignore[misc] if audio is not None: audio_chunks.append(audio)