feat: Kokoro voice blending — blend builder UI + weighted tensor synthesis
Add voice blend support to TTS pipeline and settings UI. Users can mix 2–5 Kokoro voices with per-voice weight sliders; the blended style tensor replaces the single voice when enabled. Settings persist as JSON and auto- load on synthesis when no explicit voice is supplied in the request. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -639,6 +639,11 @@ export interface VoiceEntry {
|
||||
label: string
|
||||
}
|
||||
|
||||
export interface VoiceBlendEntry {
|
||||
voice: string
|
||||
weight: number
|
||||
}
|
||||
|
||||
export const getVoiceStatus = () => apiGet<VoiceStatusResult>('/api/voice/status')
|
||||
|
||||
export const getVoiceList = () =>
|
||||
@@ -658,12 +663,19 @@ export async function transcribeAudio(blob: Blob): Promise<{ transcript: string;
|
||||
export async function synthesiseSpeech(
|
||||
text: string,
|
||||
voice?: string,
|
||||
speed?: number
|
||||
speed?: number,
|
||||
voiceBlend?: VoiceBlendEntry[]
|
||||
): Promise<Blob> {
|
||||
const body: Record<string, unknown> = { text, speed: speed ?? 1.0 }
|
||||
if (voiceBlend && voiceBlend.length >= 2) {
|
||||
body.voice_blend = voiceBlend
|
||||
} else {
|
||||
body.voice = voice ?? 'af_heart'
|
||||
}
|
||||
const res = await fetch('/api/voice/synthesise', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ text, voice: voice ?? 'af_heart', speed: speed ?? 1.0 }),
|
||||
body: JSON.stringify(body),
|
||||
})
|
||||
if (!res.ok) {
|
||||
const err = await res.json().catch(() => ({ error: `HTTP ${res.status}` }))
|
||||
|
||||
@@ -3,7 +3,7 @@ import { ref, watch, onMounted } from "vue";
|
||||
import { useSettingsStore } from "@/stores/settings";
|
||||
import { useAuthStore } from "@/stores/auth";
|
||||
import { useToastStore } from "@/stores/toast";
|
||||
import { apiGet, apiPost, apiPut, apiDelete, listGroups, createGroup, deleteGroup, listGroupMembers, addGroupMember, removeGroupMember, searchUsers, getBriefingConfig, saveBriefingConfig, getBriefingFeeds, createBriefingFeed, deleteBriefingFeed, refreshBriefingFeeds, geocodeAddress, getFableMcpInfo, listApiKeys, createApiKey as apiCreateApiKey, revokeApiKey as apiRevokeApiKey, getVoiceStatus, getVoiceList, getProfile, updateProfile, consolidateProfile, clearProfileObservations, type ApiKeyEntry, type GroupEntry, type GroupMember, type UserSearchResult, type BriefingConfig, type BriefingFeed, type VoiceStatusResult, type VoiceEntry, type UserProfile } from "@/api/client";
|
||||
import { apiGet, apiPost, apiPut, apiDelete, listGroups, createGroup, deleteGroup, listGroupMembers, addGroupMember, removeGroupMember, searchUsers, getBriefingConfig, saveBriefingConfig, getBriefingFeeds, createBriefingFeed, deleteBriefingFeed, refreshBriefingFeeds, geocodeAddress, getFableMcpInfo, listApiKeys, createApiKey as apiCreateApiKey, revokeApiKey as apiRevokeApiKey, getVoiceStatus, getVoiceList, synthesiseSpeech, getProfile, updateProfile, consolidateProfile, clearProfileObservations, type ApiKeyEntry, type GroupEntry, type GroupMember, type UserSearchResult, type BriefingConfig, type BriefingFeed, type VoiceStatusResult, type VoiceEntry, type VoiceBlendEntry, type UserProfile } from "@/api/client";
|
||||
import { usePushStore } from "@/stores/push";
|
||||
import type { User } from "@/types/auth";
|
||||
import PaginationBar from "@/components/PaginationBar.vue";
|
||||
@@ -514,6 +514,45 @@ const savingVoice = ref(false);
|
||||
const voiceSaved = ref(false);
|
||||
const voiceTabLoaded = ref(false);
|
||||
|
||||
// Voice blend
|
||||
const blendEnabled = ref(false);
|
||||
const voiceBlend = ref<VoiceBlendEntry[]>([
|
||||
{ voice: "af_heart", weight: 1.0 },
|
||||
{ voice: "af_bella", weight: 1.0 },
|
||||
]);
|
||||
const blendPreviewText = ref("Hello! I'm your assistant. How can I help you today?");
|
||||
const blendPreviewing = ref(false);
|
||||
let _blendAudioUrl = "";
|
||||
|
||||
function addBlendSlot() {
|
||||
voiceBlend.value.push({ voice: "af_heart", weight: 1.0 });
|
||||
}
|
||||
|
||||
function removeBlendSlot(idx: number) {
|
||||
if (voiceBlend.value.length > 2) voiceBlend.value.splice(idx, 1);
|
||||
}
|
||||
|
||||
async function previewBlend() {
|
||||
if (blendPreviewing.value || !blendPreviewText.value.trim()) return;
|
||||
blendPreviewing.value = true;
|
||||
try {
|
||||
const blob = await synthesiseSpeech(
|
||||
blendPreviewText.value,
|
||||
undefined,
|
||||
voiceTtsSpeed.value,
|
||||
voiceBlend.value,
|
||||
);
|
||||
if (_blendAudioUrl) URL.revokeObjectURL(_blendAudioUrl);
|
||||
_blendAudioUrl = URL.createObjectURL(blob);
|
||||
const audio = new Audio(_blendAudioUrl);
|
||||
audio.play();
|
||||
} catch {
|
||||
toastStore.show("Preview failed — TTS may not be ready", "error");
|
||||
} finally {
|
||||
blendPreviewing.value = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function loadVoiceTab() {
|
||||
if (voiceTabLoaded.value) return;
|
||||
voiceTabLoaded.value = true;
|
||||
@@ -538,6 +577,7 @@ async function saveVoiceSettings() {
|
||||
voice_tts_voice: voiceTtsVoice.value,
|
||||
voice_tts_speed: String(voiceTtsSpeed.value),
|
||||
voice_speech_style: voiceSpeechStyle.value,
|
||||
voice_tts_blend: blendEnabled.value ? JSON.stringify(voiceBlend.value) : "",
|
||||
});
|
||||
voiceSaved.value = true;
|
||||
setTimeout(() => { voiceSaved.value = false; }, 2000);
|
||||
@@ -656,6 +696,15 @@ onMounted(async () => {
|
||||
if (allSettings.voice_tts_voice) voiceTtsVoice.value = allSettings.voice_tts_voice;
|
||||
if (allSettings.voice_tts_speed) voiceTtsSpeed.value = parseFloat(allSettings.voice_tts_speed);
|
||||
if (allSettings.voice_speech_style) voiceSpeechStyle.value = allSettings.voice_speech_style;
|
||||
if (allSettings.voice_tts_blend) {
|
||||
try {
|
||||
const parsed = JSON.parse(allSettings.voice_tts_blend);
|
||||
if (Array.isArray(parsed) && parsed.length >= 2) {
|
||||
voiceBlend.value = parsed;
|
||||
blendEnabled.value = true;
|
||||
}
|
||||
} catch { /* ignore malformed */ }
|
||||
}
|
||||
|
||||
// Load CalDAV settings
|
||||
try {
|
||||
@@ -2207,6 +2256,71 @@ function formatUserDate(iso: string): string {
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section v-if="voiceStatus?.enabled && voiceStatus?.tts" class="settings-section full-width">
|
||||
<h2>Voice Blend</h2>
|
||||
<p class="section-desc">
|
||||
Mix two or more voice styles by weight. The resulting blended voice is used instead of
|
||||
the single voice above when enabled.
|
||||
</p>
|
||||
|
||||
<div class="field-group">
|
||||
<label class="toggle-label">
|
||||
<input type="checkbox" v-model="blendEnabled" />
|
||||
<span>Enable voice blending</span>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<template v-if="blendEnabled">
|
||||
<div class="blend-slots">
|
||||
<div v-for="(entry, idx) in voiceBlend" :key="idx" class="blend-slot">
|
||||
<select class="input blend-voice-select" v-model="entry.voice">
|
||||
<option v-for="v in availableVoices" :key="v.id" :value="v.id">{{ v.label }}</option>
|
||||
</select>
|
||||
<div class="blend-weight-row">
|
||||
<input
|
||||
type="range"
|
||||
min="0.1"
|
||||
max="2.0"
|
||||
step="0.05"
|
||||
v-model.number="entry.weight"
|
||||
class="range-input blend-weight-slider"
|
||||
/>
|
||||
<span class="blend-weight-label">{{ entry.weight.toFixed(2) }}</span>
|
||||
<button
|
||||
v-if="voiceBlend.length > 2"
|
||||
class="btn-remove-slot"
|
||||
@click="removeBlendSlot(idx)"
|
||||
title="Remove this voice"
|
||||
>✕</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="blend-actions">
|
||||
<button class="btn-secondary" @click="addBlendSlot" :disabled="voiceBlend.length >= 5">
|
||||
+ Add voice
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div class="field-group" style="margin-top: 1rem;">
|
||||
<label class="field-label">Preview text</label>
|
||||
<input
|
||||
type="text"
|
||||
class="input"
|
||||
v-model="blendPreviewText"
|
||||
placeholder="Text to speak for preview…"
|
||||
maxlength="300"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div class="form-actions">
|
||||
<button class="btn-secondary" @click="previewBlend" :disabled="blendPreviewing">
|
||||
{{ blendPreviewing ? 'Generating…' : '▶ Preview blend' }}
|
||||
</button>
|
||||
</div>
|
||||
</template>
|
||||
</section>
|
||||
|
||||
</div>
|
||||
|
||||
<!-- ── API Keys ── -->
|
||||
@@ -4046,6 +4160,66 @@ FABLE_API_KEY=<your-api-key></pre>
|
||||
.voice-admin-spinner span:nth-child(2) { animation-delay: 0.2s; }
|
||||
.voice-admin-spinner span:nth-child(3) { animation-delay: 0.4s; }
|
||||
|
||||
/* ── Voice blend ────────────────────────────────────────────────────────── */
|
||||
.blend-slots {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.75rem;
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
.blend-slot {
|
||||
background: color-mix(in srgb, var(--color-surface) 60%, transparent);
|
||||
border: 1px solid var(--color-border);
|
||||
border-radius: var(--radius-md);
|
||||
padding: 0.75rem 1rem;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
.blend-voice-select {
|
||||
width: 100%;
|
||||
}
|
||||
.blend-weight-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.75rem;
|
||||
}
|
||||
.blend-weight-slider {
|
||||
flex: 1;
|
||||
}
|
||||
.blend-weight-label {
|
||||
font-size: 0.85rem;
|
||||
font-variant-numeric: tabular-nums;
|
||||
min-width: 2.5rem;
|
||||
text-align: right;
|
||||
color: var(--color-text-secondary);
|
||||
}
|
||||
.btn-remove-slot {
|
||||
background: none;
|
||||
border: 1px solid var(--color-border);
|
||||
border-radius: var(--radius-sm);
|
||||
color: var(--color-text-muted);
|
||||
cursor: pointer;
|
||||
font-size: 0.75rem;
|
||||
padding: 0.2rem 0.45rem;
|
||||
line-height: 1;
|
||||
transition: color 0.15s, border-color 0.15s;
|
||||
}
|
||||
.btn-remove-slot:hover {
|
||||
color: var(--color-danger, #e05555);
|
||||
border-color: var(--color-danger, #e05555);
|
||||
}
|
||||
.blend-actions {
|
||||
margin-top: 0.25rem;
|
||||
}
|
||||
.toggle-label {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
cursor: pointer;
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
|
||||
/* ── Profile tab ─────────────────────────────────────────────────────────── */
|
||||
.day-picker {
|
||||
display: flex;
|
||||
|
||||
@@ -126,8 +126,27 @@ async def synthesise_speech():
|
||||
except (TypeError, ValueError):
|
||||
speed = 1.0
|
||||
|
||||
voice_blend = data.get("voice_blend")
|
||||
if not isinstance(voice_blend, list) or len(voice_blend) < 2:
|
||||
voice_blend = None
|
||||
|
||||
# Auto-load blend from user settings when no explicit voice/blend provided
|
||||
if voice_blend is None and "voice" not in data and "voice_blend" not in data:
|
||||
from fabledassistant.services.settings import get_setting
|
||||
from fabledassistant.auth import get_current_user_id
|
||||
import json as _json
|
||||
try:
|
||||
uid = get_current_user_id()
|
||||
stored = await get_setting(uid, "voice_tts_blend", "")
|
||||
if stored:
|
||||
parsed = _json.loads(stored)
|
||||
if isinstance(parsed, list) and len(parsed) >= 2:
|
||||
voice_blend = parsed
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
wav_bytes = await synthesise(text, voice=voice, speed=speed)
|
||||
wav_bytes = await synthesise(text, voice=voice, speed=speed, voice_blend=voice_blend)
|
||||
except Exception:
|
||||
logger.exception("TTS synthesis failed")
|
||||
return jsonify({"error": "Synthesis failed"}), 500
|
||||
|
||||
@@ -73,20 +73,45 @@ def list_voices() -> list[dict]:
|
||||
return _VOICES
|
||||
|
||||
|
||||
async def synthesise(text: str, voice: str = "af_heart", speed: float = 1.0) -> bytes:
|
||||
"""Synthesise text to WAV bytes (24kHz, 16-bit mono). Runs in executor."""
|
||||
async def synthesise(
|
||||
text: str,
|
||||
voice: str = "af_heart",
|
||||
speed: float = 1.0,
|
||||
voice_blend: list[dict] | None = None,
|
||||
) -> bytes:
|
||||
"""Synthesise text to WAV bytes (24kHz, 16-bit mono). Runs in executor.
|
||||
|
||||
voice_blend is a list of {"voice": str, "weight": float} dicts.
|
||||
When provided with 2+ entries the voice style tensors are merged as a
|
||||
weighted average before synthesis. Weights are normalised automatically.
|
||||
"""
|
||||
if _pipeline is None:
|
||||
raise RuntimeError("TTS pipeline not loaded")
|
||||
|
||||
speed = max(0.7, min(1.3, speed))
|
||||
|
||||
def _build_voice_param():
|
||||
"""Return either a blended style tensor or a single voice ID string."""
|
||||
if not voice_blend or len(voice_blend) < 2:
|
||||
return voice_blend[0]["voice"] if voice_blend else voice
|
||||
import numpy as np
|
||||
total_w = sum(max(0.0, e.get("weight", 1.0)) for e in voice_blend) or 1.0
|
||||
blended = None
|
||||
for entry in voice_blend:
|
||||
vid = entry.get("voice", "af_heart")
|
||||
w = max(0.0, entry.get("weight", 1.0)) / total_w
|
||||
vt = _pipeline.load_voice(vid) # type: ignore[union-attr]
|
||||
blended = vt * w if blended is None else blended + vt * w
|
||||
return blended
|
||||
|
||||
def _run() -> bytes:
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
|
||||
voice_param = _build_voice_param()
|
||||
t0 = time.monotonic()
|
||||
audio_chunks: list = []
|
||||
for _, _, audio in _pipeline(text, voice=voice, speed=speed): # type: ignore[misc]
|
||||
for _, _, audio in _pipeline(text, voice=voice_param, speed=speed): # type: ignore[misc]
|
||||
if audio is not None:
|
||||
audio_chunks.append(audio)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user