feat: voice S2S — faster-whisper STT, Kokoro TTS, PTT overlay
Implements full speech-to-speech pipeline (all 4 phases): Backend (Phase 1): - services/stt.py: lazy WhisperModel singleton, run_in_executor transcription - services/tts.py: lazy KPipeline singleton, WAV synthesis at 24kHz/16-bit - routes/voice.py: /api/voice/status, /voices, /transcribe, /synthesise - config.py: VOICE_ENABLED, STT_BACKEND, STT_MODEL, TTS_BACKEND env vars - app.py: load STT/TTS models at startup when VOICE_ENABLED=true - llm.py: voice_mode + voice_speech_style params inject speak-naturally prefix - generation_task.py: voice_mode passed through from chat route - chat.py: "voice" conversation type allowed + excluded from retention cleanup - pyproject.toml + Dockerfile: faster-whisper, kokoro, soundfile dependencies Frontend (Phases 2–4): - composables/useVoiceRecorder.ts: MediaRecorder PTT wrapper - composables/useVoiceAudio.ts: AudioContext WAV playback wrapper - BriefingView.vue: Listen button (TTS read-aloud), auto-TTS mode, mic PTT - VoiceOverlay.vue: global floating PTT button; creates/reuses voice conv; full record→transcribe→stream→TTS flow; Space bar hold-to-talk via App.vue - SettingsView.vue: Voice tab (status badge, speech style, voice/speed) - App.vue: mounts VoiceOverlay; Space keydown/keyup fires voice:ptt-toggle - api/client.ts: getVoiceStatus, getVoiceList, transcribeAudio, synthesiseSpeech Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,94 @@
|
||||
"""Text-to-speech service using Kokoro TTS (in-process)."""
|
||||
import asyncio
|
||||
import io
|
||||
import logging
|
||||
import time
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from kokoro import KPipeline
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_pipeline: "KPipeline | None" = None
|
||||
_pipeline_lock = asyncio.Lock()
|
||||
_load_error: str | None = None
|
||||
|
||||
# Static list of supported Kokoro voice IDs and display labels
|
||||
_VOICES: list[dict] = [
|
||||
{"id": "af_heart", "label": "Heart (American Female, warm)"},
|
||||
{"id": "af_bella", "label": "Bella (American Female, expressive)"},
|
||||
{"id": "af_nicole", "label": "Nicole (American Female, intimate)"},
|
||||
{"id": "af_sarah", "label": "Sarah (American Female, clear)"},
|
||||
{"id": "af_sky", "label": "Sky (American Female, bright)"},
|
||||
{"id": "am_adam", "label": "Adam (American Male, neutral)"},
|
||||
{"id": "am_michael", "label": "Michael (American Male, deep)"},
|
||||
{"id": "bf_emma", "label": "Emma (British Female)"},
|
||||
{"id": "bf_isabella", "label": "Isabella (British Female, formal)"},
|
||||
{"id": "bm_george", "label": "George (British Male)"},
|
||||
{"id": "bm_lewis", "label": "Lewis (British Male, casual)"},
|
||||
]
|
||||
|
||||
|
||||
async def load_tts_model() -> None:
|
||||
"""Load the Kokoro pipeline. Called once at startup when VOICE_ENABLED=true."""
|
||||
global _pipeline, _load_error
|
||||
from fabledassistant.config import Config
|
||||
|
||||
if not Config.VOICE_ENABLED:
|
||||
return
|
||||
|
||||
async with _pipeline_lock:
|
||||
if _pipeline is not None:
|
||||
return
|
||||
try:
|
||||
from kokoro import KPipeline
|
||||
|
||||
logger.info("Loading Kokoro TTS pipeline...")
|
||||
loop = asyncio.get_running_loop()
|
||||
_pipeline = await loop.run_in_executor(
|
||||
None,
|
||||
lambda: KPipeline(lang_code="a"), # "a" = American English
|
||||
)
|
||||
logger.info("Kokoro TTS pipeline loaded")
|
||||
except Exception:
|
||||
_load_error = "Failed to load Kokoro TTS pipeline"
|
||||
logger.exception(_load_error)
|
||||
|
||||
|
||||
def tts_available() -> bool:
|
||||
return _pipeline is not None
|
||||
|
||||
|
||||
def list_voices() -> list[dict]:
|
||||
return _VOICES
|
||||
|
||||
|
||||
async def synthesise(text: str, voice: str = "af_heart", speed: float = 1.0) -> bytes:
|
||||
"""Synthesise text to WAV bytes (24kHz, 16-bit mono). Runs in executor."""
|
||||
if _pipeline is None:
|
||||
raise RuntimeError("TTS pipeline not loaded")
|
||||
|
||||
speed = max(0.7, min(1.3, speed))
|
||||
|
||||
def _run() -> bytes:
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
|
||||
t0 = time.monotonic()
|
||||
audio_chunks: list = []
|
||||
for _, _, audio in _pipeline(text, voice=voice, speed=speed): # type: ignore[misc]
|
||||
if audio is not None:
|
||||
audio_chunks.append(audio)
|
||||
|
||||
if not audio_chunks:
|
||||
return b""
|
||||
|
||||
combined = np.concatenate(audio_chunks)
|
||||
buf = io.BytesIO()
|
||||
sf.write(buf, combined, samplerate=24000, format="WAV", subtype="PCM_16")
|
||||
logger.debug("TTS synthesis took %.2fs for %d chars", time.monotonic() - t0, len(text))
|
||||
return buf.getvalue()
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
return await loop.run_in_executor(None, _run)
|
||||
Reference in New Issue
Block a user