feat: voice S2S — faster-whisper STT, Kokoro TTS, PTT overlay
Implements full speech-to-speech pipeline (all 4 phases): Backend (Phase 1): - services/stt.py: lazy WhisperModel singleton, run_in_executor transcription - services/tts.py: lazy KPipeline singleton, WAV synthesis at 24kHz/16-bit - routes/voice.py: /api/voice/status, /voices, /transcribe, /synthesise - config.py: VOICE_ENABLED, STT_BACKEND, STT_MODEL, TTS_BACKEND env vars - app.py: load STT/TTS models at startup when VOICE_ENABLED=true - llm.py: voice_mode + voice_speech_style params inject speak-naturally prefix - generation_task.py: voice_mode passed through from chat route - chat.py: "voice" conversation type allowed + excluded from retention cleanup - pyproject.toml + Dockerfile: faster-whisper, kokoro, soundfile dependencies Frontend (Phases 2–4): - composables/useVoiceRecorder.ts: MediaRecorder PTT wrapper - composables/useVoiceAudio.ts: AudioContext WAV playback wrapper - BriefingView.vue: Listen button (TTS read-aloud), auto-TTS mode, mic PTT - VoiceOverlay.vue: global floating PTT button; creates/reuses voice conv; full record→transcribe→stream→TTS flow; Space bar hold-to-talk via App.vue - SettingsView.vue: Voice tab (status badge, speech style, voice/speed) - App.vue: mounts VoiceOverlay; Space keydown/keyup fires voice:ptt-toggle - api/client.ts: getVoiceStatus, getVoiceList, transcribeAudio, synthesiseSpeech Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,72 @@
|
||||
"""Speech-to-text service using faster-whisper (in-process, CPU/GPU)."""
|
||||
import asyncio
|
||||
import logging
|
||||
import tempfile
|
||||
import time
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from faster_whisper import WhisperModel
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_model: "WhisperModel | None" = None
|
||||
_model_lock = asyncio.Lock()
|
||||
_load_error: str | None = None
|
||||
|
||||
|
||||
async def load_stt_model() -> None:
|
||||
"""Load the Whisper model. Called once at startup when VOICE_ENABLED=true."""
|
||||
global _model, _load_error
|
||||
from fabledassistant.config import Config
|
||||
|
||||
if not Config.VOICE_ENABLED:
|
||||
return
|
||||
|
||||
async with _model_lock:
|
||||
if _model is not None:
|
||||
return
|
||||
try:
|
||||
from faster_whisper import WhisperModel
|
||||
|
||||
logger.info("Loading Whisper STT model '%s'...", Config.STT_MODEL)
|
||||
loop = asyncio.get_running_loop()
|
||||
_model = await loop.run_in_executor(
|
||||
None,
|
||||
lambda: WhisperModel(Config.STT_MODEL, device="cpu", compute_type="int8"),
|
||||
)
|
||||
logger.info("Whisper STT model '%s' loaded", Config.STT_MODEL)
|
||||
except Exception:
|
||||
_load_error = f"Failed to load Whisper model '{Config.STT_MODEL}'"
|
||||
logger.exception(_load_error)
|
||||
|
||||
|
||||
def stt_available() -> bool:
|
||||
return _model is not None
|
||||
|
||||
|
||||
async def transcribe(audio_bytes: bytes, mime_type: str = "audio/webm") -> str:
|
||||
"""Transcribe audio bytes to text. Runs the model in a thread executor."""
|
||||
if _model is None:
|
||||
raise RuntimeError("STT model not loaded")
|
||||
|
||||
suffix = ".webm"
|
||||
if "ogg" in mime_type:
|
||||
suffix = ".ogg"
|
||||
elif "wav" in mime_type:
|
||||
suffix = ".wav"
|
||||
elif "mp4" in mime_type or "m4a" in mime_type:
|
||||
suffix = ".mp4"
|
||||
|
||||
def _run() -> str:
|
||||
with tempfile.NamedTemporaryFile(suffix=suffix, delete=True) as f:
|
||||
f.write(audio_bytes)
|
||||
f.flush()
|
||||
t0 = time.monotonic()
|
||||
segments, _ = _model.transcribe(f.name, beam_size=5) # type: ignore[union-attr]
|
||||
text = " ".join(seg.text.strip() for seg in segments).strip()
|
||||
logger.debug("STT transcription took %.2fs", time.monotonic() - t0)
|
||||
return text
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
return await loop.run_in_executor(None, _run)
|
||||
Reference in New Issue
Block a user