feat: voice S2S — faster-whisper STT, Kokoro TTS, PTT overlay

Implements full speech-to-speech pipeline (all 4 phases):

Backend (Phase 1):
- services/stt.py: lazy WhisperModel singleton, run_in_executor transcription
- services/tts.py: lazy KPipeline singleton, WAV synthesis at 24kHz/16-bit
- routes/voice.py: /api/voice/status, /voices, /transcribe, /synthesise
- config.py: VOICE_ENABLED, STT_BACKEND, STT_MODEL, TTS_BACKEND env vars
- app.py: load STT/TTS models at startup when VOICE_ENABLED=true
- llm.py: voice_mode + voice_speech_style params inject speak-naturally prefix
- generation_task.py: voice_mode passed through from chat route
- chat.py: "voice" conversation type allowed + excluded from retention cleanup
- pyproject.toml + Dockerfile: faster-whisper, kokoro, soundfile dependencies

Frontend (Phases 2–4):
- composables/useVoiceRecorder.ts: MediaRecorder PTT wrapper
- composables/useVoiceAudio.ts: AudioContext WAV playback wrapper
- BriefingView.vue: Listen button (TTS read-aloud), auto-TTS mode, mic PTT
- VoiceOverlay.vue: global floating PTT button; creates/reuses voice conv;
  full record→transcribe→stream→TTS flow; Space bar hold-to-talk via App.vue
- SettingsView.vue: Voice tab (status badge, speech style, voice/speed)
- App.vue: mounts VoiceOverlay; Space keydown/keyup fires voice:ptt-toggle
- api/client.ts: getVoiceStatus, getVoiceList, transcribeAudio, synthesiseSpeech

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-29 20:03:38 -04:00
parent 3581cc1582
commit 6f84d90dff
18 changed files with 1524 additions and 6 deletions
+2 -1
View File
@@ -76,7 +76,7 @@ async def create_conversation_route():
model = data.get("model", Config.OLLAMA_MODEL)
conversation_type = data.get("conversation_type", "chat")
# Only allow known types to prevent accidental misuse
if conversation_type not in ("chat", "mcp"):
if conversation_type not in ("chat", "mcp", "voice"):
conversation_type = "chat"
conv = await create_conversation(uid, title=title, model=model, conversation_type=conversation_type)
return jsonify(conv.to_dict()), 201
@@ -187,6 +187,7 @@ async def send_message_route(conv_id: int):
rag_project_id=effective_rag_project_id,
workspace_project_id=workspace_project_id,
user_timezone=user_timezone,
voice_mode=(conv.conversation_type == "voice"),
))
return jsonify({
+130
View File
@@ -0,0 +1,130 @@
"""Voice (Speech-to-Speech) routes at /api/voice."""
import logging
import time
from quart import Blueprint, jsonify, request
from fabledassistant.auth import login_required
from fabledassistant.config import Config
logger = logging.getLogger(__name__)
voice_bp = Blueprint("voice", __name__, url_prefix="/api/voice")
@voice_bp.route("/status", methods=["GET"])
@login_required
async def voice_status():
"""Return availability of STT and TTS services."""
if not Config.VOICE_ENABLED:
return jsonify({"enabled": False, "stt": False, "tts": False})
from fabledassistant.services.stt import stt_available
from fabledassistant.services.tts import tts_available
return jsonify({
"enabled": True,
"stt": stt_available(),
"tts": tts_available(),
"stt_model": Config.STT_MODEL,
"tts_backend": Config.TTS_BACKEND,
})
@voice_bp.route("/voices", methods=["GET"])
@login_required
async def list_voices():
"""Return available Kokoro voice IDs and labels."""
if not Config.VOICE_ENABLED:
return jsonify({"error": "Voice feature is disabled"}), 503
from fabledassistant.services.tts import list_voices, tts_available
if not tts_available():
return jsonify({"error": "TTS not available"}), 503
return jsonify({"voices": list_voices()})
@voice_bp.route("/transcribe", methods=["POST"])
@login_required
async def transcribe_audio():
"""Accept a multipart audio file and return the transcript.
Request: multipart/form-data with field 'audio' (WebM/Opus blob)
Response: {"transcript": "...", "duration_ms": 123}
"""
if not Config.VOICE_ENABLED:
return jsonify({"error": "Voice feature is disabled"}), 503
from fabledassistant.services.stt import stt_available, transcribe
if not stt_available():
return jsonify({"error": "STT not available — model may still be loading"}), 503
files = await request.files
audio_file = files.get("audio")
if audio_file is None:
return jsonify({"error": "No audio file provided"}), 400
audio_bytes = audio_file.read()
if not audio_bytes:
return jsonify({"error": "Empty audio file"}), 400
if len(audio_bytes) > 25 * 1024 * 1024: # 25 MB hard cap
return jsonify({"error": "Audio file too large (max 25 MB)"}), 413
mime_type = audio_file.content_type or "audio/webm"
t0 = time.monotonic()
try:
transcript = await transcribe(audio_bytes, mime_type)
except Exception:
logger.exception("STT transcription failed")
return jsonify({"error": "Transcription failed"}), 500
duration_ms = round((time.monotonic() - t0) * 1000)
return jsonify({"transcript": transcript, "duration_ms": duration_ms})
@voice_bp.route("/synthesise", methods=["POST"])
@login_required
async def synthesise_speech():
"""Convert text to speech and return WAV bytes.
Request body: {"text": "...", "voice": "af_heart", "speed": 1.0}
Response: audio/wav bytes
"""
if not Config.VOICE_ENABLED:
return jsonify({"error": "Voice feature is disabled"}), 503
from fabledassistant.services.tts import synthesise, tts_available
if not tts_available():
return jsonify({"error": "TTS not available — model may still be loading"}), 503
data = await request.get_json()
if not data:
return jsonify({"error": "JSON body required"}), 400
text = str(data.get("text", "")).strip()
if not text:
return jsonify({"error": "text is required"}), 400
if len(text) > 8000:
return jsonify({"error": "text too long (max 8000 characters)"}), 400
voice = str(data.get("voice", "af_heart"))
try:
speed = float(data.get("speed", 1.0))
except (TypeError, ValueError):
speed = 1.0
try:
wav_bytes = await synthesise(text, voice=voice, speed=speed)
except Exception:
logger.exception("TTS synthesis failed")
return jsonify({"error": "Synthesis failed"}), 500
from quart import Response
return Response(wav_bytes, mimetype="audio/wav")