feat: voice S2S — faster-whisper STT, Kokoro TTS, PTT overlay
Implements full speech-to-speech pipeline (all 4 phases): Backend (Phase 1): - services/stt.py: lazy WhisperModel singleton, run_in_executor transcription - services/tts.py: lazy KPipeline singleton, WAV synthesis at 24kHz/16-bit - routes/voice.py: /api/voice/status, /voices, /transcribe, /synthesise - config.py: VOICE_ENABLED, STT_BACKEND, STT_MODEL, TTS_BACKEND env vars - app.py: load STT/TTS models at startup when VOICE_ENABLED=true - llm.py: voice_mode + voice_speech_style params inject speak-naturally prefix - generation_task.py: voice_mode passed through from chat route - chat.py: "voice" conversation type allowed + excluded from retention cleanup - pyproject.toml + Dockerfile: faster-whisper, kokoro, soundfile dependencies Frontend (Phases 2–4): - composables/useVoiceRecorder.ts: MediaRecorder PTT wrapper - composables/useVoiceAudio.ts: AudioContext WAV playback wrapper - BriefingView.vue: Listen button (TTS read-aloud), auto-TTS mode, mic PTT - VoiceOverlay.vue: global floating PTT button; creates/reuses voice conv; full record→transcribe→stream→TTS flow; Space bar hold-to-talk via App.vue - SettingsView.vue: Voice tab (status badge, speech style, voice/speed) - App.vue: mounts VoiceOverlay; Space keydown/keyup fires voice:ptt-toggle - api/client.ts: getVoiceStatus, getVoiceList, transcribeAudio, synthesiseSpeech Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -76,7 +76,7 @@ async def create_conversation_route():
|
||||
model = data.get("model", Config.OLLAMA_MODEL)
|
||||
conversation_type = data.get("conversation_type", "chat")
|
||||
# Only allow known types to prevent accidental misuse
|
||||
if conversation_type not in ("chat", "mcp"):
|
||||
if conversation_type not in ("chat", "mcp", "voice"):
|
||||
conversation_type = "chat"
|
||||
conv = await create_conversation(uid, title=title, model=model, conversation_type=conversation_type)
|
||||
return jsonify(conv.to_dict()), 201
|
||||
@@ -187,6 +187,7 @@ async def send_message_route(conv_id: int):
|
||||
rag_project_id=effective_rag_project_id,
|
||||
workspace_project_id=workspace_project_id,
|
||||
user_timezone=user_timezone,
|
||||
voice_mode=(conv.conversation_type == "voice"),
|
||||
))
|
||||
|
||||
return jsonify({
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
"""Voice (Speech-to-Speech) routes at /api/voice."""
|
||||
import logging
|
||||
import time
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
|
||||
from fabledassistant.auth import login_required
|
||||
from fabledassistant.config import Config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
voice_bp = Blueprint("voice", __name__, url_prefix="/api/voice")
|
||||
|
||||
|
||||
@voice_bp.route("/status", methods=["GET"])
|
||||
@login_required
|
||||
async def voice_status():
|
||||
"""Return availability of STT and TTS services."""
|
||||
if not Config.VOICE_ENABLED:
|
||||
return jsonify({"enabled": False, "stt": False, "tts": False})
|
||||
|
||||
from fabledassistant.services.stt import stt_available
|
||||
from fabledassistant.services.tts import tts_available
|
||||
|
||||
return jsonify({
|
||||
"enabled": True,
|
||||
"stt": stt_available(),
|
||||
"tts": tts_available(),
|
||||
"stt_model": Config.STT_MODEL,
|
||||
"tts_backend": Config.TTS_BACKEND,
|
||||
})
|
||||
|
||||
|
||||
@voice_bp.route("/voices", methods=["GET"])
|
||||
@login_required
|
||||
async def list_voices():
|
||||
"""Return available Kokoro voice IDs and labels."""
|
||||
if not Config.VOICE_ENABLED:
|
||||
return jsonify({"error": "Voice feature is disabled"}), 503
|
||||
|
||||
from fabledassistant.services.tts import list_voices, tts_available
|
||||
|
||||
if not tts_available():
|
||||
return jsonify({"error": "TTS not available"}), 503
|
||||
|
||||
return jsonify({"voices": list_voices()})
|
||||
|
||||
|
||||
@voice_bp.route("/transcribe", methods=["POST"])
|
||||
@login_required
|
||||
async def transcribe_audio():
|
||||
"""Accept a multipart audio file and return the transcript.
|
||||
|
||||
Request: multipart/form-data with field 'audio' (WebM/Opus blob)
|
||||
Response: {"transcript": "...", "duration_ms": 123}
|
||||
"""
|
||||
if not Config.VOICE_ENABLED:
|
||||
return jsonify({"error": "Voice feature is disabled"}), 503
|
||||
|
||||
from fabledassistant.services.stt import stt_available, transcribe
|
||||
|
||||
if not stt_available():
|
||||
return jsonify({"error": "STT not available — model may still be loading"}), 503
|
||||
|
||||
files = await request.files
|
||||
audio_file = files.get("audio")
|
||||
if audio_file is None:
|
||||
return jsonify({"error": "No audio file provided"}), 400
|
||||
|
||||
audio_bytes = audio_file.read()
|
||||
if not audio_bytes:
|
||||
return jsonify({"error": "Empty audio file"}), 400
|
||||
|
||||
if len(audio_bytes) > 25 * 1024 * 1024: # 25 MB hard cap
|
||||
return jsonify({"error": "Audio file too large (max 25 MB)"}), 413
|
||||
|
||||
mime_type = audio_file.content_type or "audio/webm"
|
||||
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
transcript = await transcribe(audio_bytes, mime_type)
|
||||
except Exception:
|
||||
logger.exception("STT transcription failed")
|
||||
return jsonify({"error": "Transcription failed"}), 500
|
||||
|
||||
duration_ms = round((time.monotonic() - t0) * 1000)
|
||||
return jsonify({"transcript": transcript, "duration_ms": duration_ms})
|
||||
|
||||
|
||||
@voice_bp.route("/synthesise", methods=["POST"])
|
||||
@login_required
|
||||
async def synthesise_speech():
|
||||
"""Convert text to speech and return WAV bytes.
|
||||
|
||||
Request body: {"text": "...", "voice": "af_heart", "speed": 1.0}
|
||||
Response: audio/wav bytes
|
||||
"""
|
||||
if not Config.VOICE_ENABLED:
|
||||
return jsonify({"error": "Voice feature is disabled"}), 503
|
||||
|
||||
from fabledassistant.services.tts import synthesise, tts_available
|
||||
|
||||
if not tts_available():
|
||||
return jsonify({"error": "TTS not available — model may still be loading"}), 503
|
||||
|
||||
data = await request.get_json()
|
||||
if not data:
|
||||
return jsonify({"error": "JSON body required"}), 400
|
||||
|
||||
text = str(data.get("text", "")).strip()
|
||||
if not text:
|
||||
return jsonify({"error": "text is required"}), 400
|
||||
|
||||
if len(text) > 8000:
|
||||
return jsonify({"error": "text too long (max 8000 characters)"}), 400
|
||||
|
||||
voice = str(data.get("voice", "af_heart"))
|
||||
try:
|
||||
speed = float(data.get("speed", 1.0))
|
||||
except (TypeError, ValueError):
|
||||
speed = 1.0
|
||||
|
||||
try:
|
||||
wav_bytes = await synthesise(text, voice=voice, speed=speed)
|
||||
except Exception:
|
||||
logger.exception("TTS synthesis failed")
|
||||
return jsonify({"error": "Synthesis failed"}), 500
|
||||
|
||||
from quart import Response
|
||||
return Response(wav_bytes, mimetype="audio/wav")
|
||||
Reference in New Issue
Block a user