feat: voice S2S — faster-whisper STT, Kokoro TTS, PTT overlay

Implements full speech-to-speech pipeline (all 4 phases):

Backend (Phase 1):
- services/stt.py: lazy WhisperModel singleton, run_in_executor transcription
- services/tts.py: lazy KPipeline singleton, WAV synthesis at 24kHz/16-bit
- routes/voice.py: /api/voice/status, /voices, /transcribe, /synthesise
- config.py: VOICE_ENABLED, STT_BACKEND, STT_MODEL, TTS_BACKEND env vars
- app.py: load STT/TTS models at startup when VOICE_ENABLED=true
- llm.py: voice_mode + voice_speech_style params inject speak-naturally prefix
- generation_task.py: voice_mode passed through from chat route
- chat.py: "voice" conversation type allowed + excluded from retention cleanup
- pyproject.toml + Dockerfile: faster-whisper, kokoro, soundfile dependencies

Frontend (Phases 2–4):
- composables/useVoiceRecorder.ts: MediaRecorder PTT wrapper
- composables/useVoiceAudio.ts: AudioContext WAV playback wrapper
- BriefingView.vue: Listen button (TTS read-aloud), auto-TTS mode, mic PTT
- VoiceOverlay.vue: global floating PTT button; creates/reuses voice conv;
  full record→transcribe→stream→TTS flow; Space bar hold-to-talk via App.vue
- SettingsView.vue: Voice tab (status badge, speech style, voice/speed)
- App.vue: mounts VoiceOverlay; Space keydown/keyup fires voice:ptt-toggle
- api/client.ts: getVoiceStatus, getVoiceList, transcribeAudio, synthesiseSpeech

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

src/fabledassistant/app.py

@@ -31,6 +31,7 @@ from fabledassistant.routes.users import users_bp
 from fabledassistant.routes.api_keys import api_keys_bp
 from fabledassistant.routes.events import events_bp
 from fabledassistant.routes.search import search_bp
+from fabledassistant.routes.voice import voice_bp
 
 STATIC_DIR = Path(__file__).parent / "static"
 logger = logging.getLogger(__name__)
@@ -92,6 +93,7 @@ def create_app() -> Quart:
     app.register_blueprint(api_keys_bp)
     app.register_blueprint(events_bp)
     app.register_blueprint(search_bp)
+    app.register_blueprint(voice_bp)
 
     @app.before_request
     async def before_request():
@@ -264,6 +266,13 @@ def create_app() -> Quart:
         from fabledassistant.services.briefing_scheduler import start_briefing_scheduler
         await start_briefing_scheduler(asyncio.get_running_loop())
 
+        # Voice model loading (only when feature is enabled)
+        if Config.VOICE_ENABLED:
+            from fabledassistant.services.stt import load_stt_model
+            from fabledassistant.services.tts import load_tts_model
+            asyncio.create_task(load_stt_model())
+            asyncio.create_task(load_tts_model())
+
     @app.after_serving
     async def shutdown():
         from fabledassistant.services.briefing_scheduler import stop_briefing_scheduler

src/fabledassistant/config.py

@@ -66,6 +66,12 @@ class Config:
     VAPID_PUBLIC_KEY: str = os.environ.get("VAPID_PUBLIC_KEY", "")
     VAPID_CLAIMS_SUB: str = os.environ.get("VAPID_CLAIMS_SUB", "mailto:admin@fabledassistant.local")
 
+    # Voice (Speech-to-Speech) feature
+    VOICE_ENABLED: bool = os.environ.get("VOICE_ENABLED", "").lower() in ("1", "true", "yes")
+    STT_BACKEND: str = os.environ.get("STT_BACKEND", "faster-whisper")
+    STT_MODEL: str = os.environ.get("STT_MODEL", "base.en")
+    TTS_BACKEND: str = os.environ.get("TTS_BACKEND", "kokoro")
+
     @classmethod
     def oidc_enabled(cls) -> bool:
         return bool(cls.OIDC_ISSUER and cls.OIDC_CLIENT_ID and cls.OIDC_CLIENT_SECRET)
@@ -93,5 +99,11 @@ class Config:
                 "SECRET_KEY is set to the insecure default but SECURE_COOKIES=true indicates "
                 "a production deployment. Set SECRET_KEY or SECRET_KEY_FILE before starting."
             )
+        _valid_stt_models = {"tiny.en", "base.en", "small.en", "medium.en"}
+        if cls.VOICE_ENABLED and cls.STT_MODEL not in _valid_stt_models:
+            errors.append(
+                f"STT_MODEL='{cls.STT_MODEL}' is not supported. "
+                f"Valid values: {', '.join(sorted(_valid_stt_models))}"
+            )
         if errors:
             raise ValueError("Configuration errors:\n" + "\n".join(f"  - {e}" for e in errors))

src/fabledassistant/routes/chat.py

@@ -76,7 +76,7 @@ async def create_conversation_route():
     model = data.get("model", Config.OLLAMA_MODEL)
     conversation_type = data.get("conversation_type", "chat")
     # Only allow known types to prevent accidental misuse
-    if conversation_type not in ("chat", "mcp"):
+    if conversation_type not in ("chat", "mcp", "voice"):
         conversation_type = "chat"
     conv = await create_conversation(uid, title=title, model=model, conversation_type=conversation_type)
     return jsonify(conv.to_dict()), 201
@@ -187,6 +187,7 @@ async def send_message_route(conv_id: int):
         rag_project_id=effective_rag_project_id,
         workspace_project_id=workspace_project_id,
         user_timezone=user_timezone,
+        voice_mode=(conv.conversation_type == "voice"),
     ))
 
     return jsonify({

src/fabledassistant/routes/voice.py

@@ -0,0 +1,130 @@
+"""Voice (Speech-to-Speech) routes at /api/voice."""
+import logging
+import time
+
+from quart import Blueprint, jsonify, request
+
+from fabledassistant.auth import login_required
+from fabledassistant.config import Config
+
+logger = logging.getLogger(__name__)
+
+voice_bp = Blueprint("voice", __name__, url_prefix="/api/voice")
+
+
+@voice_bp.route("/status", methods=["GET"])
+@login_required
+async def voice_status():
+    """Return availability of STT and TTS services."""
+    if not Config.VOICE_ENABLED:
+        return jsonify({"enabled": False, "stt": False, "tts": False})
+
+    from fabledassistant.services.stt import stt_available
+    from fabledassistant.services.tts import tts_available
+
+    return jsonify({
+        "enabled": True,
+        "stt": stt_available(),
+        "tts": tts_available(),
+        "stt_model": Config.STT_MODEL,
+        "tts_backend": Config.TTS_BACKEND,
+    })
+
+
+@voice_bp.route("/voices", methods=["GET"])
+@login_required
+async def list_voices():
+    """Return available Kokoro voice IDs and labels."""
+    if not Config.VOICE_ENABLED:
+        return jsonify({"error": "Voice feature is disabled"}), 503
+
+    from fabledassistant.services.tts import list_voices, tts_available
+
+    if not tts_available():
+        return jsonify({"error": "TTS not available"}), 503
+
+    return jsonify({"voices": list_voices()})
+
+
+@voice_bp.route("/transcribe", methods=["POST"])
+@login_required
+async def transcribe_audio():
+    """Accept a multipart audio file and return the transcript.
+
+    Request: multipart/form-data with field 'audio' (WebM/Opus blob)
+    Response: {"transcript": "...", "duration_ms": 123}
+    """
+    if not Config.VOICE_ENABLED:
+        return jsonify({"error": "Voice feature is disabled"}), 503
+
+    from fabledassistant.services.stt import stt_available, transcribe
+
+    if not stt_available():
+        return jsonify({"error": "STT not available — model may still be loading"}), 503
+
+    files = await request.files
+    audio_file = files.get("audio")
+    if audio_file is None:
+        return jsonify({"error": "No audio file provided"}), 400
+
+    audio_bytes = audio_file.read()
+    if not audio_bytes:
+        return jsonify({"error": "Empty audio file"}), 400
+
+    if len(audio_bytes) > 25 * 1024 * 1024:  # 25 MB hard cap
+        return jsonify({"error": "Audio file too large (max 25 MB)"}), 413
+
+    mime_type = audio_file.content_type or "audio/webm"
+
+    t0 = time.monotonic()
+    try:
+        transcript = await transcribe(audio_bytes, mime_type)
+    except Exception:
+        logger.exception("STT transcription failed")
+        return jsonify({"error": "Transcription failed"}), 500
+
+    duration_ms = round((time.monotonic() - t0) * 1000)
+    return jsonify({"transcript": transcript, "duration_ms": duration_ms})
+
+
+@voice_bp.route("/synthesise", methods=["POST"])
+@login_required
+async def synthesise_speech():
+    """Convert text to speech and return WAV bytes.
+
+    Request body: {"text": "...", "voice": "af_heart", "speed": 1.0}
+    Response: audio/wav bytes
+    """
+    if not Config.VOICE_ENABLED:
+        return jsonify({"error": "Voice feature is disabled"}), 503
+
+    from fabledassistant.services.tts import synthesise, tts_available
+
+    if not tts_available():
+        return jsonify({"error": "TTS not available — model may still be loading"}), 503
+
+    data = await request.get_json()
+    if not data:
+        return jsonify({"error": "JSON body required"}), 400
+
+    text = str(data.get("text", "")).strip()
+    if not text:
+        return jsonify({"error": "text is required"}), 400
+
+    if len(text) > 8000:
+        return jsonify({"error": "text too long (max 8000 characters)"}), 400
+
+    voice = str(data.get("voice", "af_heart"))
+    try:
+        speed = float(data.get("speed", 1.0))
+    except (TypeError, ValueError):
+        speed = 1.0
+
+    try:
+        wav_bytes = await synthesise(text, voice=voice, speed=speed)
+    except Exception:
+        logger.exception("TTS synthesis failed")
+        return jsonify({"error": "Synthesis failed"}), 500
+
+    from quart import Response
+    return Response(wav_bytes, mimetype="audio/wav")

src/fabledassistant/services/chat.py

@@ -131,7 +131,8 @@ async def cleanup_old_conversations(user_id: int, days: int) -> int:
             .where(
                 Conversation.user_id == user_id,
                 Conversation.updated_at < cutoff,
-                Conversation.conversation_type != "mcp",  # preserve MCP audit trail
+                Conversation.conversation_type != "mcp",   # preserve MCP audit trail
+                Conversation.conversation_type != "voice",  # voice convs managed separately
             )
             .returning(Conversation.id)
         )

src/fabledassistant/services/generation_task.py

@@ -152,6 +152,7 @@ async def run_generation(
     rag_project_id: int | None = None,
     workspace_project_id: int | None = None,
     user_timezone: str | None = None,
+    voice_mode: bool = False,
 ) -> None:
     """Stream LLM response into buffer with periodic DB flushes."""
     MAX_TOOL_ROUNDS = 5
@@ -177,6 +178,12 @@ async def run_generation(
     # Phase 3: Build context and wait for model in parallel.
     model_load_task = asyncio.create_task(wait_for_model_loaded(model, timeout=180.0))
 
+    # Fetch voice_speech_style from user settings when voice_mode is active.
+    voice_speech_style = "conversational"
+    if voice_mode:
+        from fabledassistant.services.settings import get_setting
+        voice_speech_style = await get_setting(user_id, "voice_speech_style", "conversational")
+
     context_task = asyncio.create_task(build_context(
         user_id, history_to_use, context_note_id, user_content,
         history_summary=history_summary,
@@ -186,6 +193,8 @@ async def run_generation(
         workspace_project_id=workspace_project_id,
         user_timezone=user_timezone,
         conv_id=conv_id,
+        voice_mode=voice_mode,
+        voice_speech_style=voice_speech_style,
     ))
 
     messages, context_meta = await context_task

src/fabledassistant/services/llm.py

@@ -454,6 +454,8 @@ async def build_context(
     workspace_project_id: int | None = None,
     user_timezone: str | None = None,
     conv_id: int | None = None,
+    voice_mode: bool = False,
+    voice_speech_style: str = "conversational",
 ) -> tuple[list[dict], dict]:
     """Build messages array for Ollama with system prompt and context.
 
@@ -516,6 +518,19 @@ async def build_context(
         f"{tool_guidance}"
     ]
 
+    if voice_mode:
+        _style_hints = {
+            "conversational": "Be warm, natural, and conversational — like speaking to a friend.",
+            "concise": "Be brief and to the point. One or two sentences maximum unless detail is essential.",
+            "detailed": "Give thorough, informative responses as if narrating an explanation aloud.",
+        }
+        style_hint = _style_hints.get(voice_speech_style, _style_hints["conversational"])
+        system_parts.insert(0,
+            "VOICE MODE: Respond naturally as if speaking aloud. "
+            "No markdown, bullet points, headers, or code blocks. Complete sentences only. "
+            f"{style_hint}\n\n"
+        )
+
     context_meta: dict = {
         "context_note_id": None,
         "context_note_title": None,

src/fabledassistant/services/stt.py

@@ -0,0 +1,72 @@
+"""Speech-to-text service using faster-whisper (in-process, CPU/GPU)."""
+import asyncio
+import logging
+import tempfile
+import time
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from faster_whisper import WhisperModel
+
+logger = logging.getLogger(__name__)
+
+_model: "WhisperModel | None" = None
+_model_lock = asyncio.Lock()
+_load_error: str | None = None
+
+
+async def load_stt_model() -> None:
+    """Load the Whisper model. Called once at startup when VOICE_ENABLED=true."""
+    global _model, _load_error
+    from fabledassistant.config import Config
+
+    if not Config.VOICE_ENABLED:
+        return
+
+    async with _model_lock:
+        if _model is not None:
+            return
+        try:
+            from faster_whisper import WhisperModel
+
+            logger.info("Loading Whisper STT model '%s'...", Config.STT_MODEL)
+            loop = asyncio.get_running_loop()
+            _model = await loop.run_in_executor(
+                None,
+                lambda: WhisperModel(Config.STT_MODEL, device="cpu", compute_type="int8"),
+            )
+            logger.info("Whisper STT model '%s' loaded", Config.STT_MODEL)
+        except Exception:
+            _load_error = f"Failed to load Whisper model '{Config.STT_MODEL}'"
+            logger.exception(_load_error)
+
+
+def stt_available() -> bool:
+    return _model is not None
+
+
+async def transcribe(audio_bytes: bytes, mime_type: str = "audio/webm") -> str:
+    """Transcribe audio bytes to text. Runs the model in a thread executor."""
+    if _model is None:
+        raise RuntimeError("STT model not loaded")
+
+    suffix = ".webm"
+    if "ogg" in mime_type:
+        suffix = ".ogg"
+    elif "wav" in mime_type:
+        suffix = ".wav"
+    elif "mp4" in mime_type or "m4a" in mime_type:
+        suffix = ".mp4"
+
+    def _run() -> str:
+        with tempfile.NamedTemporaryFile(suffix=suffix, delete=True) as f:
+            f.write(audio_bytes)
+            f.flush()
+            t0 = time.monotonic()
+            segments, _ = _model.transcribe(f.name, beam_size=5)  # type: ignore[union-attr]
+            text = " ".join(seg.text.strip() for seg in segments).strip()
+            logger.debug("STT transcription took %.2fs", time.monotonic() - t0)
+            return text
+
+    loop = asyncio.get_running_loop()
+    return await loop.run_in_executor(None, _run)

src/fabledassistant/services/tts.py

@@ -0,0 +1,94 @@
+"""Text-to-speech service using Kokoro TTS (in-process)."""
+import asyncio
+import io
+import logging
+import time
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from kokoro import KPipeline
+
+logger = logging.getLogger(__name__)
+
+_pipeline: "KPipeline | None" = None
+_pipeline_lock = asyncio.Lock()
+_load_error: str | None = None
+
+# Static list of supported Kokoro voice IDs and display labels
+_VOICES: list[dict] = [
+    {"id": "af_heart", "label": "Heart (American Female, warm)"},
+    {"id": "af_bella", "label": "Bella (American Female, expressive)"},
+    {"id": "af_nicole", "label": "Nicole (American Female, intimate)"},
+    {"id": "af_sarah", "label": "Sarah (American Female, clear)"},
+    {"id": "af_sky", "label": "Sky (American Female, bright)"},
+    {"id": "am_adam", "label": "Adam (American Male, neutral)"},
+    {"id": "am_michael", "label": "Michael (American Male, deep)"},
+    {"id": "bf_emma", "label": "Emma (British Female)"},
+    {"id": "bf_isabella", "label": "Isabella (British Female, formal)"},
+    {"id": "bm_george", "label": "George (British Male)"},
+    {"id": "bm_lewis", "label": "Lewis (British Male, casual)"},
+]
+
+
+async def load_tts_model() -> None:
+    """Load the Kokoro pipeline. Called once at startup when VOICE_ENABLED=true."""
+    global _pipeline, _load_error
+    from fabledassistant.config import Config
+
+    if not Config.VOICE_ENABLED:
+        return
+
+    async with _pipeline_lock:
+        if _pipeline is not None:
+            return
+        try:
+            from kokoro import KPipeline
+
+            logger.info("Loading Kokoro TTS pipeline...")
+            loop = asyncio.get_running_loop()
+            _pipeline = await loop.run_in_executor(
+                None,
+                lambda: KPipeline(lang_code="a"),  # "a" = American English
+            )
+            logger.info("Kokoro TTS pipeline loaded")
+        except Exception:
+            _load_error = "Failed to load Kokoro TTS pipeline"
+            logger.exception(_load_error)
+
+
+def tts_available() -> bool:
+    return _pipeline is not None
+
+
+def list_voices() -> list[dict]:
+    return _VOICES
+
+
+async def synthesise(text: str, voice: str = "af_heart", speed: float = 1.0) -> bytes:
+    """Synthesise text to WAV bytes (24kHz, 16-bit mono). Runs in executor."""
+    if _pipeline is None:
+        raise RuntimeError("TTS pipeline not loaded")
+
+    speed = max(0.7, min(1.3, speed))
+
+    def _run() -> bytes:
+        import numpy as np
+        import soundfile as sf
+
+        t0 = time.monotonic()
+        audio_chunks: list = []
+        for _, _, audio in _pipeline(text, voice=voice, speed=speed):  # type: ignore[misc]
+            if audio is not None:
+                audio_chunks.append(audio)
+
+        if not audio_chunks:
+            return b""
+
+        combined = np.concatenate(audio_chunks)
+        buf = io.BytesIO()
+        sf.write(buf, combined, samplerate=24000, format="WAV", subtype="PCM_16")
+        logger.debug("TTS synthesis took %.2fs for %d chars", time.monotonic() - t0, len(text))
+        return buf.getvalue()
+
+    loop = asyncio.get_running_loop()
+    return await loop.run_in_executor(None, _run)