diff --git a/Dockerfile b/Dockerfile index c5d5342..1d5d203 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,8 @@ WORKDIR /app COPY pyproject.toml . COPY src/ src/ RUN pip install --no-cache-dir . +# Voice dependencies (faster-whisper, Kokoro TTS, soundfile) — activated at runtime via VOICE_ENABLED +RUN pip install --no-cache-dir faster-whisper kokoro soundfile # Build the fable-mcp wheel so it can be served for download COPY fable-mcp/ fable-mcp/ diff --git a/frontend/src/App.vue b/frontend/src/App.vue index 3021118..7220313 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -3,6 +3,7 @@ import { onMounted, onUnmounted, ref, watch } from "vue"; import { useRouter } from "vue-router"; import AppHeader from "@/components/AppHeader.vue"; import ToastNotification from "@/components/ToastNotification.vue"; +import VoiceOverlay from "@/components/VoiceOverlay.vue"; import { useTheme } from "@/composables/useTheme"; import { useShortcuts } from "@/composables/useShortcuts"; import { useAuthStore } from "@/stores/auth"; @@ -47,9 +48,21 @@ function clearPrefix() { prefixTimeout = null; } +let spaceHeld = false; + function onGlobalKeydown(e: KeyboardEvent) { if (!authStore.isAuthenticated) return; + // Space — PTT start (only when not typing, no modifiers, no repeat) + if (e.key === " " && !isInputActive() && !e.ctrlKey && !e.metaKey && !e.altKey && !e.repeat) { + e.preventDefault(); + if (!spaceHeld) { + spaceHeld = true; + document.dispatchEvent(new CustomEvent("voice:ptt-toggle")); + } + return; + } + // ? — toggle shortcuts overlay (only when not typing) if (e.key === "?" && !isInputActive() && !e.ctrlKey && !e.metaKey) { toggleShortcuts(); @@ -122,8 +135,16 @@ function onGlobalKeydown(e: KeyboardEvent) { } } +function onGlobalKeyup(e: KeyboardEvent) { + if (e.key === " " && spaceHeld) { + spaceHeld = false; + document.dispatchEvent(new CustomEvent("voice:ptt-toggle")); + } +} + onMounted(async () => { document.addEventListener("keydown", onGlobalKeydown); + document.addEventListener("keyup", onGlobalKeyup); await authStore.checkAuth(); if (authStore.isAuthenticated) { startAppServices(); @@ -149,6 +170,7 @@ watch( onUnmounted(() => { document.removeEventListener("keydown", onGlobalKeydown); + document.removeEventListener("keyup", onGlobalKeyup); stopAppServices(); }); @@ -164,6 +186,9 @@ onUnmounted(() => { + + +
@@ -268,6 +293,10 @@ onUnmounted(() => { Enter New line
+
+ Space + Hold to speak (voice, when enabled) +
diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 8af972c..9e1b2e7 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -619,3 +619,51 @@ export function getNewsItems(params: GetNewsItemsParams = {}) { `/api/briefing/news?${p}` ) } + +// ─── Voice ──────────────────────────────────────────────────────────────────── + +export interface VoiceStatusResult { + enabled: boolean + stt: boolean + tts: boolean + stt_model?: string + tts_backend?: string +} + +export interface VoiceEntry { + id: string + label: string +} + +export const getVoiceStatus = () => apiGet('/api/voice/status') + +export const getVoiceList = () => + apiGet<{ voices: VoiceEntry[] }>('/api/voice/voices').then(r => r.voices) + +export async function transcribeAudio(blob: Blob): Promise<{ transcript: string; duration_ms: number }> { + const form = new FormData() + form.append('audio', blob, 'audio.webm') + const res = await fetch('/api/voice/transcribe', { method: 'POST', body: form }) + if (!res.ok) { + const err = await res.json().catch(() => ({ error: `HTTP ${res.status}` })) + throw new ApiError(res.status, err) + } + return res.json() +} + +export async function synthesiseSpeech( + text: string, + voice?: string, + speed?: number +): Promise { + const res = await fetch('/api/voice/synthesise', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ text, voice: voice ?? 'af_heart', speed: speed ?? 1.0 }), + }) + if (!res.ok) { + const err = await res.json().catch(() => ({ error: `HTTP ${res.status}` })) + throw new ApiError(res.status, err) + } + return res.blob() +} diff --git a/frontend/src/components/VoiceOverlay.vue b/frontend/src/components/VoiceOverlay.vue new file mode 100644 index 0000000..7445a30 --- /dev/null +++ b/frontend/src/components/VoiceOverlay.vue @@ -0,0 +1,536 @@ + + + + + diff --git a/frontend/src/composables/useVoiceAudio.ts b/frontend/src/composables/useVoiceAudio.ts new file mode 100644 index 0000000..6645f65 --- /dev/null +++ b/frontend/src/composables/useVoiceAudio.ts @@ -0,0 +1,60 @@ +import { ref, readonly } from 'vue' + +/** + * Audio playback composable wrapping the Web Audio API. + * + * Usage: + * const { playing, isSupported, play, stop } = useVoiceAudio() + * await play(wavBlob) + */ +export function useVoiceAudio() { + const playing = ref(false) + const isSupported = typeof AudioContext !== 'undefined' || typeof (window as unknown as Record).webkitAudioContext !== 'undefined' + + let audioCtx: AudioContext | null = null + let currentSource: AudioBufferSourceNode | null = null + + function _getCtx(): AudioContext { + if (!audioCtx || audioCtx.state === 'closed') { + const Ctx = window.AudioContext ?? (window as unknown as Record).webkitAudioContext + audioCtx = new Ctx() + } + return audioCtx + } + + async function play(blob: Blob): Promise { + stop() + const ctx = _getCtx() + if (ctx.state === 'suspended') await ctx.resume() + + const arrayBuffer = await blob.arrayBuffer() + const audioBuffer = await ctx.decodeAudioData(arrayBuffer) + + const source = ctx.createBufferSource() + source.buffer = audioBuffer + source.connect(ctx.destination) + currentSource = source + playing.value = true + + source.onended = () => { + playing.value = false + currentSource = null + } + source.start(0) + } + + function stop(): void { + if (currentSource) { + try { currentSource.stop() } catch { /* already stopped */ } + currentSource = null + } + playing.value = false + } + + return { + playing: readonly(playing), + isSupported, + play, + stop, + } +} diff --git a/frontend/src/composables/useVoiceRecorder.ts b/frontend/src/composables/useVoiceRecorder.ts new file mode 100644 index 0000000..2b55a57 --- /dev/null +++ b/frontend/src/composables/useVoiceRecorder.ts @@ -0,0 +1,92 @@ +import { ref, readonly } from 'vue' + +/** + * Push-to-talk recorder wrapping the browser MediaRecorder API. + * + * Usage: + * const { recording, error, isSupported, startRecording, stopRecording } = useVoiceRecorder() + * await startRecording() + * const blob = await stopRecording() // resolves with the recorded audio Blob + */ +export function useVoiceRecorder() { + const recording = ref(false) + const error = ref(null) + const isSupported = typeof MediaRecorder !== 'undefined' && !!navigator.mediaDevices?.getUserMedia + + let mediaRecorder: MediaRecorder | null = null + let chunks: Blob[] = [] + let stream: MediaStream | null = null + let resolveStop: ((blob: Blob) => void) | null = null + let rejectStop: ((err: Error) => void) | null = null + + async function startRecording(): Promise { + error.value = null + if (!isSupported) { + error.value = 'Audio recording is not supported in this browser' + return + } + if (recording.value) return + + try { + stream = await navigator.mediaDevices.getUserMedia({ audio: true }) + } catch (e) { + error.value = 'Microphone access denied' + return + } + + chunks = [] + const mimeType = MediaRecorder.isTypeSupported('audio/webm;codecs=opus') + ? 'audio/webm;codecs=opus' + : MediaRecorder.isTypeSupported('audio/webm') + ? 'audio/webm' + : '' + + mediaRecorder = mimeType ? new MediaRecorder(stream, { mimeType }) : new MediaRecorder(stream) + + mediaRecorder.ondataavailable = (e) => { + if (e.data.size > 0) chunks.push(e.data) + } + + mediaRecorder.onstop = () => { + const blob = new Blob(chunks, { type: mediaRecorder?.mimeType ?? 'audio/webm' }) + chunks = [] + stream?.getTracks().forEach((t) => t.stop()) + stream = null + recording.value = false + resolveStop?.(blob) + resolveStop = null + rejectStop = null + } + + mediaRecorder.onerror = () => { + recording.value = false + error.value = 'Recording error' + rejectStop?.(new Error('MediaRecorder error')) + resolveStop = null + rejectStop = null + } + + mediaRecorder.start(100) // collect in 100ms chunks + recording.value = true + } + + function stopRecording(): Promise { + return new Promise((resolve, reject) => { + if (!mediaRecorder || !recording.value) { + reject(new Error('Not recording')) + return + } + resolveStop = resolve + rejectStop = reject + mediaRecorder.stop() + }) + } + + return { + recording: readonly(recording), + error: readonly(error), + isSupported, + startRecording, + stopRecording, + } +} diff --git a/frontend/src/views/BriefingView.vue b/frontend/src/views/BriefingView.vue index ff05bc9..97f83cb 100644 --- a/frontend/src/views/BriefingView.vue +++ b/frontend/src/views/BriefingView.vue @@ -1,6 +1,8 @@ @@ -281,6 +368,29 @@ onMounted(async () => { + + + + + + + +
@@ -3528,4 +3676,60 @@ FABLE_API_KEY=<your-api-key> word-break: break-all; overflow-x: auto; } + +/* Voice tab */ +.voice-status-row { + display: flex; + align-items: center; + gap: 0.5rem; + flex-wrap: wrap; + margin-top: 0.5rem; +} +.status-badge { + font-size: 0.75rem; + font-weight: 600; + padding: 0.15rem 0.55rem; + border-radius: 999px; +} +.status-on { + background: color-mix(in srgb, var(--color-success, #22c55e) 15%, transparent); + color: var(--color-success, #22c55e); + border: 1px solid color-mix(in srgb, var(--color-success, #22c55e) 40%, transparent); +} +.status-off { + background: color-mix(in srgb, var(--color-text-muted) 10%, transparent); + color: var(--color-text-muted); + border: 1px solid var(--color-border); +} +.radio-group { + display: flex; + flex-direction: column; + gap: 0.65rem; + margin-top: 0.35rem; +} +.radio-option { + display: flex; + align-items: flex-start; + gap: 0.55rem; + cursor: pointer; +} +.radio-option input[type="radio"] { margin-top: 0.2rem; flex-shrink: 0; } +.radio-option span { display: flex; flex-direction: column; gap: 0.15rem; } +.radio-option strong { font-size: 0.875rem; } +.range-input { + width: 100%; + max-width: 360px; + accent-color: var(--color-primary); + margin: 0.35rem 0 0.2rem; +} +.range-labels { + display: flex; + justify-content: space-between; + max-width: 360px; + font-size: 0.75rem; + color: var(--color-text-muted); +} +.form-actions { + margin-top: 1rem; +} diff --git a/pyproject.toml b/pyproject.toml index 085493d..f097880 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,11 @@ dev = [ "pytest-asyncio>=0.23", "ruff>=0.6", ] +voice = [ + "faster-whisper>=1.0", + "kokoro>=0.9", + "soundfile>=0.12", +] [tool.setuptools.packages.find] where = ["src"] diff --git a/src/fabledassistant/app.py b/src/fabledassistant/app.py index 4bac5db..3d9062b 100644 --- a/src/fabledassistant/app.py +++ b/src/fabledassistant/app.py @@ -31,6 +31,7 @@ from fabledassistant.routes.users import users_bp from fabledassistant.routes.api_keys import api_keys_bp from fabledassistant.routes.events import events_bp from fabledassistant.routes.search import search_bp +from fabledassistant.routes.voice import voice_bp STATIC_DIR = Path(__file__).parent / "static" logger = logging.getLogger(__name__) @@ -92,6 +93,7 @@ def create_app() -> Quart: app.register_blueprint(api_keys_bp) app.register_blueprint(events_bp) app.register_blueprint(search_bp) + app.register_blueprint(voice_bp) @app.before_request async def before_request(): @@ -264,6 +266,13 @@ def create_app() -> Quart: from fabledassistant.services.briefing_scheduler import start_briefing_scheduler await start_briefing_scheduler(asyncio.get_running_loop()) + # Voice model loading (only when feature is enabled) + if Config.VOICE_ENABLED: + from fabledassistant.services.stt import load_stt_model + from fabledassistant.services.tts import load_tts_model + asyncio.create_task(load_stt_model()) + asyncio.create_task(load_tts_model()) + @app.after_serving async def shutdown(): from fabledassistant.services.briefing_scheduler import stop_briefing_scheduler diff --git a/src/fabledassistant/config.py b/src/fabledassistant/config.py index f8b08b4..e15a364 100644 --- a/src/fabledassistant/config.py +++ b/src/fabledassistant/config.py @@ -66,6 +66,12 @@ class Config: VAPID_PUBLIC_KEY: str = os.environ.get("VAPID_PUBLIC_KEY", "") VAPID_CLAIMS_SUB: str = os.environ.get("VAPID_CLAIMS_SUB", "mailto:admin@fabledassistant.local") + # Voice (Speech-to-Speech) feature + VOICE_ENABLED: bool = os.environ.get("VOICE_ENABLED", "").lower() in ("1", "true", "yes") + STT_BACKEND: str = os.environ.get("STT_BACKEND", "faster-whisper") + STT_MODEL: str = os.environ.get("STT_MODEL", "base.en") + TTS_BACKEND: str = os.environ.get("TTS_BACKEND", "kokoro") + @classmethod def oidc_enabled(cls) -> bool: return bool(cls.OIDC_ISSUER and cls.OIDC_CLIENT_ID and cls.OIDC_CLIENT_SECRET) @@ -93,5 +99,11 @@ class Config: "SECRET_KEY is set to the insecure default but SECURE_COOKIES=true indicates " "a production deployment. Set SECRET_KEY or SECRET_KEY_FILE before starting." ) + _valid_stt_models = {"tiny.en", "base.en", "small.en", "medium.en"} + if cls.VOICE_ENABLED and cls.STT_MODEL not in _valid_stt_models: + errors.append( + f"STT_MODEL='{cls.STT_MODEL}' is not supported. " + f"Valid values: {', '.join(sorted(_valid_stt_models))}" + ) if errors: raise ValueError("Configuration errors:\n" + "\n".join(f" - {e}" for e in errors)) diff --git a/src/fabledassistant/routes/chat.py b/src/fabledassistant/routes/chat.py index 7883d60..5949308 100644 --- a/src/fabledassistant/routes/chat.py +++ b/src/fabledassistant/routes/chat.py @@ -76,7 +76,7 @@ async def create_conversation_route(): model = data.get("model", Config.OLLAMA_MODEL) conversation_type = data.get("conversation_type", "chat") # Only allow known types to prevent accidental misuse - if conversation_type not in ("chat", "mcp"): + if conversation_type not in ("chat", "mcp", "voice"): conversation_type = "chat" conv = await create_conversation(uid, title=title, model=model, conversation_type=conversation_type) return jsonify(conv.to_dict()), 201 @@ -187,6 +187,7 @@ async def send_message_route(conv_id: int): rag_project_id=effective_rag_project_id, workspace_project_id=workspace_project_id, user_timezone=user_timezone, + voice_mode=(conv.conversation_type == "voice"), )) return jsonify({ diff --git a/src/fabledassistant/routes/voice.py b/src/fabledassistant/routes/voice.py new file mode 100644 index 0000000..10e18b5 --- /dev/null +++ b/src/fabledassistant/routes/voice.py @@ -0,0 +1,130 @@ +"""Voice (Speech-to-Speech) routes at /api/voice.""" +import logging +import time + +from quart import Blueprint, jsonify, request + +from fabledassistant.auth import login_required +from fabledassistant.config import Config + +logger = logging.getLogger(__name__) + +voice_bp = Blueprint("voice", __name__, url_prefix="/api/voice") + + +@voice_bp.route("/status", methods=["GET"]) +@login_required +async def voice_status(): + """Return availability of STT and TTS services.""" + if not Config.VOICE_ENABLED: + return jsonify({"enabled": False, "stt": False, "tts": False}) + + from fabledassistant.services.stt import stt_available + from fabledassistant.services.tts import tts_available + + return jsonify({ + "enabled": True, + "stt": stt_available(), + "tts": tts_available(), + "stt_model": Config.STT_MODEL, + "tts_backend": Config.TTS_BACKEND, + }) + + +@voice_bp.route("/voices", methods=["GET"]) +@login_required +async def list_voices(): + """Return available Kokoro voice IDs and labels.""" + if not Config.VOICE_ENABLED: + return jsonify({"error": "Voice feature is disabled"}), 503 + + from fabledassistant.services.tts import list_voices, tts_available + + if not tts_available(): + return jsonify({"error": "TTS not available"}), 503 + + return jsonify({"voices": list_voices()}) + + +@voice_bp.route("/transcribe", methods=["POST"]) +@login_required +async def transcribe_audio(): + """Accept a multipart audio file and return the transcript. + + Request: multipart/form-data with field 'audio' (WebM/Opus blob) + Response: {"transcript": "...", "duration_ms": 123} + """ + if not Config.VOICE_ENABLED: + return jsonify({"error": "Voice feature is disabled"}), 503 + + from fabledassistant.services.stt import stt_available, transcribe + + if not stt_available(): + return jsonify({"error": "STT not available — model may still be loading"}), 503 + + files = await request.files + audio_file = files.get("audio") + if audio_file is None: + return jsonify({"error": "No audio file provided"}), 400 + + audio_bytes = audio_file.read() + if not audio_bytes: + return jsonify({"error": "Empty audio file"}), 400 + + if len(audio_bytes) > 25 * 1024 * 1024: # 25 MB hard cap + return jsonify({"error": "Audio file too large (max 25 MB)"}), 413 + + mime_type = audio_file.content_type or "audio/webm" + + t0 = time.monotonic() + try: + transcript = await transcribe(audio_bytes, mime_type) + except Exception: + logger.exception("STT transcription failed") + return jsonify({"error": "Transcription failed"}), 500 + + duration_ms = round((time.monotonic() - t0) * 1000) + return jsonify({"transcript": transcript, "duration_ms": duration_ms}) + + +@voice_bp.route("/synthesise", methods=["POST"]) +@login_required +async def synthesise_speech(): + """Convert text to speech and return WAV bytes. + + Request body: {"text": "...", "voice": "af_heart", "speed": 1.0} + Response: audio/wav bytes + """ + if not Config.VOICE_ENABLED: + return jsonify({"error": "Voice feature is disabled"}), 503 + + from fabledassistant.services.tts import synthesise, tts_available + + if not tts_available(): + return jsonify({"error": "TTS not available — model may still be loading"}), 503 + + data = await request.get_json() + if not data: + return jsonify({"error": "JSON body required"}), 400 + + text = str(data.get("text", "")).strip() + if not text: + return jsonify({"error": "text is required"}), 400 + + if len(text) > 8000: + return jsonify({"error": "text too long (max 8000 characters)"}), 400 + + voice = str(data.get("voice", "af_heart")) + try: + speed = float(data.get("speed", 1.0)) + except (TypeError, ValueError): + speed = 1.0 + + try: + wav_bytes = await synthesise(text, voice=voice, speed=speed) + except Exception: + logger.exception("TTS synthesis failed") + return jsonify({"error": "Synthesis failed"}), 500 + + from quart import Response + return Response(wav_bytes, mimetype="audio/wav") diff --git a/src/fabledassistant/services/chat.py b/src/fabledassistant/services/chat.py index 0e5cff8..de00cd9 100644 --- a/src/fabledassistant/services/chat.py +++ b/src/fabledassistant/services/chat.py @@ -131,7 +131,8 @@ async def cleanup_old_conversations(user_id: int, days: int) -> int: .where( Conversation.user_id == user_id, Conversation.updated_at < cutoff, - Conversation.conversation_type != "mcp", # preserve MCP audit trail + Conversation.conversation_type != "mcp", # preserve MCP audit trail + Conversation.conversation_type != "voice", # voice convs managed separately ) .returning(Conversation.id) ) diff --git a/src/fabledassistant/services/generation_task.py b/src/fabledassistant/services/generation_task.py index c696cac..4f26c51 100644 --- a/src/fabledassistant/services/generation_task.py +++ b/src/fabledassistant/services/generation_task.py @@ -152,6 +152,7 @@ async def run_generation( rag_project_id: int | None = None, workspace_project_id: int | None = None, user_timezone: str | None = None, + voice_mode: bool = False, ) -> None: """Stream LLM response into buffer with periodic DB flushes.""" MAX_TOOL_ROUNDS = 5 @@ -177,6 +178,12 @@ async def run_generation( # Phase 3: Build context and wait for model in parallel. model_load_task = asyncio.create_task(wait_for_model_loaded(model, timeout=180.0)) + # Fetch voice_speech_style from user settings when voice_mode is active. + voice_speech_style = "conversational" + if voice_mode: + from fabledassistant.services.settings import get_setting + voice_speech_style = await get_setting(user_id, "voice_speech_style", "conversational") + context_task = asyncio.create_task(build_context( user_id, history_to_use, context_note_id, user_content, history_summary=history_summary, @@ -186,6 +193,8 @@ async def run_generation( workspace_project_id=workspace_project_id, user_timezone=user_timezone, conv_id=conv_id, + voice_mode=voice_mode, + voice_speech_style=voice_speech_style, )) messages, context_meta = await context_task diff --git a/src/fabledassistant/services/llm.py b/src/fabledassistant/services/llm.py index 9903f96..935cf68 100644 --- a/src/fabledassistant/services/llm.py +++ b/src/fabledassistant/services/llm.py @@ -454,6 +454,8 @@ async def build_context( workspace_project_id: int | None = None, user_timezone: str | None = None, conv_id: int | None = None, + voice_mode: bool = False, + voice_speech_style: str = "conversational", ) -> tuple[list[dict], dict]: """Build messages array for Ollama with system prompt and context. @@ -516,6 +518,19 @@ async def build_context( f"{tool_guidance}" ] + if voice_mode: + _style_hints = { + "conversational": "Be warm, natural, and conversational — like speaking to a friend.", + "concise": "Be brief and to the point. One or two sentences maximum unless detail is essential.", + "detailed": "Give thorough, informative responses as if narrating an explanation aloud.", + } + style_hint = _style_hints.get(voice_speech_style, _style_hints["conversational"]) + system_parts.insert(0, + "VOICE MODE: Respond naturally as if speaking aloud. " + "No markdown, bullet points, headers, or code blocks. Complete sentences only. " + f"{style_hint}\n\n" + ) + context_meta: dict = { "context_note_id": None, "context_note_title": None, diff --git a/src/fabledassistant/services/stt.py b/src/fabledassistant/services/stt.py new file mode 100644 index 0000000..38283be --- /dev/null +++ b/src/fabledassistant/services/stt.py @@ -0,0 +1,72 @@ +"""Speech-to-text service using faster-whisper (in-process, CPU/GPU).""" +import asyncio +import logging +import tempfile +import time +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from faster_whisper import WhisperModel + +logger = logging.getLogger(__name__) + +_model: "WhisperModel | None" = None +_model_lock = asyncio.Lock() +_load_error: str | None = None + + +async def load_stt_model() -> None: + """Load the Whisper model. Called once at startup when VOICE_ENABLED=true.""" + global _model, _load_error + from fabledassistant.config import Config + + if not Config.VOICE_ENABLED: + return + + async with _model_lock: + if _model is not None: + return + try: + from faster_whisper import WhisperModel + + logger.info("Loading Whisper STT model '%s'...", Config.STT_MODEL) + loop = asyncio.get_running_loop() + _model = await loop.run_in_executor( + None, + lambda: WhisperModel(Config.STT_MODEL, device="cpu", compute_type="int8"), + ) + logger.info("Whisper STT model '%s' loaded", Config.STT_MODEL) + except Exception: + _load_error = f"Failed to load Whisper model '{Config.STT_MODEL}'" + logger.exception(_load_error) + + +def stt_available() -> bool: + return _model is not None + + +async def transcribe(audio_bytes: bytes, mime_type: str = "audio/webm") -> str: + """Transcribe audio bytes to text. Runs the model in a thread executor.""" + if _model is None: + raise RuntimeError("STT model not loaded") + + suffix = ".webm" + if "ogg" in mime_type: + suffix = ".ogg" + elif "wav" in mime_type: + suffix = ".wav" + elif "mp4" in mime_type or "m4a" in mime_type: + suffix = ".mp4" + + def _run() -> str: + with tempfile.NamedTemporaryFile(suffix=suffix, delete=True) as f: + f.write(audio_bytes) + f.flush() + t0 = time.monotonic() + segments, _ = _model.transcribe(f.name, beam_size=5) # type: ignore[union-attr] + text = " ".join(seg.text.strip() for seg in segments).strip() + logger.debug("STT transcription took %.2fs", time.monotonic() - t0) + return text + + loop = asyncio.get_running_loop() + return await loop.run_in_executor(None, _run) diff --git a/src/fabledassistant/services/tts.py b/src/fabledassistant/services/tts.py new file mode 100644 index 0000000..3a20a4a --- /dev/null +++ b/src/fabledassistant/services/tts.py @@ -0,0 +1,94 @@ +"""Text-to-speech service using Kokoro TTS (in-process).""" +import asyncio +import io +import logging +import time +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from kokoro import KPipeline + +logger = logging.getLogger(__name__) + +_pipeline: "KPipeline | None" = None +_pipeline_lock = asyncio.Lock() +_load_error: str | None = None + +# Static list of supported Kokoro voice IDs and display labels +_VOICES: list[dict] = [ + {"id": "af_heart", "label": "Heart (American Female, warm)"}, + {"id": "af_bella", "label": "Bella (American Female, expressive)"}, + {"id": "af_nicole", "label": "Nicole (American Female, intimate)"}, + {"id": "af_sarah", "label": "Sarah (American Female, clear)"}, + {"id": "af_sky", "label": "Sky (American Female, bright)"}, + {"id": "am_adam", "label": "Adam (American Male, neutral)"}, + {"id": "am_michael", "label": "Michael (American Male, deep)"}, + {"id": "bf_emma", "label": "Emma (British Female)"}, + {"id": "bf_isabella", "label": "Isabella (British Female, formal)"}, + {"id": "bm_george", "label": "George (British Male)"}, + {"id": "bm_lewis", "label": "Lewis (British Male, casual)"}, +] + + +async def load_tts_model() -> None: + """Load the Kokoro pipeline. Called once at startup when VOICE_ENABLED=true.""" + global _pipeline, _load_error + from fabledassistant.config import Config + + if not Config.VOICE_ENABLED: + return + + async with _pipeline_lock: + if _pipeline is not None: + return + try: + from kokoro import KPipeline + + logger.info("Loading Kokoro TTS pipeline...") + loop = asyncio.get_running_loop() + _pipeline = await loop.run_in_executor( + None, + lambda: KPipeline(lang_code="a"), # "a" = American English + ) + logger.info("Kokoro TTS pipeline loaded") + except Exception: + _load_error = "Failed to load Kokoro TTS pipeline" + logger.exception(_load_error) + + +def tts_available() -> bool: + return _pipeline is not None + + +def list_voices() -> list[dict]: + return _VOICES + + +async def synthesise(text: str, voice: str = "af_heart", speed: float = 1.0) -> bytes: + """Synthesise text to WAV bytes (24kHz, 16-bit mono). Runs in executor.""" + if _pipeline is None: + raise RuntimeError("TTS pipeline not loaded") + + speed = max(0.7, min(1.3, speed)) + + def _run() -> bytes: + import numpy as np + import soundfile as sf + + t0 = time.monotonic() + audio_chunks: list = [] + for _, _, audio in _pipeline(text, voice=voice, speed=speed): # type: ignore[misc] + if audio is not None: + audio_chunks.append(audio) + + if not audio_chunks: + return b"" + + combined = np.concatenate(audio_chunks) + buf = io.BytesIO() + sf.write(buf, combined, samplerate=24000, format="WAV", subtype="PCM_16") + logger.debug("TTS synthesis took %.2fs for %d chars", time.monotonic() - t0, len(text)) + return buf.getvalue() + + loop = asyncio.get_running_loop() + return await loop.run_in_executor(None, _run)