feat: voice S2S — faster-whisper STT, Kokoro TTS, PTT overlay

Implements full speech-to-speech pipeline (all 4 phases):

Backend (Phase 1):
- services/stt.py: lazy WhisperModel singleton, run_in_executor transcription
- services/tts.py: lazy KPipeline singleton, WAV synthesis at 24kHz/16-bit
- routes/voice.py: /api/voice/status, /voices, /transcribe, /synthesise
- config.py: VOICE_ENABLED, STT_BACKEND, STT_MODEL, TTS_BACKEND env vars
- app.py: load STT/TTS models at startup when VOICE_ENABLED=true
- llm.py: voice_mode + voice_speech_style params inject speak-naturally prefix
- generation_task.py: voice_mode passed through from chat route
- chat.py: "voice" conversation type allowed + excluded from retention cleanup
- pyproject.toml + Dockerfile: faster-whisper, kokoro, soundfile dependencies

Frontend (Phases 2–4):
- composables/useVoiceRecorder.ts: MediaRecorder PTT wrapper
- composables/useVoiceAudio.ts: AudioContext WAV playback wrapper
- BriefingView.vue: Listen button (TTS read-aloud), auto-TTS mode, mic PTT
- VoiceOverlay.vue: global floating PTT button; creates/reuses voice conv;
  full record→transcribe→stream→TTS flow; Space bar hold-to-talk via App.vue
- SettingsView.vue: Voice tab (status badge, speech style, voice/speed)
- App.vue: mounts VoiceOverlay; Space keydown/keyup fires voice:ptt-toggle
- api/client.ts: getVoiceStatus, getVoiceList, transcribeAudio, synthesiseSpeech

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-29 20:03:38 -04:00
parent 3581cc1582
commit 6f84d90dff
18 changed files with 1524 additions and 6 deletions
+200 -1
View File
@@ -1,6 +1,8 @@
<script setup lang="ts">
import { ref, computed, onMounted, watch, nextTick } from 'vue'
import { useBackgroundRefresh } from '@/composables/useBackgroundRefresh'
import { useVoiceRecorder } from '@/composables/useVoiceRecorder'
import { useVoiceAudio } from '@/composables/useVoiceAudio'
import { useChatStore } from '@/stores/chat'
import ChatMessage from '@/components/ChatMessage.vue'
import WeatherCard from '@/components/WeatherCard.vue'
@@ -15,6 +17,9 @@ import {
postRssReaction,
deleteRssReaction,
getNewsItems,
getVoiceStatus,
transcribeAudio,
synthesiseSpeech,
type BriefingConversation,
type BriefingMessage,
} from '@/api/client'
@@ -224,6 +229,87 @@ function convLabel(c: BriefingConversation): string {
return c.title || 'Briefing'
}
// ─── Voice ────────────────────────────────────────────────────────────────────
const voiceEnabled = ref(false)
const listenMode = ref(false)
const transcribing = ref(false)
const synthesising = ref(false)
const recorder = useVoiceRecorder()
const audio = useVoiceAudio()
// Check voice availability once on mount
async function checkVoice() {
try {
const status = await getVoiceStatus()
voiceEnabled.value = status.enabled && status.stt && status.tts
} catch { /* voice feature absent */ }
}
// Read the latest assistant message aloud
async function listenToLatest() {
const lastAssistant = [...messages.value].reverse().find((m) => m.role === 'assistant')
if (!lastAssistant?.content) return
await speakText(lastAssistant.content)
}
async function speakText(text: string) {
if (!voiceEnabled.value || synthesising.value) return
// Strip markdown for cleaner TTS output
const plain = text
.replace(/```[\s\S]*?```/g, '')
.replace(/`[^`]+`/g, (m) => m.slice(1, -1))
.replace(/#{1,6}\s+/g, '')
.replace(/\*\*([^*]+)\*\*/g, '$1')
.replace(/\*([^*]+)\*/g, '$1')
.replace(/\[([^\]]+)\]\([^)]+\)/g, '$1')
.replace(/^\s*[-*+]\s+/gm, '')
.replace(/\n{2,}/g, ' ')
.trim()
if (!plain) return
synthesising.value = true
try {
const blob = await synthesiseSpeech(plain)
await audio.play(blob)
} catch {
// TTS failure is non-critical
} finally {
synthesising.value = false
}
}
// PTT: hold to record, release to transcribe and send
async function startPtt() {
if (!voiceEnabled.value || recorder.recording.value) return
await recorder.startRecording()
}
async function stopPtt() {
if (!recorder.recording.value) return
transcribing.value = true
try {
const blob = await recorder.stopRecording()
const { transcript } = await transcribeAudio(blob)
if (transcript.trim()) {
input.value = transcript.trim()
await send()
}
} catch {
// transcription failure — leave input empty
} finally {
transcribing.value = false
}
}
// Auto-TTS when listen mode is on and streaming ends
watch(() => chatStore.streaming, async (streaming) => {
if (!streaming && listenMode.value && voiceEnabled.value) {
// Small delay to let messages update after stream
await new Promise((r) => setTimeout(r, 200))
await listenToLatest()
}
})
// Convert BriefingMessage to Message for ChatMessage component
function toMsg(m: BriefingMessage): Message {
return {
@@ -261,6 +347,7 @@ useBackgroundRefresh(
onMounted(async () => {
await checkSetup()
if (!showWizard.value) await loadAll()
checkVoice()
})
</script>
@@ -281,6 +368,29 @@ onMounted(async () => {
<select v-if="conversations.length" v-model="selectedConvId" class="briefing-conv-select">
<option v-for="c in conversations" :key="c.id" :value="c.id">{{ convLabel(c) }}</option>
</select>
<!-- Listen button: reads latest assistant message; toggles auto-TTS mode -->
<button
v-if="voiceEnabled"
class="btn-voice-header"
:class="{ 'btn-voice-active': listenMode, 'btn-voice-busy': synthesising || audio.playing.value }"
@click="listenMode ? (listenMode = false) : (listenMode = true, listenToLatest())"
:disabled="synthesising"
:title="listenMode ? 'Stop auto-read' : 'Listen to briefing'"
>
<svg v-if="!synthesising && !audio.playing.value" width="15" height="15" viewBox="0 0 24 24" fill="currentColor">
<path d="M3 9v6h4l5 5V4L7 9H3zm13.5 3c0-1.77-1.02-3.29-2.5-4.03v8.05c1.48-.73 2.5-2.25 2.5-4.02zM14 3.23v2.06c2.89.86 5 3.54 5 6.71s-2.11 5.85-5 6.71v2.06c4.01-.91 7-4.49 7-8.77s-2.99-7.86-7-8.77z"/>
</svg>
<svg v-else width="15" height="15" viewBox="0 0 24 24" fill="currentColor">
<path d="M12 3v10.55c-.59-.34-1.27-.55-2-.55-2.21 0-4 1.79-4 4s1.79 4 4 4 4-1.79 4-4V7h4V3h-6z"/>
</svg>
{{ listenMode ? 'Listening' : 'Listen' }}
</button>
<button
v-if="voiceEnabled && (synthesising || audio.playing.value)"
class="btn-trigger"
@click="audio.stop()"
title="Stop playback"
>Stop</button>
<button
class="btn-trigger"
@click="triggerNow"
@@ -343,10 +453,31 @@ onMounted(async () => {
<textarea
v-model="input"
class="briefing-input"
placeholder="Reply to your briefing…"
:placeholder="transcribing ? 'Transcribing…' : recorder.recording.value ? 'Recording…' : 'Reply to your briefing…'"
rows="1"
@keydown="onKeydown"
:disabled="transcribing || recorder.recording.value"
></textarea>
<!-- Mic PTT button (hold to record) -->
<button
v-if="voiceEnabled && recorder.isSupported"
class="btn-mic"
:class="{ 'btn-mic-active': recorder.recording.value, 'btn-mic-busy': transcribing }"
@mousedown.prevent="startPtt"
@mouseup.prevent="stopPtt"
@touchstart.prevent="startPtt"
@touchend.prevent="stopPtt"
:disabled="transcribing || chatStore.streaming || sending"
:title="recorder.recording.value ? 'Release to send' : 'Hold to speak'"
aria-label="Push to talk"
>
<svg v-if="!transcribing" width="16" height="16" viewBox="0 0 24 24" fill="currentColor">
<path d="M12 14c1.66 0 3-1.34 3-3V5c0-1.66-1.34-3-3-3S9 3.34 9 5v6c0 1.66 1.34 3 3 3zm-1-9c0-.55.45-1 1-1s1 .45 1 1v6c0 .55-.45 1-1 1s-1-.45-1-1V5zm6 6c0 2.76-2.24 5-5 5s-5-2.24-5-5H5c0 3.53 2.61 6.43 6 6.92V21h2v-3.08c3.39-.49 6-3.39 6-6.92h-2z"/>
</svg>
<svg v-else width="16" height="16" viewBox="0 0 24 24" fill="currentColor">
<path d="M12 4V1L8 5l4 4V6c3.31 0 6 2.69 6 6 0 1.01-.25 1.97-.7 2.8l1.46 1.46C19.54 15.03 20 13.57 20 12c0-4.42-3.58-8-8-8zm0 14c-3.31 0-6-2.69-6-6 0-1.01.25-1.97.7-2.8L5.24 7.74C4.46 8.97 4 10.43 4 12c0 4.42 3.58 8 8 8v3l4-4-4-4v3z"/>
</svg>
</button>
<button
class="btn-send"
@click="send"
@@ -706,6 +837,74 @@ a.news-title:hover { text-decoration: underline; color: var(--color-primary); }
background: color-mix(in srgb, var(--color-primary) 12%, transparent);
}
/* ─── Voice buttons ──────────────────────────────────────────────────────── */
.btn-voice-header {
display: flex;
align-items: center;
gap: 0.35rem;
padding: 0.35rem 0.75rem;
border: 1px solid var(--color-border);
border-radius: 6px;
background: var(--color-bg-card);
color: var(--color-text-muted);
font-size: 0.8rem;
cursor: pointer;
white-space: nowrap;
transition: all 0.15s;
font-family: inherit;
}
.btn-voice-header:hover:not(:disabled) {
border-color: var(--color-primary);
color: var(--color-primary);
}
.btn-voice-header:disabled { opacity: 0.5; cursor: not-allowed; }
.btn-voice-active {
border-color: var(--color-primary) !important;
color: var(--color-primary) !important;
background: color-mix(in srgb, var(--color-primary) 10%, transparent) !important;
}
.btn-voice-busy {
border-color: var(--color-primary);
color: var(--color-primary);
animation: pulse-border 1.2s ease-in-out infinite;
}
@keyframes pulse-border {
0%, 100% { opacity: 1; }
50% { opacity: 0.55; }
}
.btn-mic {
padding: 0.55rem 0.65rem;
background: var(--color-bg-secondary);
border: 1px solid var(--color-border);
border-radius: 10px;
color: var(--color-text-muted);
cursor: pointer;
display: flex;
align-items: center;
justify-content: center;
flex-shrink: 0;
transition: all 0.15s;
touch-action: none;
user-select: none;
}
.btn-mic:hover:not(:disabled) {
border-color: var(--color-primary);
color: var(--color-primary);
}
.btn-mic:disabled { opacity: 0.4; cursor: not-allowed; }
.btn-mic-active {
background: color-mix(in srgb, #ef4444 15%, transparent) !important;
border-color: #ef4444 !important;
color: #ef4444 !important;
animation: pulse-border 0.8s ease-in-out infinite;
}
.btn-mic-busy {
border-color: var(--color-primary);
color: var(--color-primary);
}
/* ─── Responsive ─────────────────────────────────────────────────────────── */
@media (max-width: 900px) {
+207 -3
View File
@@ -3,7 +3,7 @@ import { ref, watch, onMounted } from "vue";
import { useSettingsStore } from "@/stores/settings";
import { useAuthStore } from "@/stores/auth";
import { useToastStore } from "@/stores/toast";
import { apiGet, apiPost, apiPut, apiDelete, listGroups, createGroup, deleteGroup, listGroupMembers, addGroupMember, removeGroupMember, searchUsers, getBriefingConfig, saveBriefingConfig, getBriefingFeeds, createBriefingFeed, deleteBriefingFeed, refreshBriefingFeeds, geocodeAddress, getFableMcpInfo, listApiKeys, createApiKey as apiCreateApiKey, revokeApiKey as apiRevokeApiKey, type ApiKeyEntry, type GroupEntry, type GroupMember, type UserSearchResult, type BriefingConfig, type BriefingFeed } from "@/api/client";
import { apiGet, apiPost, apiPut, apiDelete, listGroups, createGroup, deleteGroup, listGroupMembers, addGroupMember, removeGroupMember, searchUsers, getBriefingConfig, saveBriefingConfig, getBriefingFeeds, createBriefingFeed, deleteBriefingFeed, refreshBriefingFeeds, geocodeAddress, getFableMcpInfo, listApiKeys, createApiKey as apiCreateApiKey, revokeApiKey as apiRevokeApiKey, getVoiceStatus, getVoiceList, type ApiKeyEntry, type GroupEntry, type GroupMember, type UserSearchResult, type BriefingConfig, type BriefingFeed, type VoiceStatusResult, type VoiceEntry } from "@/api/client";
import { usePushStore } from "@/stores/push";
import type { User } from "@/types/auth";
import PaginationBar from "@/components/PaginationBar.vue";
@@ -40,7 +40,7 @@ const appVersion = ref('dev');
const restoreFileInput = ref<HTMLInputElement | null>(null);
// Migrate stored "admin" → "config"; unknown tabs fall back to "general"
const VALID_TABS = new Set(["general", "account", "notifications", "integrations", "data", "briefing", "apikeys", "config", "users", "logs", "groups"]);
const VALID_TABS = new Set(["general", "account", "notifications", "integrations", "data", "briefing", "voice", "apikeys", "config", "users", "logs", "groups"]);
const _stored = localStorage.getItem("settings_tab") ?? "general";
const activeTab = ref(VALID_TABS.has(_stored) ? (_stored === "admin" ? "config" : _stored) : "general");
@@ -51,6 +51,7 @@ function _loadTabContent(tab: string) {
else if (tab === "groups") loadGroupsPanel();
}
if (tab === "briefing") loadBriefingTab();
if (tab === "voice") loadVoiceTab();
if (tab === "apikeys") { fetchApiKeys(); loadMcpInfo(); }
}
@@ -452,6 +453,49 @@ const searchResults = ref<{ url: string; title: string; snippet: string }[]>([])
const searchLoading = ref(false);
const searchError = ref("");
// Voice settings
const voiceStatus = ref<VoiceStatusResult | null>(null);
const voiceStatusLoading = ref(false);
const availableVoices = ref<VoiceEntry[]>([]);
const voiceTtsVoice = ref("af_heart");
const voiceTtsSpeed = ref(1.0);
const voiceSpeechStyle = ref("conversational");
const savingVoice = ref(false);
const voiceSaved = ref(false);
const voiceTabLoaded = ref(false);
async function loadVoiceTab() {
if (voiceTabLoaded.value) return;
voiceTabLoaded.value = true;
voiceStatusLoading.value = true;
try {
voiceStatus.value = await getVoiceStatus();
if (voiceStatus.value.tts) {
availableVoices.value = await getVoiceList();
}
} catch {
// Voice feature may be disabled
} finally {
voiceStatusLoading.value = false;
}
}
async function saveVoiceSettings() {
savingVoice.value = true;
voiceSaved.value = false;
try {
await apiPut("/api/settings", {
voice_tts_voice: voiceTtsVoice.value,
voice_tts_speed: String(voiceTtsSpeed.value),
voice_speech_style: voiceSpeechStyle.value,
});
voiceSaved.value = true;
setTimeout(() => { voiceSaved.value = false; }, 2000);
} finally {
savingVoice.value = false;
}
}
onMounted(async () => {
try {
const v = await apiGet<{ version: string }>('/api/version')
@@ -485,6 +529,11 @@ onMounted(async () => {
notifySecurityAlerts.value = allSettings.notify_security_alerts !== "false";
}
// Load voice settings
if (allSettings.voice_tts_voice) voiceTtsVoice.value = allSettings.voice_tts_voice;
if (allSettings.voice_tts_speed) voiceTtsSpeed.value = parseFloat(allSettings.voice_tts_speed);
if (allSettings.voice_speech_style) voiceSpeechStyle.value = allSettings.voice_speech_style;
// Load CalDAV settings
try {
const caldavConfig = await apiGet<Record<string, string>>("/api/settings/caldav");
@@ -1134,7 +1183,7 @@ function formatUserDate(iso: string): string {
<div class="sidebar-group">
<div class="sidebar-group-label">User</div>
<button
v-for="tab in ['general', 'account', 'notifications', 'integrations', 'data', 'briefing', 'apikeys']"
v-for="tab in ['general', 'account', 'notifications', 'integrations', 'data', 'briefing', 'voice', 'apikeys']"
:key="tab"
:class="['sidebar-item', { active: activeTab === tab }]"
@click="activeTab = tab"
@@ -1810,6 +1859,105 @@ function formatUserDate(iso: string): string {
</div>
<!-- ── Voice ── -->
<div v-show="activeTab === 'voice'" class="settings-grid">
<section class="settings-section full-width">
<h2>Voice</h2>
<p class="section-desc">
Configure text-to-speech and speech-to-text for voice conversations.
Requires <code>VOICE_ENABLED=true</code> on the server.
</p>
<!-- Status indicator -->
<div v-if="voiceStatusLoading" class="field-hint">Loading voice status…</div>
<div v-else-if="voiceStatus === null" class="field-hint">Voice status unavailable.</div>
<div v-else class="voice-status-row">
<span :class="['status-badge', voiceStatus.enabled ? 'status-on' : 'status-off']">
{{ voiceStatus.enabled ? 'Enabled' : 'Disabled' }}
</span>
<span v-if="voiceStatus.enabled" class="status-badge" :class="voiceStatus.stt ? 'status-on' : 'status-off'">
STT {{ voiceStatus.stt ? 'ready' : 'loading' }}
</span>
<span v-if="voiceStatus.enabled" class="status-badge" :class="voiceStatus.tts ? 'status-on' : 'status-off'">
TTS {{ voiceStatus.tts ? 'ready' : 'loading' }}
</span>
<span v-if="voiceStatus.stt_model" class="field-hint" style="margin-left:0.5rem;">
Model: {{ voiceStatus.stt_model }}
</span>
</div>
<div v-if="!voiceStatus?.enabled" class="field-hint" style="margin-top:0.75rem;">
Set <code>VOICE_ENABLED=true</code> in your server environment to activate voice features.
Optionally set <code>STT_MODEL</code> (tiny.en / base.en / small.en / medium.en) and
install <code>pip install fabledassistant[voice]</code>.
</div>
</section>
<section v-if="voiceStatus?.enabled" class="settings-section full-width">
<h2>Speech Style</h2>
<p class="section-desc">Controls how the assistant phrases spoken responses.</p>
<div class="field-group">
<label class="field-label">Style</label>
<div class="radio-group">
<label v-for="opt in [
{ value: 'conversational', label: 'Conversational', hint: 'Warm and natural, like speaking to a friend' },
{ value: 'concise', label: 'Concise', hint: 'Short and to the point' },
{ value: 'detailed', label: 'Detailed', hint: 'Thorough explanations spoken aloud' },
]" :key="opt.value" class="radio-option">
<input type="radio" v-model="voiceSpeechStyle" :value="opt.value" />
<span>
<strong>{{ opt.label }}</strong>
<span class="field-hint">{{ opt.hint }}</span>
</span>
</label>
</div>
</div>
</section>
<section v-if="voiceStatus?.enabled" class="settings-section full-width">
<h2>Voice &amp; Speed</h2>
<p class="section-desc">Choose the TTS voice and playback speed.</p>
<div class="field-group">
<label class="field-label" for="voice-select">Voice</label>
<select id="voice-select" class="input" v-model="voiceTtsVoice" :disabled="availableVoices.length === 0">
<option v-if="availableVoices.length === 0" value="af_heart">af_heart (default)</option>
<option v-for="v in availableVoices" :key="v.id" :value="v.id">{{ v.label }}</option>
</select>
<p class="field-hint">Voice character and accent. Applies to all voice conversations.</p>
</div>
<div class="field-group">
<label class="field-label" for="voice-speed">
Speed: {{ voiceTtsSpeed.toFixed(1) }}×
</label>
<input
id="voice-speed"
type="range"
min="0.7"
max="1.3"
step="0.05"
v-model.number="voiceTtsSpeed"
class="range-input"
/>
<div class="range-labels">
<span>0.7× (slow)</span>
<span>1.0× (normal)</span>
<span>1.3× (fast)</span>
</div>
</div>
<div class="form-actions">
<button class="btn-save" @click="saveVoiceSettings" :disabled="savingVoice">
{{ voiceSaved ? 'Saved ' : savingVoice ? 'Saving' : 'Save Voice Settings' }}
</button>
</div>
</section>
</div>
<!-- ── API Keys ── -->
<div v-show="activeTab === 'apikeys'" class="settings-grid">
<section class="settings-section full-width">
@@ -3528,4 +3676,60 @@ FABLE_API_KEY=&lt;your-api-key&gt;</pre>
word-break: break-all;
overflow-x: auto;
}
/* Voice tab */
.voice-status-row {
display: flex;
align-items: center;
gap: 0.5rem;
flex-wrap: wrap;
margin-top: 0.5rem;
}
.status-badge {
font-size: 0.75rem;
font-weight: 600;
padding: 0.15rem 0.55rem;
border-radius: 999px;
}
.status-on {
background: color-mix(in srgb, var(--color-success, #22c55e) 15%, transparent);
color: var(--color-success, #22c55e);
border: 1px solid color-mix(in srgb, var(--color-success, #22c55e) 40%, transparent);
}
.status-off {
background: color-mix(in srgb, var(--color-text-muted) 10%, transparent);
color: var(--color-text-muted);
border: 1px solid var(--color-border);
}
.radio-group {
display: flex;
flex-direction: column;
gap: 0.65rem;
margin-top: 0.35rem;
}
.radio-option {
display: flex;
align-items: flex-start;
gap: 0.55rem;
cursor: pointer;
}
.radio-option input[type="radio"] { margin-top: 0.2rem; flex-shrink: 0; }
.radio-option span { display: flex; flex-direction: column; gap: 0.15rem; }
.radio-option strong { font-size: 0.875rem; }
.range-input {
width: 100%;
max-width: 360px;
accent-color: var(--color-primary);
margin: 0.35rem 0 0.2rem;
}
.range-labels {
display: flex;
justify-content: space-between;
max-width: 360px;
font-size: 0.75rem;
color: var(--color-text-muted);
}
.form-actions {
margin-top: 1rem;
}
</style>