feat: voice S2S — faster-whisper STT, Kokoro TTS, PTT overlay
Implements full speech-to-speech pipeline (all 4 phases): Backend (Phase 1): - services/stt.py: lazy WhisperModel singleton, run_in_executor transcription - services/tts.py: lazy KPipeline singleton, WAV synthesis at 24kHz/16-bit - routes/voice.py: /api/voice/status, /voices, /transcribe, /synthesise - config.py: VOICE_ENABLED, STT_BACKEND, STT_MODEL, TTS_BACKEND env vars - app.py: load STT/TTS models at startup when VOICE_ENABLED=true - llm.py: voice_mode + voice_speech_style params inject speak-naturally prefix - generation_task.py: voice_mode passed through from chat route - chat.py: "voice" conversation type allowed + excluded from retention cleanup - pyproject.toml + Dockerfile: faster-whisper, kokoro, soundfile dependencies Frontend (Phases 2–4): - composables/useVoiceRecorder.ts: MediaRecorder PTT wrapper - composables/useVoiceAudio.ts: AudioContext WAV playback wrapper - BriefingView.vue: Listen button (TTS read-aloud), auto-TTS mode, mic PTT - VoiceOverlay.vue: global floating PTT button; creates/reuses voice conv; full record→transcribe→stream→TTS flow; Space bar hold-to-talk via App.vue - SettingsView.vue: Voice tab (status badge, speech style, voice/speed) - App.vue: mounts VoiceOverlay; Space keydown/keyup fires voice:ptt-toggle - api/client.ts: getVoiceStatus, getVoiceList, transcribeAudio, synthesiseSpeech Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
<script setup lang="ts">
|
||||
import { ref, computed, onMounted, watch, nextTick } from 'vue'
|
||||
import { useBackgroundRefresh } from '@/composables/useBackgroundRefresh'
|
||||
import { useVoiceRecorder } from '@/composables/useVoiceRecorder'
|
||||
import { useVoiceAudio } from '@/composables/useVoiceAudio'
|
||||
import { useChatStore } from '@/stores/chat'
|
||||
import ChatMessage from '@/components/ChatMessage.vue'
|
||||
import WeatherCard from '@/components/WeatherCard.vue'
|
||||
@@ -15,6 +17,9 @@ import {
|
||||
postRssReaction,
|
||||
deleteRssReaction,
|
||||
getNewsItems,
|
||||
getVoiceStatus,
|
||||
transcribeAudio,
|
||||
synthesiseSpeech,
|
||||
type BriefingConversation,
|
||||
type BriefingMessage,
|
||||
} from '@/api/client'
|
||||
@@ -224,6 +229,87 @@ function convLabel(c: BriefingConversation): string {
|
||||
return c.title || 'Briefing'
|
||||
}
|
||||
|
||||
// ─── Voice ────────────────────────────────────────────────────────────────────
|
||||
const voiceEnabled = ref(false)
|
||||
const listenMode = ref(false)
|
||||
const transcribing = ref(false)
|
||||
const synthesising = ref(false)
|
||||
|
||||
const recorder = useVoiceRecorder()
|
||||
const audio = useVoiceAudio()
|
||||
|
||||
// Check voice availability once on mount
|
||||
async function checkVoice() {
|
||||
try {
|
||||
const status = await getVoiceStatus()
|
||||
voiceEnabled.value = status.enabled && status.stt && status.tts
|
||||
} catch { /* voice feature absent */ }
|
||||
}
|
||||
|
||||
// Read the latest assistant message aloud
|
||||
async function listenToLatest() {
|
||||
const lastAssistant = [...messages.value].reverse().find((m) => m.role === 'assistant')
|
||||
if (!lastAssistant?.content) return
|
||||
await speakText(lastAssistant.content)
|
||||
}
|
||||
|
||||
async function speakText(text: string) {
|
||||
if (!voiceEnabled.value || synthesising.value) return
|
||||
// Strip markdown for cleaner TTS output
|
||||
const plain = text
|
||||
.replace(/```[\s\S]*?```/g, '')
|
||||
.replace(/`[^`]+`/g, (m) => m.slice(1, -1))
|
||||
.replace(/#{1,6}\s+/g, '')
|
||||
.replace(/\*\*([^*]+)\*\*/g, '$1')
|
||||
.replace(/\*([^*]+)\*/g, '$1')
|
||||
.replace(/\[([^\]]+)\]\([^)]+\)/g, '$1')
|
||||
.replace(/^\s*[-*+]\s+/gm, '')
|
||||
.replace(/\n{2,}/g, ' ')
|
||||
.trim()
|
||||
if (!plain) return
|
||||
synthesising.value = true
|
||||
try {
|
||||
const blob = await synthesiseSpeech(plain)
|
||||
await audio.play(blob)
|
||||
} catch {
|
||||
// TTS failure is non-critical
|
||||
} finally {
|
||||
synthesising.value = false
|
||||
}
|
||||
}
|
||||
|
||||
// PTT: hold to record, release to transcribe and send
|
||||
async function startPtt() {
|
||||
if (!voiceEnabled.value || recorder.recording.value) return
|
||||
await recorder.startRecording()
|
||||
}
|
||||
|
||||
async function stopPtt() {
|
||||
if (!recorder.recording.value) return
|
||||
transcribing.value = true
|
||||
try {
|
||||
const blob = await recorder.stopRecording()
|
||||
const { transcript } = await transcribeAudio(blob)
|
||||
if (transcript.trim()) {
|
||||
input.value = transcript.trim()
|
||||
await send()
|
||||
}
|
||||
} catch {
|
||||
// transcription failure — leave input empty
|
||||
} finally {
|
||||
transcribing.value = false
|
||||
}
|
||||
}
|
||||
|
||||
// Auto-TTS when listen mode is on and streaming ends
|
||||
watch(() => chatStore.streaming, async (streaming) => {
|
||||
if (!streaming && listenMode.value && voiceEnabled.value) {
|
||||
// Small delay to let messages update after stream
|
||||
await new Promise((r) => setTimeout(r, 200))
|
||||
await listenToLatest()
|
||||
}
|
||||
})
|
||||
|
||||
// Convert BriefingMessage to Message for ChatMessage component
|
||||
function toMsg(m: BriefingMessage): Message {
|
||||
return {
|
||||
@@ -261,6 +347,7 @@ useBackgroundRefresh(
|
||||
onMounted(async () => {
|
||||
await checkSetup()
|
||||
if (!showWizard.value) await loadAll()
|
||||
checkVoice()
|
||||
})
|
||||
</script>
|
||||
|
||||
@@ -281,6 +368,29 @@ onMounted(async () => {
|
||||
<select v-if="conversations.length" v-model="selectedConvId" class="briefing-conv-select">
|
||||
<option v-for="c in conversations" :key="c.id" :value="c.id">{{ convLabel(c) }}</option>
|
||||
</select>
|
||||
<!-- Listen button: reads latest assistant message; toggles auto-TTS mode -->
|
||||
<button
|
||||
v-if="voiceEnabled"
|
||||
class="btn-voice-header"
|
||||
:class="{ 'btn-voice-active': listenMode, 'btn-voice-busy': synthesising || audio.playing.value }"
|
||||
@click="listenMode ? (listenMode = false) : (listenMode = true, listenToLatest())"
|
||||
:disabled="synthesising"
|
||||
:title="listenMode ? 'Stop auto-read' : 'Listen to briefing'"
|
||||
>
|
||||
<svg v-if="!synthesising && !audio.playing.value" width="15" height="15" viewBox="0 0 24 24" fill="currentColor">
|
||||
<path d="M3 9v6h4l5 5V4L7 9H3zm13.5 3c0-1.77-1.02-3.29-2.5-4.03v8.05c1.48-.73 2.5-2.25 2.5-4.02zM14 3.23v2.06c2.89.86 5 3.54 5 6.71s-2.11 5.85-5 6.71v2.06c4.01-.91 7-4.49 7-8.77s-2.99-7.86-7-8.77z"/>
|
||||
</svg>
|
||||
<svg v-else width="15" height="15" viewBox="0 0 24 24" fill="currentColor">
|
||||
<path d="M12 3v10.55c-.59-.34-1.27-.55-2-.55-2.21 0-4 1.79-4 4s1.79 4 4 4 4-1.79 4-4V7h4V3h-6z"/>
|
||||
</svg>
|
||||
{{ listenMode ? 'Listening' : 'Listen' }}
|
||||
</button>
|
||||
<button
|
||||
v-if="voiceEnabled && (synthesising || audio.playing.value)"
|
||||
class="btn-trigger"
|
||||
@click="audio.stop()"
|
||||
title="Stop playback"
|
||||
>Stop</button>
|
||||
<button
|
||||
class="btn-trigger"
|
||||
@click="triggerNow"
|
||||
@@ -343,10 +453,31 @@ onMounted(async () => {
|
||||
<textarea
|
||||
v-model="input"
|
||||
class="briefing-input"
|
||||
placeholder="Reply to your briefing…"
|
||||
:placeholder="transcribing ? 'Transcribing…' : recorder.recording.value ? 'Recording…' : 'Reply to your briefing…'"
|
||||
rows="1"
|
||||
@keydown="onKeydown"
|
||||
:disabled="transcribing || recorder.recording.value"
|
||||
></textarea>
|
||||
<!-- Mic PTT button (hold to record) -->
|
||||
<button
|
||||
v-if="voiceEnabled && recorder.isSupported"
|
||||
class="btn-mic"
|
||||
:class="{ 'btn-mic-active': recorder.recording.value, 'btn-mic-busy': transcribing }"
|
||||
@mousedown.prevent="startPtt"
|
||||
@mouseup.prevent="stopPtt"
|
||||
@touchstart.prevent="startPtt"
|
||||
@touchend.prevent="stopPtt"
|
||||
:disabled="transcribing || chatStore.streaming || sending"
|
||||
:title="recorder.recording.value ? 'Release to send' : 'Hold to speak'"
|
||||
aria-label="Push to talk"
|
||||
>
|
||||
<svg v-if="!transcribing" width="16" height="16" viewBox="0 0 24 24" fill="currentColor">
|
||||
<path d="M12 14c1.66 0 3-1.34 3-3V5c0-1.66-1.34-3-3-3S9 3.34 9 5v6c0 1.66 1.34 3 3 3zm-1-9c0-.55.45-1 1-1s1 .45 1 1v6c0 .55-.45 1-1 1s-1-.45-1-1V5zm6 6c0 2.76-2.24 5-5 5s-5-2.24-5-5H5c0 3.53 2.61 6.43 6 6.92V21h2v-3.08c3.39-.49 6-3.39 6-6.92h-2z"/>
|
||||
</svg>
|
||||
<svg v-else width="16" height="16" viewBox="0 0 24 24" fill="currentColor">
|
||||
<path d="M12 4V1L8 5l4 4V6c3.31 0 6 2.69 6 6 0 1.01-.25 1.97-.7 2.8l1.46 1.46C19.54 15.03 20 13.57 20 12c0-4.42-3.58-8-8-8zm0 14c-3.31 0-6-2.69-6-6 0-1.01.25-1.97.7-2.8L5.24 7.74C4.46 8.97 4 10.43 4 12c0 4.42 3.58 8 8 8v3l4-4-4-4v3z"/>
|
||||
</svg>
|
||||
</button>
|
||||
<button
|
||||
class="btn-send"
|
||||
@click="send"
|
||||
@@ -706,6 +837,74 @@ a.news-title:hover { text-decoration: underline; color: var(--color-primary); }
|
||||
background: color-mix(in srgb, var(--color-primary) 12%, transparent);
|
||||
}
|
||||
|
||||
/* ─── Voice buttons ──────────────────────────────────────────────────────── */
|
||||
|
||||
.btn-voice-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.35rem;
|
||||
padding: 0.35rem 0.75rem;
|
||||
border: 1px solid var(--color-border);
|
||||
border-radius: 6px;
|
||||
background: var(--color-bg-card);
|
||||
color: var(--color-text-muted);
|
||||
font-size: 0.8rem;
|
||||
cursor: pointer;
|
||||
white-space: nowrap;
|
||||
transition: all 0.15s;
|
||||
font-family: inherit;
|
||||
}
|
||||
.btn-voice-header:hover:not(:disabled) {
|
||||
border-color: var(--color-primary);
|
||||
color: var(--color-primary);
|
||||
}
|
||||
.btn-voice-header:disabled { opacity: 0.5; cursor: not-allowed; }
|
||||
.btn-voice-active {
|
||||
border-color: var(--color-primary) !important;
|
||||
color: var(--color-primary) !important;
|
||||
background: color-mix(in srgb, var(--color-primary) 10%, transparent) !important;
|
||||
}
|
||||
.btn-voice-busy {
|
||||
border-color: var(--color-primary);
|
||||
color: var(--color-primary);
|
||||
animation: pulse-border 1.2s ease-in-out infinite;
|
||||
}
|
||||
@keyframes pulse-border {
|
||||
0%, 100% { opacity: 1; }
|
||||
50% { opacity: 0.55; }
|
||||
}
|
||||
|
||||
.btn-mic {
|
||||
padding: 0.55rem 0.65rem;
|
||||
background: var(--color-bg-secondary);
|
||||
border: 1px solid var(--color-border);
|
||||
border-radius: 10px;
|
||||
color: var(--color-text-muted);
|
||||
cursor: pointer;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
flex-shrink: 0;
|
||||
transition: all 0.15s;
|
||||
touch-action: none;
|
||||
user-select: none;
|
||||
}
|
||||
.btn-mic:hover:not(:disabled) {
|
||||
border-color: var(--color-primary);
|
||||
color: var(--color-primary);
|
||||
}
|
||||
.btn-mic:disabled { opacity: 0.4; cursor: not-allowed; }
|
||||
.btn-mic-active {
|
||||
background: color-mix(in srgb, #ef4444 15%, transparent) !important;
|
||||
border-color: #ef4444 !important;
|
||||
color: #ef4444 !important;
|
||||
animation: pulse-border 0.8s ease-in-out infinite;
|
||||
}
|
||||
.btn-mic-busy {
|
||||
border-color: var(--color-primary);
|
||||
color: var(--color-primary);
|
||||
}
|
||||
|
||||
/* ─── Responsive ─────────────────────────────────────────────────────────── */
|
||||
|
||||
@media (max-width: 900px) {
|
||||
|
||||
Reference in New Issue
Block a user