feat: voice S2S — faster-whisper STT, Kokoro TTS, PTT overlay

Implements full speech-to-speech pipeline (all 4 phases):

Backend (Phase 1):
- services/stt.py: lazy WhisperModel singleton, run_in_executor transcription
- services/tts.py: lazy KPipeline singleton, WAV synthesis at 24kHz/16-bit
- routes/voice.py: /api/voice/status, /voices, /transcribe, /synthesise
- config.py: VOICE_ENABLED, STT_BACKEND, STT_MODEL, TTS_BACKEND env vars
- app.py: load STT/TTS models at startup when VOICE_ENABLED=true
- llm.py: voice_mode + voice_speech_style params inject speak-naturally prefix
- generation_task.py: voice_mode passed through from chat route
- chat.py: "voice" conversation type allowed + excluded from retention cleanup
- pyproject.toml + Dockerfile: faster-whisper, kokoro, soundfile dependencies

Frontend (Phases 2–4):
- composables/useVoiceRecorder.ts: MediaRecorder PTT wrapper
- composables/useVoiceAudio.ts: AudioContext WAV playback wrapper
- BriefingView.vue: Listen button (TTS read-aloud), auto-TTS mode, mic PTT
- VoiceOverlay.vue: global floating PTT button; creates/reuses voice conv;
  full record→transcribe→stream→TTS flow; Space bar hold-to-talk via App.vue
- SettingsView.vue: Voice tab (status badge, speech style, voice/speed)
- App.vue: mounts VoiceOverlay; Space keydown/keyup fires voice:ptt-toggle
- api/client.ts: getVoiceStatus, getVoiceList, transcribeAudio, synthesiseSpeech

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-29 20:03:38 -04:00
parent 3581cc1582
commit 6f84d90dff
18 changed files with 1524 additions and 6 deletions
+536
View File
@@ -0,0 +1,536 @@
<script setup lang="ts">
/**
* VoiceOverlay — global floating push-to-talk button.
*
* Full flow: record → transcribe → send to voice conv → stream → TTS → play.
* Manages its own "voice" conversation; does NOT touch the chat store so it
* never disrupts an open chat session.
*
* Space bar toggles PTT when no input field is focused (wired from App.vue via
* the "voice:ptt-toggle" custom event).
*/
import { ref, onMounted, onUnmounted, computed } from 'vue'
import { useVoiceRecorder } from '@/composables/useVoiceRecorder'
import { useVoiceAudio } from '@/composables/useVoiceAudio'
import { apiPost, apiSSEStream, getVoiceStatus, transcribeAudio, synthesiseSpeech } from '@/api/client'
// ─── Voice service availability ──────────────────────────────────────────────
const voiceEnabled = ref(false)
async function checkVoice() {
try {
const s = await getVoiceStatus()
voiceEnabled.value = s.enabled && s.stt && s.tts
} catch { /* feature absent */ }
}
// ─── Conversation management ─────────────────────────────────────────────────
const STORAGE_KEY = 'voice_overlay_conv_id'
const convId = ref<number | null>(Number(localStorage.getItem(STORAGE_KEY)) || null)
interface VoiceMessage { role: 'user' | 'assistant'; content: string }
const messages = ref<VoiceMessage[]>([])
async function ensureConversation(): Promise<number> {
if (convId.value) {
// Verify it still exists
try {
await fetch(`/api/chat/conversations/${convId.value}`)
.then((r) => { if (!r.ok) throw new Error('gone') })
return convId.value
} catch {
convId.value = null
localStorage.removeItem(STORAGE_KEY)
}
}
const conv = await apiPost<{ id: number }>('/api/chat/conversations', {
title: 'Voice',
conversation_type: 'voice',
})
convId.value = conv.id
localStorage.setItem(STORAGE_KEY, String(conv.id))
return conv.id
}
// ─── State machine ────────────────────────────────────────────────────────────
type Phase = 'idle' | 'recording' | 'transcribing' | 'generating' | 'speaking' | 'error'
const phase = ref<Phase>('idle')
const errorMsg = ref('')
const streamContent = ref('')
const open = ref(false)
const isBusy = computed(() => phase.value !== 'idle' && phase.value !== 'error')
// ─── Composables ─────────────────────────────────────────────────────────────
const recorder = useVoiceRecorder()
const audio = useVoiceAudio()
// ─── Core PTT flow ────────────────────────────────────────────────────────────
async function startPtt() {
if (!voiceEnabled.value || isBusy.value) return
errorMsg.value = ''
open.value = true
await recorder.startRecording()
if (recorder.error.value) {
phase.value = 'error'
errorMsg.value = recorder.error.value
return
}
phase.value = 'recording'
}
async function stopPtt() {
if (phase.value !== 'recording') return
phase.value = 'transcribing'
let blob: Blob
try {
blob = await recorder.stopRecording()
} catch {
phase.value = 'error'
errorMsg.value = 'Recording failed'
return
}
let transcript: string
try {
const result = await transcribeAudio(blob)
transcript = result.transcript.trim()
} catch {
phase.value = 'error'
errorMsg.value = 'Transcription failed'
return
}
if (!transcript) {
phase.value = 'idle'
return
}
messages.value.push({ role: 'user', content: transcript })
scrollToBottom()
// Send to voice conversation
phase.value = 'generating'
streamContent.value = ''
let cid: number
try {
cid = await ensureConversation()
} catch {
phase.value = 'error'
errorMsg.value = 'Could not create voice conversation'
return
}
let assistantMessageId: number
try {
const resp = await apiPost<{ assistant_message_id: number }>(
`/api/chat/conversations/${cid}/messages`,
{
content: transcript,
user_timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
}
)
assistantMessageId = resp.assistant_message_id
} catch {
phase.value = 'error'
errorMsg.value = 'Failed to send message'
return
}
// Stream the response
await new Promise<void>((resolve) => {
const handle = apiSSEStream(
`/api/chat/conversations/${cid}/generation/stream`,
(event) => {
switch (event.event) {
case 'chunk':
streamContent.value += event.data.chunk as string
break
case 'done':
handle.close()
resolve()
break
case 'error':
handle.close()
resolve()
break
}
}
)
// Safety timeout: 3 minutes
setTimeout(() => { handle.close(); resolve() }, 180_000)
})
const responseText = streamContent.value.trim()
if (responseText) {
messages.value.push({ role: 'assistant', content: responseText })
streamContent.value = ''
scrollToBottom()
}
// Synthesise and play
if (responseText) {
phase.value = 'speaking'
try {
const wavBlob = await synthesiseSpeech(responseText)
await audio.play(wavBlob)
} catch {
// TTS failure is non-critical; show response text
}
}
phase.value = 'idle'
assistantMessageId // consumed; suppress lint
}
function cancelAll() {
recorder.stopRecording().catch(() => {})
audio.stop()
phase.value = 'idle'
streamContent.value = ''
errorMsg.value = ''
}
// ─── Space bar PTT (event from App.vue) ──────────────────────────────────────
function onPttToggle() {
if (!voiceEnabled.value) return
if (phase.value === 'recording') {
stopPtt()
} else if (phase.value === 'idle' || phase.value === 'error') {
startPtt()
}
}
// ─── Scroll ───────────────────────────────────────────────────────────────────
const transcriptEl = ref<HTMLElement | null>(null)
function scrollToBottom() {
setTimeout(() => {
if (transcriptEl.value) {
transcriptEl.value.scrollTop = transcriptEl.value.scrollHeight
}
}, 50)
}
// ─── Lifecycle ────────────────────────────────────────────────────────────────
onMounted(() => {
checkVoice()
document.addEventListener('voice:ptt-toggle', onPttToggle)
})
onUnmounted(() => {
document.removeEventListener('voice:ptt-toggle', onPttToggle)
cancelAll()
})
</script>
<template>
<Teleport to="body">
<div v-if="voiceEnabled" class="voice-overlay">
<!-- Expanded transcript panel -->
<Transition name="panel-slide">
<div v-if="open && messages.length > 0" class="voice-panel">
<div class="voice-panel-header">
<span class="voice-panel-title">Voice</span>
<button class="voice-panel-close" @click="open = false" aria-label="Close">×</button>
</div>
<div class="voice-transcript" ref="transcriptEl">
<div
v-for="(msg, i) in messages.slice(-10)"
:key="i"
:class="['voice-msg', `voice-msg--${msg.role}`]"
>{{ msg.content }}</div>
<!-- Live streaming text -->
<div v-if="phase === 'generating' && streamContent" class="voice-msg voice-msg--assistant voice-msg--streaming">
{{ streamContent }}<span class="voice-cursor"></span>
</div>
</div>
</div>
</Transition>
<!-- PTT button -->
<div class="voice-btn-wrap">
<!-- Status label -->
<div v-if="phase !== 'idle'" class="voice-status-label">
<span v-if="phase === 'recording'">Recording</span>
<span v-else-if="phase === 'transcribing'">Transcribing</span>
<span v-else-if="phase === 'generating'">Thinking</span>
<span v-else-if="phase === 'speaking'">Speaking</span>
<span v-else-if="phase === 'error'" class="voice-error-label">{{ errorMsg || 'Error' }}</span>
</div>
<div v-else-if="!open || !messages.length" class="voice-status-label voice-hint">
Hold <kbd>Space</kbd> or tap
</div>
<!-- Cancel button (shown while busy or speaking) -->
<button
v-if="isBusy || audio.playing.value"
class="voice-cancel"
@click="cancelAll"
aria-label="Cancel"
title="Cancel"
>
<svg width="13" height="13" viewBox="0 0 24 24" fill="currentColor">
<path d="M19 6.41L17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/>
</svg>
</button>
<!-- Main PTT button -->
<button
class="voice-ptt-btn"
:class="{
'voice-ptt--recording': phase === 'recording',
'voice-ptt--busy': phase === 'generating' || phase === 'transcribing',
'voice-ptt--speaking': phase === 'speaking' || audio.playing.value,
'voice-ptt--error': phase === 'error',
}"
@mousedown.prevent="startPtt"
@mouseup.prevent="stopPtt"
@touchstart.prevent="startPtt"
@touchend.prevent="stopPtt"
@click.prevent="phase === 'error' ? (phase = 'idle') : undefined"
:disabled="phase === 'transcribing' || phase === 'generating'"
:aria-label="phase === 'recording' ? 'Release to send' : 'Hold to speak'"
:title="phase === 'recording' ? 'Release to send' : 'Hold Space or tap to speak'"
>
<!-- Idle: mic icon -->
<svg v-if="phase === 'idle' || phase === 'error'" width="22" height="22" viewBox="0 0 24 24" fill="currentColor">
<path d="M12 14c1.66 0 3-1.34 3-3V5c0-1.66-1.34-3-3-3S9 3.34 9 5v6c0 1.66 1.34 3 3 3zm-1-9c0-.55.45-1 1-1s1 .45 1 1v6c0 .55-.45 1-1 1s-1-.45-1-1V5zm6 6c0 2.76-2.24 5-5 5s-5-2.24-5-5H5c0 3.53 2.61 6.43 6 6.92V21h2v-3.08c3.39-.49 6-3.39 6-6.92h-2z"/>
</svg>
<!-- Recording: waveform / stop icon -->
<svg v-else-if="phase === 'recording'" width="22" height="22" viewBox="0 0 24 24" fill="currentColor">
<path d="M6 19h4V5H6v14zm8-14v14h4V5h-4z"/>
</svg>
<!-- Busy: spinner dots -->
<span v-else-if="phase === 'transcribing' || phase === 'generating'" class="voice-spinner">
<span></span><span></span><span></span>
</span>
<!-- Speaking: sound waves -->
<svg v-else-if="phase === 'speaking'" width="22" height="22" viewBox="0 0 24 24" fill="currentColor">
<path d="M3 9v6h4l5 5V4L7 9H3zm13.5 3c0-1.77-1.02-3.29-2.5-4.03v8.05c1.48-.73 2.5-2.25 2.5-4.02zM14 3.23v2.06c2.89.86 5 3.54 5 6.71s-2.11 5.85-5 6.71v2.06c4.01-.91 7-4.49 7-8.77s-2.99-7.86-7-8.77z"/>
</svg>
</button>
</div>
</div>
</Teleport>
</template>
<style scoped>
.voice-overlay {
position: fixed;
bottom: 2rem;
right: 1.5rem;
z-index: 8000;
display: flex;
flex-direction: column;
align-items: flex-end;
gap: 0.5rem;
pointer-events: none;
}
/* ─── Panel ──────────────────────────────────────────────────────────────── */
.voice-panel {
pointer-events: all;
width: min(320px, 90vw);
background: var(--color-bg-card);
border: 1px solid var(--color-border);
border-radius: 14px;
box-shadow: 0 8px 32px var(--color-shadow, rgba(0,0,0,0.22));
overflow: hidden;
display: flex;
flex-direction: column;
max-height: 340px;
}
.voice-panel-header {
display: flex;
align-items: center;
justify-content: space-between;
padding: 0.6rem 0.85rem 0.5rem;
border-bottom: 1px solid var(--color-border);
flex-shrink: 0;
}
.voice-panel-title {
font-size: 0.75rem;
font-weight: 700;
text-transform: uppercase;
letter-spacing: 0.07em;
color: var(--color-primary);
}
.voice-panel-close {
background: none;
border: none;
font-size: 1.3rem;
line-height: 1;
color: var(--color-text-muted);
cursor: pointer;
padding: 0 0.1rem;
}
.voice-panel-close:hover { color: var(--color-text); }
.voice-transcript {
flex: 1;
overflow-y: auto;
padding: 0.65rem 0.85rem;
display: flex;
flex-direction: column;
gap: 0.45rem;
}
.voice-msg {
font-size: 0.875rem;
line-height: 1.45;
padding: 0.4rem 0.65rem;
border-radius: 10px;
max-width: 88%;
white-space: pre-wrap;
word-break: break-word;
}
.voice-msg--user {
align-self: flex-end;
background: color-mix(in srgb, var(--color-primary) 12%, transparent);
border: 1px solid color-mix(in srgb, var(--color-primary) 25%, transparent);
color: var(--color-text);
}
.voice-msg--assistant {
align-self: flex-start;
background: var(--color-bg-secondary);
border: 1px solid var(--color-border);
color: var(--color-text);
}
.voice-msg--streaming { opacity: 0.85; }
.voice-cursor {
display: inline-block;
animation: blink 0.9s step-end infinite;
margin-left: 1px;
color: var(--color-primary);
}
@keyframes blink { 0%, 100% { opacity: 1; } 50% { opacity: 0; } }
/* ─── Button cluster ─────────────────────────────────────────────────────── */
.voice-btn-wrap {
pointer-events: all;
display: flex;
flex-direction: column;
align-items: flex-end;
gap: 0.35rem;
}
.voice-status-label {
font-size: 0.72rem;
color: var(--color-text-muted);
background: var(--color-bg-card);
border: 1px solid var(--color-border);
border-radius: 6px;
padding: 0.18rem 0.5rem;
white-space: nowrap;
box-shadow: 0 2px 6px var(--color-shadow, rgba(0,0,0,0.12));
}
.voice-hint { opacity: 0.7; }
.voice-hint kbd {
font-family: ui-monospace, monospace;
font-size: 0.68rem;
padding: 0.05rem 0.25rem;
border: 1px solid var(--color-border);
border-radius: 3px;
background: var(--color-bg-secondary);
}
.voice-error-label { color: #ef4444; }
.voice-cancel {
background: var(--color-bg-card);
border: 1px solid var(--color-border);
border-radius: 50%;
width: 28px;
height: 28px;
display: flex;
align-items: center;
justify-content: center;
cursor: pointer;
color: var(--color-text-muted);
box-shadow: 0 2px 6px var(--color-shadow, rgba(0,0,0,0.12));
transition: all 0.15s;
}
.voice-cancel:hover { color: #ef4444; border-color: #ef4444; }
.voice-ptt-btn {
width: 58px;
height: 58px;
border-radius: 50%;
border: none;
background: linear-gradient(135deg, #6366f1, #4f46e5);
color: #fff;
display: flex;
align-items: center;
justify-content: center;
cursor: pointer;
box-shadow: 0 4px 16px rgba(99, 102, 241, 0.45);
transition: transform 0.12s, box-shadow 0.12s, background 0.2s;
touch-action: none;
user-select: none;
flex-shrink: 0;
}
.voice-ptt-btn:hover:not(:disabled) {
transform: scale(1.06);
box-shadow: 0 6px 20px rgba(99, 102, 241, 0.55);
}
.voice-ptt-btn:disabled { opacity: 0.6; cursor: not-allowed; }
.voice-ptt--recording {
background: linear-gradient(135deg, #ef4444, #dc2626) !important;
box-shadow: 0 4px 16px rgba(239, 68, 68, 0.5) !important;
animation: ptt-pulse 0.9s ease-in-out infinite;
}
.voice-ptt--busy {
background: linear-gradient(135deg, #8b5cf6, #7c3aed) !important;
box-shadow: 0 4px 16px rgba(139, 92, 246, 0.45) !important;
}
.voice-ptt--speaking {
background: linear-gradient(135deg, #10b981, #059669) !important;
box-shadow: 0 4px 16px rgba(16, 185, 129, 0.45) !important;
animation: ptt-pulse 1.4s ease-in-out infinite;
}
.voice-ptt--error {
background: linear-gradient(135deg, #6b7280, #4b5563) !important;
box-shadow: none !important;
}
@keyframes ptt-pulse {
0%, 100% { transform: scale(1); }
50% { transform: scale(1.08); }
}
/* ─── Spinner dots ───────────────────────────────────────────────────────── */
.voice-spinner {
display: flex;
gap: 4px;
align-items: center;
}
.voice-spinner span {
width: 5px;
height: 5px;
border-radius: 50%;
background: #fff;
animation: dot-bounce 1.2s ease-in-out infinite;
}
.voice-spinner span:nth-child(2) { animation-delay: 0.2s; }
.voice-spinner span:nth-child(3) { animation-delay: 0.4s; }
@keyframes dot-bounce {
0%, 80%, 100% { transform: scale(0.7); opacity: 0.5; }
40% { transform: scale(1); opacity: 1; }
}
/* ─── Transition ─────────────────────────────────────────────────────────── */
.panel-slide-enter-active,
.panel-slide-leave-active {
transition: opacity 0.2s ease, transform 0.2s ease;
}
.panel-slide-enter-from,
.panel-slide-leave-to {
opacity: 0;
transform: translateY(8px);
}
</style>