feat: voice S2S — faster-whisper STT, Kokoro TTS, PTT overlay
Implements full speech-to-speech pipeline (all 4 phases): Backend (Phase 1): - services/stt.py: lazy WhisperModel singleton, run_in_executor transcription - services/tts.py: lazy KPipeline singleton, WAV synthesis at 24kHz/16-bit - routes/voice.py: /api/voice/status, /voices, /transcribe, /synthesise - config.py: VOICE_ENABLED, STT_BACKEND, STT_MODEL, TTS_BACKEND env vars - app.py: load STT/TTS models at startup when VOICE_ENABLED=true - llm.py: voice_mode + voice_speech_style params inject speak-naturally prefix - generation_task.py: voice_mode passed through from chat route - chat.py: "voice" conversation type allowed + excluded from retention cleanup - pyproject.toml + Dockerfile: faster-whisper, kokoro, soundfile dependencies Frontend (Phases 2–4): - composables/useVoiceRecorder.ts: MediaRecorder PTT wrapper - composables/useVoiceAudio.ts: AudioContext WAV playback wrapper - BriefingView.vue: Listen button (TTS read-aloud), auto-TTS mode, mic PTT - VoiceOverlay.vue: global floating PTT button; creates/reuses voice conv; full record→transcribe→stream→TTS flow; Space bar hold-to-talk via App.vue - SettingsView.vue: Voice tab (status badge, speech style, voice/speed) - App.vue: mounts VoiceOverlay; Space keydown/keyup fires voice:ptt-toggle - api/client.ts: getVoiceStatus, getVoiceList, transcribeAudio, synthesiseSpeech Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,60 @@
|
||||
import { ref, readonly } from 'vue'
|
||||
|
||||
/**
|
||||
* Audio playback composable wrapping the Web Audio API.
|
||||
*
|
||||
* Usage:
|
||||
* const { playing, isSupported, play, stop } = useVoiceAudio()
|
||||
* await play(wavBlob)
|
||||
*/
|
||||
export function useVoiceAudio() {
|
||||
const playing = ref(false)
|
||||
const isSupported = typeof AudioContext !== 'undefined' || typeof (window as unknown as Record<string, unknown>).webkitAudioContext !== 'undefined'
|
||||
|
||||
let audioCtx: AudioContext | null = null
|
||||
let currentSource: AudioBufferSourceNode | null = null
|
||||
|
||||
function _getCtx(): AudioContext {
|
||||
if (!audioCtx || audioCtx.state === 'closed') {
|
||||
const Ctx = window.AudioContext ?? (window as unknown as Record<string, typeof AudioContext>).webkitAudioContext
|
||||
audioCtx = new Ctx()
|
||||
}
|
||||
return audioCtx
|
||||
}
|
||||
|
||||
async function play(blob: Blob): Promise<void> {
|
||||
stop()
|
||||
const ctx = _getCtx()
|
||||
if (ctx.state === 'suspended') await ctx.resume()
|
||||
|
||||
const arrayBuffer = await blob.arrayBuffer()
|
||||
const audioBuffer = await ctx.decodeAudioData(arrayBuffer)
|
||||
|
||||
const source = ctx.createBufferSource()
|
||||
source.buffer = audioBuffer
|
||||
source.connect(ctx.destination)
|
||||
currentSource = source
|
||||
playing.value = true
|
||||
|
||||
source.onended = () => {
|
||||
playing.value = false
|
||||
currentSource = null
|
||||
}
|
||||
source.start(0)
|
||||
}
|
||||
|
||||
function stop(): void {
|
||||
if (currentSource) {
|
||||
try { currentSource.stop() } catch { /* already stopped */ }
|
||||
currentSource = null
|
||||
}
|
||||
playing.value = false
|
||||
}
|
||||
|
||||
return {
|
||||
playing: readonly(playing),
|
||||
isSupported,
|
||||
play,
|
||||
stop,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
import { ref, readonly } from 'vue'
|
||||
|
||||
/**
|
||||
* Push-to-talk recorder wrapping the browser MediaRecorder API.
|
||||
*
|
||||
* Usage:
|
||||
* const { recording, error, isSupported, startRecording, stopRecording } = useVoiceRecorder()
|
||||
* await startRecording()
|
||||
* const blob = await stopRecording() // resolves with the recorded audio Blob
|
||||
*/
|
||||
export function useVoiceRecorder() {
|
||||
const recording = ref(false)
|
||||
const error = ref<string | null>(null)
|
||||
const isSupported = typeof MediaRecorder !== 'undefined' && !!navigator.mediaDevices?.getUserMedia
|
||||
|
||||
let mediaRecorder: MediaRecorder | null = null
|
||||
let chunks: Blob[] = []
|
||||
let stream: MediaStream | null = null
|
||||
let resolveStop: ((blob: Blob) => void) | null = null
|
||||
let rejectStop: ((err: Error) => void) | null = null
|
||||
|
||||
async function startRecording(): Promise<void> {
|
||||
error.value = null
|
||||
if (!isSupported) {
|
||||
error.value = 'Audio recording is not supported in this browser'
|
||||
return
|
||||
}
|
||||
if (recording.value) return
|
||||
|
||||
try {
|
||||
stream = await navigator.mediaDevices.getUserMedia({ audio: true })
|
||||
} catch (e) {
|
||||
error.value = 'Microphone access denied'
|
||||
return
|
||||
}
|
||||
|
||||
chunks = []
|
||||
const mimeType = MediaRecorder.isTypeSupported('audio/webm;codecs=opus')
|
||||
? 'audio/webm;codecs=opus'
|
||||
: MediaRecorder.isTypeSupported('audio/webm')
|
||||
? 'audio/webm'
|
||||
: ''
|
||||
|
||||
mediaRecorder = mimeType ? new MediaRecorder(stream, { mimeType }) : new MediaRecorder(stream)
|
||||
|
||||
mediaRecorder.ondataavailable = (e) => {
|
||||
if (e.data.size > 0) chunks.push(e.data)
|
||||
}
|
||||
|
||||
mediaRecorder.onstop = () => {
|
||||
const blob = new Blob(chunks, { type: mediaRecorder?.mimeType ?? 'audio/webm' })
|
||||
chunks = []
|
||||
stream?.getTracks().forEach((t) => t.stop())
|
||||
stream = null
|
||||
recording.value = false
|
||||
resolveStop?.(blob)
|
||||
resolveStop = null
|
||||
rejectStop = null
|
||||
}
|
||||
|
||||
mediaRecorder.onerror = () => {
|
||||
recording.value = false
|
||||
error.value = 'Recording error'
|
||||
rejectStop?.(new Error('MediaRecorder error'))
|
||||
resolveStop = null
|
||||
rejectStop = null
|
||||
}
|
||||
|
||||
mediaRecorder.start(100) // collect in 100ms chunks
|
||||
recording.value = true
|
||||
}
|
||||
|
||||
function stopRecording(): Promise<Blob> {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (!mediaRecorder || !recording.value) {
|
||||
reject(new Error('Not recording'))
|
||||
return
|
||||
}
|
||||
resolveStop = resolve
|
||||
rejectStop = reject
|
||||
mediaRecorder.stop()
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
recording: readonly(recording),
|
||||
error: readonly(error),
|
||||
isSupported,
|
||||
startRecording,
|
||||
stopRecording,
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user