diff --git a/frontend/src/composables/useVad.ts b/frontend/src/composables/useVad.ts new file mode 100644 index 0000000..ad0718b --- /dev/null +++ b/frontend/src/composables/useVad.ts @@ -0,0 +1,157 @@ +import { ref, readonly } from 'vue' +import type { MicVAD } from '@ricky0123/vad-web' + +export interface UseVadOptions { + /** Called when VAD detects a real speech-end event after the grace period. */ + onSpeechEnd: () => void + /** Called when stopAndCheck() is invoked but no speech was ever detected. */ + onNoSpeech: () => void + /** Minimum ms between speech-start and speech-end before the end event fires. */ + graceMs?: number + /** Minimum total recording duration before speech-end can fire. */ + minRecordingMs?: number +} + +/** + * Silero VAD-based speech detector. + * + * Replaces amplitude-based silence detection. Uses the Silero VAD v5 ONNX + * model via `@ricky0123/vad-web`. The package ships the model + audio + * worklet, and depends on `onnxruntime-web`'s WASM. Vite is configured + * (see `vite.config.ts`) to serve these assets from the site root, which + * is why we pass `baseAssetPath: "/"` and `onnxWASMBasePath: "/"`. + * + * `amplitude` is computed separately from a Web Audio API analyser node + * on the same stream — it drives the mic pulse animation and is not part + * of the stop decision. VAD's `onSpeechEnd` is the stop trigger. + * + * If VAD never detects speech during a session, calling `stopAndCheck()` + * fires `onNoSpeech` instead of treating the recording as transcribable. + */ +export function useVad(options: UseVadOptions) { + const { + onSpeechEnd, + onNoSpeech, + graceMs = 1500, + minRecordingMs = 500, + } = options + + const speaking = ref(false) + const amplitude = ref(0) + const loaded = ref(false) + + let micVad: MicVAD | null = null + let audioCtx: AudioContext | null = null + let analyserInterval: ReturnType | null = null + let speechDetected = false + let speechStartedAt = 0 + let recordingStartedAt = 0 + let stopped = false + + async function start(stream: MediaStream): Promise { + await stop() + stopped = false + speechDetected = false + speechStartedAt = 0 + recordingStartedAt = Date.now() + + // Visual-only amplitude signal. This does NOT drive the stop decision. + audioCtx = new AudioContext() + await audioCtx.resume().catch(() => {}) + const source = audioCtx.createMediaStreamSource(stream) + const analyser = audioCtx.createAnalyser() + analyser.fftSize = 1024 + source.connect(analyser) + const samples = new Float32Array(analyser.fftSize) + + analyserInterval = setInterval(() => { + analyser.getFloatTimeDomainData(samples) + let sumSq = 0 + for (let i = 0; i < samples.length; i++) sumSq += samples[i] * samples[i] + const rms = Math.sqrt(sumSq / samples.length) + amplitude.value = Math.min(rms * 5, 1) + }, 100) + + try { + const { MicVAD } = await import('@ricky0123/vad-web') + micVad = await MicVAD.new({ + model: 'v5', + baseAssetPath: '/', + onnxWASMBasePath: '/', + // MicVAD manages its own audio graph, so we hand it the same stream + // the MediaRecorder is using. It won't consume or alter the stream. + getStream: async () => stream, + onSpeechStart: () => { + speaking.value = true + if (!speechDetected) { + speechDetected = true + speechStartedAt = Date.now() + } + }, + onSpeechEnd: () => { + speaking.value = false + if (stopped) return + const now = Date.now() + const totalElapsed = now - recordingStartedAt + const sinceSpeechStart = speechStartedAt > 0 ? now - speechStartedAt : 0 + if (totalElapsed >= minRecordingMs && sinceSpeechStart >= graceMs) { + stopped = true + onSpeechEnd() + } + }, + onVADMisfire: () => { + // Speech-like blip that was too short to count. Reset the local flag + // so a true "no speech" session still hits onNoSpeech. + speaking.value = false + }, + }) + await micVad.start() + loaded.value = true + } catch (e) { + // VAD init failed — the user can still manual-stop. Log for debugging. + console.error('VAD init failed:', e) + } + } + + async function stop(): Promise<{ hadSpeech: boolean }> { + const had = speechDetected + stopped = true + speaking.value = false + amplitude.value = 0 + + if (analyserInterval !== null) { + clearInterval(analyserInterval) + analyserInterval = null + } + if (audioCtx) { + await audioCtx.close().catch(() => {}) + audioCtx = null + } + if (micVad) { + try { + await micVad.pause() + await micVad.destroy() + } catch { + // Already destroyed or failed — nothing to clean up further. + } + micVad = null + } + return { hadSpeech: had } + } + + async function stopAndCheck(): Promise { + const { hadSpeech } = await stop() + if (!hadSpeech) { + onNoSpeech() + } + } + + return { + speaking: readonly(speaking), + amplitude: readonly(amplitude), + loaded: readonly(loaded), + start, + stop, + stopAndCheck, + } +}