From 5f11b344a3968775681d637bfefa80cf5085d9cb Mon Sep 17 00:00:00 2001 From: bvandeusen Date: Thu, 16 Apr 2026 20:57:52 -0400 Subject: [PATCH] feat(voice): replace amplitude silence detection with Silero VAD Use VadHandler from the vad package (Silero VAD v5 ONNX) for speech detection instead of amplitude thresholds. VAD runs its own PCM stream while the file recorder captures AAC-LC for Whisper. Grace period starts on speech-start, auto-stop on speech-end after grace. No-speech guard shows error on manual stop without detected speech. Co-Authored-By: Claude Opus 4.6 --- lib/providers/voice_provider.dart | 114 ++++++++++++++---------------- 1 file changed, 52 insertions(+), 62 deletions(-) diff --git a/lib/providers/voice_provider.dart b/lib/providers/voice_provider.dart index 1887ae1..2cd07d0 100644 --- a/lib/providers/voice_provider.dart +++ b/lib/providers/voice_provider.dart @@ -8,6 +8,7 @@ import 'package:just_audio/just_audio.dart'; import 'package:path_provider/path_provider.dart'; import 'package:permission_handler/permission_handler.dart'; import 'package:record/record.dart'; +import 'package:vad/vad.dart'; import 'api_client_provider.dart'; @@ -98,25 +99,13 @@ class VoiceNotifier extends Notifier { AudioPlayer? _player; StreamSubscription? _amplitudeSubscription; - // Recording / silence detection - // - // Silence threshold is dynamic: we track the session peak dBFS and treat - // "silent" as "current level is at least _dropFromPeakDb below peak." - // This auto-calibrates to whatever mic + room the user is on rather than - // assuming a fixed ambient level. Until the peak climbs above - // _dynamicArmDb we fall back to a conservative static threshold so a - // quiet room doesn't spin forever. A grace period at the start of the - // recording gives the user time to begin speaking before silence checks - // arm. - int _recordingStartMs = 0; - int _silenceMs = 0; - double _peakDb = -100.0; - static const _fallbackThresholdDb = -35.0; - static const _dropFromPeakDb = 15.0; - static const _dynamicArmDb = -20.0; - static const _graceMs = 1500; - static const _silenceDurationMs = 2000; - static const _minRecordingMs = 300; + // VAD-based speech detection + VadHandler? _vadHandler; + StreamSubscription? _vadSpeechStartSub; + StreamSubscription>? _vadSpeechEndSub; + bool _speechDetected = false; + int _speechStartMs = 0; + static const _vadGraceMs = 1500; // Voice mode callbacks Future Function(String transcript)? _onTranscript; @@ -164,6 +153,9 @@ class VoiceNotifier extends Notifier { required void Function(String message) onError, }) async { if (state.voiceModeActive) { + if (state.mode == VoiceMode.recording && !_speechDetected) { + onError('No speech detected'); + } exitVoiceMode(); return; } @@ -211,12 +203,14 @@ class VoiceNotifier extends Notifier { _amplitudeSubscription?.cancel(); _amplitudeSubscription = null; _recorder?.stop(); + _stopVad(); _player?.stop(); _ttsQueue.clear(); _ttsPlaying = false; _sentenceBuffer = ''; _lastSeenLength = 0; _streamComplete = false; + _speechDetected = false; _onTranscript = null; _onError = null; state = const VoiceState(); @@ -248,9 +242,8 @@ class VoiceNotifier extends Notifier { Future _startListening() async { if (!state.voiceModeActive) return; - _silenceMs = 0; - _peakDb = -100.0; - _recordingStartMs = DateTime.now().millisecondsSinceEpoch; + _speechDetected = false; + _speechStartMs = 0; state = state.copyWith(mode: VoiceMode.recording); final dir = _tempDir ?? await getTemporaryDirectory(); @@ -258,9 +251,6 @@ class VoiceNotifier extends Notifier { '${dir.path}/voice_rec_${DateTime.now().millisecondsSinceEpoch}.m4a'; try { - // Recreate the recorder each session — the record package can leave - // the native AudioRecord in a bad state after stop/error cycles, - // and a stale instance is the most common cause of "could not start". _recorder?.dispose(); _recorder = AudioRecorder(); @@ -279,52 +269,52 @@ class VoiceNotifier extends Notifier { _amplitudeSubscription = _recorder! .onAmplitudeChanged(const Duration(milliseconds: 200)) .listen(_onAmplitude); + + // VAD for speech detection — uses its own AudioRecorder internally + await _stopVad(); + _vadHandler = VadHandler.create(); + _vadSpeechStartSub = _vadHandler!.onSpeechStart.listen((_) { + if (!_speechDetected) { + _speechDetected = true; + _speechStartMs = DateTime.now().millisecondsSinceEpoch; + } + }); + _vadSpeechEndSub = _vadHandler!.onSpeechEnd.listen((_) { + if (!state.voiceModeActive) return; + final now = DateTime.now().millisecondsSinceEpoch; + final sinceStart = _speechStartMs > 0 ? now - _speechStartMs : 0; + if (_speechDetected && sinceStart >= _vadGraceMs) { + _amplitudeSubscription?.cancel(); + _amplitudeSubscription = null; + _stopVad(); + _handleSilence(); + } + }); + await _vadHandler!.startListening(model: 'v5'); } catch (e) { _onError?.call('Microphone error: $e'); exitVoiceMode(); } } + Future _stopVad() async { + await _vadSpeechStartSub?.cancel(); + _vadSpeechStartSub = null; + await _vadSpeechEndSub?.cancel(); + _vadSpeechEndSub = null; + if (_vadHandler != null) { + await _vadHandler!.dispose(); + _vadHandler = null; + } + } + void _onAmplitude(Amplitude event) { if (!state.voiceModeActive) return; - final db = event.current; - final validDb = !(db.isNaN || db.isInfinite); - - // Normalize dB to 0..1 for the pulse animation. -60dB is dead-quiet, - // 0dB is peak; we clamp and bias so the button visibly breathes even - // on soft speech without exploding on loud input. - if (validDb) { - final norm = ((db + 60.0) / 60.0).clamp(0.0, 1.0); - if ((norm - state.amplitude).abs() > 0.02) { - state = state.copyWith(amplitude: norm); - } - if (db > _peakDb) _peakDb = db; - } - - final elapsed = - DateTime.now().millisecondsSinceEpoch - _recordingStartMs; - // Grace period at the start — let the user begin speaking before we - // start silence-counting. Also suppresses any false triggers while - // the native recorder is still warming up. - if (elapsed < _graceMs) return; - if (elapsed < _minRecordingMs) return; - - // Dynamic threshold once we've seen real speech; static fallback - // before that so a dead-silent session doesn't spin forever. - final threshold = - _peakDb > _dynamicArmDb ? _peakDb - _dropFromPeakDb : _fallbackThresholdDb; - final isSilent = !validDb || db < threshold; - - if (isSilent) { - _silenceMs += 200; - if (_silenceMs >= _silenceDurationMs) { - _amplitudeSubscription?.cancel(); - _amplitudeSubscription = null; - _handleSilence(); - } - } else { - _silenceMs = 0; + if (db.isNaN || db.isInfinite) return; + final norm = ((db + 60.0) / 60.0).clamp(0.0, 1.0); + if ((norm - state.amplitude).abs() > 0.02) { + state = state.copyWith(amplitude: norm); } }