From fc6c9648f9d394f7d5e6c5d05315d830365e3d6d Mon Sep 17 00:00:00 2001 From: bvandeusen Date: Fri, 17 Apr 2026 08:04:03 -0400 Subject: [PATCH] fix(voice): eliminate dual-recorder conflict and defunct element crashes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit VadHandler is now the sole mic owner — removed separate AudioRecorder that caused audio focus contention on Android. Audio from onSpeechEnd is encoded as WAV for Whisper. Provider switched to autoDispose to match widget lifecycle, preventing defunct element assertions. Recording UI deferred until mic is open. Co-Authored-By: Claude Opus 4.6 --- lib/app.dart | 1 - lib/data/api/voice_api.dart | 6 +- lib/providers/voice_provider.dart | 259 +++++++++++------- lib/screens/briefing/briefing_screen.dart | 1 - lib/screens/chat/chat_screen.dart | 2 - linux/flutter/generated_plugins.cmake | 1 + macos/Flutter/GeneratedPluginRegistrant.swift | 4 +- windows/flutter/generated_plugins.cmake | 1 + 8 files changed, 160 insertions(+), 115 deletions(-) diff --git a/lib/app.dart b/lib/app.dart index 63fa63a..763ac89 100644 --- a/lib/app.dart +++ b/lib/app.dart @@ -529,7 +529,6 @@ class _QuickCaptureBarState extends ConsumerState<_QuickCaptureBar> { @override void dispose() { _controller.dispose(); - ref.read(voiceProvider.notifier).exitVoiceMode(); super.dispose(); } diff --git a/lib/data/api/voice_api.dart b/lib/data/api/voice_api.dart index 6f57378..907ab01 100644 --- a/lib/data/api/voice_api.dart +++ b/lib/data/api/voice_api.dart @@ -40,7 +40,7 @@ class VoiceApi { } } - /// POST WebM/Opus audio bytes and return the transcript string. + /// POST audio bytes (WAV) and return the transcript string. /// [context] is optional recent conversation text passed as initial_prompt /// to Whisper, reducing mishearings of domain-specific words. /// Returns empty string on empty or error response. @@ -49,8 +49,8 @@ class VoiceApi { final fields = { 'audio': MultipartFile.fromBytes( audioBytes, - filename: 'audio.m4a', - contentType: DioMediaType('audio', 'mp4'), + filename: 'audio.wav', + contentType: DioMediaType('audio', 'wav'), ), if (context != null && context.isNotEmpty) 'context': context, }; diff --git a/lib/providers/voice_provider.dart b/lib/providers/voice_provider.dart index 2cd07d0..0df349b 100644 --- a/lib/providers/voice_provider.dart +++ b/lib/providers/voice_provider.dart @@ -1,13 +1,13 @@ import 'dart:async'; import 'dart:collection'; import 'dart:io'; +import 'dart:math'; import 'dart:typed_data'; import 'package:flutter_riverpod/flutter_riverpod.dart'; import 'package:just_audio/just_audio.dart'; import 'package:path_provider/path_provider.dart'; import 'package:permission_handler/permission_handler.dart'; -import 'package:record/record.dart'; import 'package:vad/vad.dart'; import 'api_client_provider.dart'; @@ -52,6 +52,53 @@ String stripMarkdownForTts(String text) { .trim(); } +/// Encode float PCM samples (-1..1) as a 16-bit mono WAV at 16 kHz. +Uint8List encodeWav(List samples, {int sampleRate = 16000}) { + final numSamples = samples.length; + final dataSize = numSamples * 2; + final fileSize = 44 + dataSize; + final buf = ByteData(fileSize); + + // RIFF header + buf.setUint8(0, 0x52); // R + buf.setUint8(1, 0x49); // I + buf.setUint8(2, 0x46); // F + buf.setUint8(3, 0x46); // F + buf.setUint32(4, fileSize - 8, Endian.little); + buf.setUint8(8, 0x57); // W + buf.setUint8(9, 0x41); // A + buf.setUint8(10, 0x56); // V + buf.setUint8(11, 0x45); // E + + // fmt chunk + buf.setUint8(12, 0x66); // f + buf.setUint8(13, 0x6D); // m + buf.setUint8(14, 0x74); // t + buf.setUint8(15, 0x20); // (space) + buf.setUint32(16, 16, Endian.little); // chunk size + buf.setUint16(20, 1, Endian.little); // PCM format + buf.setUint16(22, 1, Endian.little); // mono + buf.setUint32(24, sampleRate, Endian.little); + buf.setUint32(28, sampleRate * 2, Endian.little); // byte rate + buf.setUint16(32, 2, Endian.little); // block align + buf.setUint16(34, 16, Endian.little); // bits per sample + + // data chunk + buf.setUint8(36, 0x64); // d + buf.setUint8(37, 0x61); // a + buf.setUint8(38, 0x74); // t + buf.setUint8(39, 0x61); // a + buf.setUint32(40, dataSize, Endian.little); + + for (var i = 0; i < numSamples; i++) { + final clamped = samples[i].clamp(-1.0, 1.0); + final int16 = (clamped * 32767).round().clamp(-32768, 32767); + buf.setInt16(44 + i * 2, int16, Endian.little); + } + + return buf.buffer.asUint8List(); +} + // ── State ───────────────────────────────────────────────────────────────────── enum VoiceMode { idle, recording, transcribing, playing } @@ -60,9 +107,7 @@ class VoiceState { final VoiceMode mode; final bool voiceModeActive; final bool available; - /// Normalized mic amplitude 0.0–1.0 while recording. Drives the live - /// pulse on VoiceMicButton so the user has obvious feedback that audio - /// is actually being picked up. + /// Normalized mic amplitude 0.0–1.0 while recording. final double amplitude; const VoiceState({ @@ -89,23 +134,25 @@ class VoiceState { // ── Provider ────────────────────────────────────────────────────────────────── final voiceProvider = - NotifierProvider(VoiceNotifier.new); + NotifierProvider.autoDispose(VoiceNotifier.new); // ── Notifier ────────────────────────────────────────────────────────────────── class VoiceNotifier extends Notifier { - // Audio I/O - AudioRecorder? _recorder; + // Audio playback AudioPlayer? _player; - StreamSubscription? _amplitudeSubscription; - // VAD-based speech detection + // VAD — sole owner of the microphone VadHandler? _vadHandler; StreamSubscription? _vadSpeechStartSub; StreamSubscription>? _vadSpeechEndSub; + StreamSubscription<({double isSpeech, double notSpeech, List frame})>? + _vadFrameSub; + StreamSubscription? _vadErrorSub; bool _speechDetected = false; int _speechStartMs = 0; static const _vadGraceMs = 1500; + bool _disposed = false; // Voice mode callbacks Future Function(String transcript)? _onTranscript; @@ -117,11 +164,10 @@ class VoiceNotifier extends Notifier { int _lastSeenLength = 0; bool _streamComplete = false; - // Last complete assistant response — passed to Whisper as initial_prompt - // to reduce STT mishearings of domain-specific words. + // Whisper context hint String _lastAssistantContent = ''; - // Empty transcript counter — show feedback after consecutive blanks + // Empty transcript counter int _emptyTranscriptCount = 0; // TTS playback queue @@ -132,21 +178,21 @@ class VoiceNotifier extends Notifier { @override VoiceState build() { + _disposed = false; _player = AudioPlayer(); ref.onDispose(() { - _amplitudeSubscription?.cancel(); - _recorder?.dispose(); + _disposed = true; + _cancelSubscriptions(); + _vadHandler?.dispose(); + _vadHandler = null; _player?.dispose(); + _player = null; }); return const VoiceState(); } // ── Public API ────────────────────────────────────────────────────────────── - /// Enter voice mode. Checks server availability and mic permission first. - /// [onTranscript] is called with the transcript when a recording completes. - /// [enableTts] — if true, TTS plays when [feedContent] is called. - /// [onError] — called with a human-readable message on failure. Future enterVoiceMode({ required Future Function(String transcript) onTranscript, bool enableTts = false, @@ -160,27 +206,23 @@ class VoiceNotifier extends Notifier { return; } - // Check server availability — STT is required, TTS is optional. try { final status = await ref.read(voiceRepositoryProvider).checkStatus(); if (!status.enabled || !status.stt) { onError('Speech-to-text not available on this server'); return; } - // Downgrade to STT-only when TTS is unavailable if (!status.tts) enableTts = false; } catch (_) { onError('Could not reach voice service'); return; } - // Check microphone permission var permStatus = await Permission.microphone.request(); if (permStatus == PermissionStatus.permanentlyDenied) { onError('Microphone blocked — opening settings'); final opened = await openAppSettings(); if (!opened) return; - // Re-check after user returns from settings permStatus = await Permission.microphone.status; } if (!permStatus.isGranted) { @@ -198,27 +240,11 @@ class VoiceNotifier extends Notifier { await _startListening(); } - /// Exit voice mode, stop all recording and TTS. void exitVoiceMode() { - _amplitudeSubscription?.cancel(); - _amplitudeSubscription = null; - _recorder?.stop(); - _stopVad(); - _player?.stop(); - _ttsQueue.clear(); - _ttsPlaying = false; - _sentenceBuffer = ''; - _lastSeenLength = 0; - _streamComplete = false; - _speechDetected = false; - _onTranscript = null; - _onError = null; - state = const VoiceState(); + _cleanup(); + if (!_disposed) state = const VoiceState(); } - /// Feed streaming assistant content for TTS synthesis. - /// Call from screens with the full [fullContent] string on each update. - /// Set [isComplete] to true when the stream has finished. void feedContent(String fullContent, {required bool isComplete}) { if (!state.voiceModeActive || !_enableTts) return; @@ -237,112 +263,131 @@ class VoiceNotifier extends Notifier { } } + // ── Internal helpers ──────────────────────────────────────────────────────── + + void _cleanup() { + _cancelSubscriptions(); + final handler = _vadHandler; + _vadHandler = null; + handler?.dispose(); + _player?.stop(); + _ttsQueue.clear(); + _ttsPlaying = false; + _sentenceBuffer = ''; + _lastSeenLength = 0; + _streamComplete = false; + _speechDetected = false; + _onTranscript = null; + _onError = null; + } + + void _cancelSubscriptions() { + _vadSpeechStartSub?.cancel(); + _vadSpeechStartSub = null; + _vadSpeechEndSub?.cancel(); + _vadSpeechEndSub = null; + _vadFrameSub?.cancel(); + _vadFrameSub = null; + _vadErrorSub?.cancel(); + _vadErrorSub = null; + } + // ── Internal recording ────────────────────────────────────────────────────── Future _startListening() async { - if (!state.voiceModeActive) return; + if (_disposed || !state.voiceModeActive) return; _speechDetected = false; _speechStartMs = 0; - state = state.copyWith(mode: VoiceMode.recording); - - final dir = _tempDir ?? await getTemporaryDirectory(); - final path = - '${dir.path}/voice_rec_${DateTime.now().millisecondsSinceEpoch}.m4a'; try { - _recorder?.dispose(); - _recorder = AudioRecorder(); - - if (!await _recorder!.hasPermission()) { - _onError?.call('Microphone permission was revoked'); - exitVoiceMode(); - return; - } - - await _recorder!.start( - const RecordConfig(encoder: AudioEncoder.aacLc, sampleRate: 16000), - path: path, - ); - - _amplitudeSubscription?.cancel(); - _amplitudeSubscription = _recorder! - .onAmplitudeChanged(const Duration(milliseconds: 200)) - .listen(_onAmplitude); - - // VAD for speech detection — uses its own AudioRecorder internally await _stopVad(); _vadHandler = VadHandler.create(); + _vadSpeechStartSub = _vadHandler!.onSpeechStart.listen((_) { + if (_disposed || !state.voiceModeActive) return; if (!_speechDetected) { _speechDetected = true; _speechStartMs = DateTime.now().millisecondsSinceEpoch; } }); - _vadSpeechEndSub = _vadHandler!.onSpeechEnd.listen((_) { - if (!state.voiceModeActive) return; + + _vadSpeechEndSub = _vadHandler!.onSpeechEnd.listen((audioSamples) { + if (_disposed || !state.voiceModeActive) return; final now = DateTime.now().millisecondsSinceEpoch; final sinceStart = _speechStartMs > 0 ? now - _speechStartMs : 0; if (_speechDetected && sinceStart >= _vadGraceMs) { - _amplitudeSubscription?.cancel(); - _amplitudeSubscription = null; _stopVad(); - _handleSilence(); + _handleSpeechEnd(audioSamples); } }); + + _vadFrameSub = _vadHandler!.onFrameProcessed.listen((event) { + if (_disposed || !state.voiceModeActive) return; + final frame = event.frame; + if (frame.isEmpty) return; + double sumSq = 0; + for (final s in frame) { + sumSq += s * s; + } + final rms = sqrt(sumSq / frame.length); + final norm = (rms * 4.0).clamp(0.0, 1.0); + if ((norm - state.amplitude).abs() > 0.02) { + state = state.copyWith(amplitude: norm); + } + }); + + _vadErrorSub = _vadHandler!.onError.listen((msg) { + if (_disposed) return; + _onError?.call('VAD error: $msg'); + }); + await _vadHandler!.startListening(model: 'v5'); + + // Only show recording UI after the mic is actually open. + if (!_disposed && state.voiceModeActive) { + state = state.copyWith(mode: VoiceMode.recording, amplitude: 0.0); + } } catch (e) { _onError?.call('Microphone error: $e'); - exitVoiceMode(); + _cleanup(); + if (!_disposed) state = const VoiceState(); } } Future _stopVad() async { - await _vadSpeechStartSub?.cancel(); - _vadSpeechStartSub = null; - await _vadSpeechEndSub?.cancel(); - _vadSpeechEndSub = null; + _cancelSubscriptions(); if (_vadHandler != null) { - await _vadHandler!.dispose(); + final handler = _vadHandler!; _vadHandler = null; + await handler.dispose(); } } - void _onAmplitude(Amplitude event) { - if (!state.voiceModeActive) return; - final db = event.current; - if (db.isNaN || db.isInfinite) return; - final norm = ((db + 60.0) / 60.0).clamp(0.0, 1.0); - if ((norm - state.amplitude).abs() > 0.02) { - state = state.copyWith(amplitude: norm); - } - } - - Future _handleSilence() async { - if (!state.voiceModeActive) return; + Future _handleSpeechEnd(List audioSamples) async { + if (_disposed || !state.voiceModeActive) return; state = state.copyWith(mode: VoiceMode.transcribing); - final path = await _recorder!.stop(); - if (path == null || !state.voiceModeActive) return; - try { - final bytes = await File(path).readAsBytes(); - await File(path).delete().catchError((_) => File(path)); + final wavBytes = encodeWav(audioSamples); - if (!state.voiceModeActive) return; + if (_disposed || !state.voiceModeActive) return; final transcript = await ref.read(voiceRepositoryProvider).transcribe( - bytes, - context: _lastAssistantContent.isNotEmpty ? _lastAssistantContent : null, + wavBytes, + context: + _lastAssistantContent.isNotEmpty ? _lastAssistantContent : null, ); - if (!state.voiceModeActive) return; + if (_disposed || !state.voiceModeActive) return; if (transcript.isEmpty) { _emptyTranscriptCount++; if (_emptyTranscriptCount >= 3) { _onError?.call('No speech detected — tap the mic to try again'); - exitVoiceMode(); + _cleanup(); + if (!_disposed) state = const VoiceState(); + return; } await _startListening(); @@ -350,24 +395,25 @@ class VoiceNotifier extends Notifier { } _emptyTranscriptCount = 0; - // Reset TTS state for this new turn _sentenceBuffer = ''; _lastSeenLength = 0; _streamComplete = false; - if (_enableTts) { + if (_enableTts && !_disposed) { state = state.copyWith(mode: VoiceMode.playing); } await _onTranscript?.call(transcript); - // If TTS is not enabled, loop immediately - if (!_enableTts && state.voiceModeActive) { - await _startListening(); + // In STT-only mode (no TTS), return to idle after transcript is sent. + // The user taps the mic again to record another message. + if (!_enableTts && !_disposed && state.voiceModeActive) { + state = state.copyWith(mode: VoiceMode.idle); } } catch (e) { _onError?.call('Voice error: transcription failed'); - exitVoiceMode(); + _cleanup(); + if (!_disposed) state = const VoiceState(); } } @@ -395,7 +441,7 @@ class VoiceNotifier extends Notifier { try { final wavBytes = await ref.read(voiceRepositoryProvider).synthesise(text); - if (!state.voiceModeActive) return; + if (_disposed || !state.voiceModeActive) return; _ttsQueue.add(wavBytes); if (!_ttsPlaying) _drainTtsQueue(); } catch (_) { @@ -434,6 +480,7 @@ class VoiceNotifier extends Notifier { } void _checkRestartListening() { + if (_disposed) return; if (_streamComplete && _ttsQueue.isEmpty && !_ttsPlaying && diff --git a/lib/screens/briefing/briefing_screen.dart b/lib/screens/briefing/briefing_screen.dart index d2d3080..41678d0 100644 --- a/lib/screens/briefing/briefing_screen.dart +++ b/lib/screens/briefing/briefing_screen.dart @@ -76,7 +76,6 @@ class _BriefingScreenState extends ConsumerState WidgetsBinding.instance.removeObserver(this); _controller.dispose(); _scrollController.dispose(); - ref.read(voiceProvider.notifier).exitVoiceMode(); super.dispose(); } diff --git a/lib/screens/chat/chat_screen.dart b/lib/screens/chat/chat_screen.dart index 89347f9..af341f5 100644 --- a/lib/screens/chat/chat_screen.dart +++ b/lib/screens/chat/chat_screen.dart @@ -68,8 +68,6 @@ class _ChatScreenState extends ConsumerState WidgetsBinding.instance.removeObserver(this); _controller.dispose(); _scrollController.dispose(); - // Exit voice mode if the user navigates away mid-session. - ref.read(voiceProvider.notifier).exitVoiceMode(); super.dispose(); } diff --git a/linux/flutter/generated_plugins.cmake b/linux/flutter/generated_plugins.cmake index 641253b..61405b3 100644 --- a/linux/flutter/generated_plugins.cmake +++ b/linux/flutter/generated_plugins.cmake @@ -10,6 +10,7 @@ list(APPEND FLUTTER_PLUGIN_LIST ) list(APPEND FLUTTER_FFI_PLUGIN_LIST + vad ) set(PLUGIN_BUNDLED_LIBRARIES) diff --git a/macos/Flutter/GeneratedPluginRegistrant.swift b/macos/Flutter/GeneratedPluginRegistrant.swift index 27878f5..74ec380 100644 --- a/macos/Flutter/GeneratedPluginRegistrant.swift +++ b/macos/Flutter/GeneratedPluginRegistrant.swift @@ -11,7 +11,7 @@ import flutter_timezone import just_audio import open_file_mac import package_info_plus -import record_darwin +import record_macos import shared_preferences_foundation import url_launcher_macos @@ -22,7 +22,7 @@ func RegisterGeneratedPlugins(registry: FlutterPluginRegistry) { JustAudioPlugin.register(with: registry.registrar(forPlugin: "JustAudioPlugin")) OpenFilePlugin.register(with: registry.registrar(forPlugin: "OpenFilePlugin")) FPPPackageInfoPlusPlugin.register(with: registry.registrar(forPlugin: "FPPPackageInfoPlusPlugin")) - RecordPlugin.register(with: registry.registrar(forPlugin: "RecordPlugin")) + RecordMacOsPlugin.register(with: registry.registrar(forPlugin: "RecordMacOsPlugin")) SharedPreferencesPlugin.register(with: registry.registrar(forPlugin: "SharedPreferencesPlugin")) UrlLauncherPlugin.register(with: registry.registrar(forPlugin: "UrlLauncherPlugin")) } diff --git a/windows/flutter/generated_plugins.cmake b/windows/flutter/generated_plugins.cmake index f53b4aa..c57f7ba 100644 --- a/windows/flutter/generated_plugins.cmake +++ b/windows/flutter/generated_plugins.cmake @@ -11,6 +11,7 @@ list(APPEND FLUTTER_PLUGIN_LIST ) list(APPEND FLUTTER_FFI_PLUGIN_LIST + vad ) set(PLUGIN_BUNDLED_LIBRARIES)