diff --git a/lib/app.dart b/lib/app.dart index 63fa63a..763ac89 100644 --- a/lib/app.dart +++ b/lib/app.dart @@ -529,7 +529,6 @@ class _QuickCaptureBarState extends ConsumerState<_QuickCaptureBar> { @override void dispose() { _controller.dispose(); - ref.read(voiceProvider.notifier).exitVoiceMode(); super.dispose(); } diff --git a/lib/data/api/voice_api.dart b/lib/data/api/voice_api.dart index 6f57378..907ab01 100644 --- a/lib/data/api/voice_api.dart +++ b/lib/data/api/voice_api.dart @@ -40,7 +40,7 @@ class VoiceApi { } } - /// POST WebM/Opus audio bytes and return the transcript string. + /// POST audio bytes (WAV) and return the transcript string. /// [context] is optional recent conversation text passed as initial_prompt /// to Whisper, reducing mishearings of domain-specific words. /// Returns empty string on empty or error response. @@ -49,8 +49,8 @@ class VoiceApi { final fields = { 'audio': MultipartFile.fromBytes( audioBytes, - filename: 'audio.m4a', - contentType: DioMediaType('audio', 'mp4'), + filename: 'audio.wav', + contentType: DioMediaType('audio', 'wav'), ), if (context != null && context.isNotEmpty) 'context': context, }; diff --git a/lib/providers/voice_provider.dart b/lib/providers/voice_provider.dart index 2cd07d0..0df349b 100644 --- a/lib/providers/voice_provider.dart +++ b/lib/providers/voice_provider.dart @@ -1,13 +1,13 @@ import 'dart:async'; import 'dart:collection'; import 'dart:io'; +import 'dart:math'; import 'dart:typed_data'; import 'package:flutter_riverpod/flutter_riverpod.dart'; import 'package:just_audio/just_audio.dart'; import 'package:path_provider/path_provider.dart'; import 'package:permission_handler/permission_handler.dart'; -import 'package:record/record.dart'; import 'package:vad/vad.dart'; import 'api_client_provider.dart'; @@ -52,6 +52,53 @@ String stripMarkdownForTts(String text) { .trim(); } +/// Encode float PCM samples (-1..1) as a 16-bit mono WAV at 16 kHz. +Uint8List encodeWav(List samples, {int sampleRate = 16000}) { + final numSamples = samples.length; + final dataSize = numSamples * 2; + final fileSize = 44 + dataSize; + final buf = ByteData(fileSize); + + // RIFF header + buf.setUint8(0, 0x52); // R + buf.setUint8(1, 0x49); // I + buf.setUint8(2, 0x46); // F + buf.setUint8(3, 0x46); // F + buf.setUint32(4, fileSize - 8, Endian.little); + buf.setUint8(8, 0x57); // W + buf.setUint8(9, 0x41); // A + buf.setUint8(10, 0x56); // V + buf.setUint8(11, 0x45); // E + + // fmt chunk + buf.setUint8(12, 0x66); // f + buf.setUint8(13, 0x6D); // m + buf.setUint8(14, 0x74); // t + buf.setUint8(15, 0x20); // (space) + buf.setUint32(16, 16, Endian.little); // chunk size + buf.setUint16(20, 1, Endian.little); // PCM format + buf.setUint16(22, 1, Endian.little); // mono + buf.setUint32(24, sampleRate, Endian.little); + buf.setUint32(28, sampleRate * 2, Endian.little); // byte rate + buf.setUint16(32, 2, Endian.little); // block align + buf.setUint16(34, 16, Endian.little); // bits per sample + + // data chunk + buf.setUint8(36, 0x64); // d + buf.setUint8(37, 0x61); // a + buf.setUint8(38, 0x74); // t + buf.setUint8(39, 0x61); // a + buf.setUint32(40, dataSize, Endian.little); + + for (var i = 0; i < numSamples; i++) { + final clamped = samples[i].clamp(-1.0, 1.0); + final int16 = (clamped * 32767).round().clamp(-32768, 32767); + buf.setInt16(44 + i * 2, int16, Endian.little); + } + + return buf.buffer.asUint8List(); +} + // ── State ───────────────────────────────────────────────────────────────────── enum VoiceMode { idle, recording, transcribing, playing } @@ -60,9 +107,7 @@ class VoiceState { final VoiceMode mode; final bool voiceModeActive; final bool available; - /// Normalized mic amplitude 0.0–1.0 while recording. Drives the live - /// pulse on VoiceMicButton so the user has obvious feedback that audio - /// is actually being picked up. + /// Normalized mic amplitude 0.0–1.0 while recording. final double amplitude; const VoiceState({ @@ -89,23 +134,25 @@ class VoiceState { // ── Provider ────────────────────────────────────────────────────────────────── final voiceProvider = - NotifierProvider(VoiceNotifier.new); + NotifierProvider.autoDispose(VoiceNotifier.new); // ── Notifier ────────────────────────────────────────────────────────────────── class VoiceNotifier extends Notifier { - // Audio I/O - AudioRecorder? _recorder; + // Audio playback AudioPlayer? _player; - StreamSubscription? _amplitudeSubscription; - // VAD-based speech detection + // VAD — sole owner of the microphone VadHandler? _vadHandler; StreamSubscription? _vadSpeechStartSub; StreamSubscription>? _vadSpeechEndSub; + StreamSubscription<({double isSpeech, double notSpeech, List frame})>? + _vadFrameSub; + StreamSubscription? _vadErrorSub; bool _speechDetected = false; int _speechStartMs = 0; static const _vadGraceMs = 1500; + bool _disposed = false; // Voice mode callbacks Future Function(String transcript)? _onTranscript; @@ -117,11 +164,10 @@ class VoiceNotifier extends Notifier { int _lastSeenLength = 0; bool _streamComplete = false; - // Last complete assistant response — passed to Whisper as initial_prompt - // to reduce STT mishearings of domain-specific words. + // Whisper context hint String _lastAssistantContent = ''; - // Empty transcript counter — show feedback after consecutive blanks + // Empty transcript counter int _emptyTranscriptCount = 0; // TTS playback queue @@ -132,21 +178,21 @@ class VoiceNotifier extends Notifier { @override VoiceState build() { + _disposed = false; _player = AudioPlayer(); ref.onDispose(() { - _amplitudeSubscription?.cancel(); - _recorder?.dispose(); + _disposed = true; + _cancelSubscriptions(); + _vadHandler?.dispose(); + _vadHandler = null; _player?.dispose(); + _player = null; }); return const VoiceState(); } // ── Public API ────────────────────────────────────────────────────────────── - /// Enter voice mode. Checks server availability and mic permission first. - /// [onTranscript] is called with the transcript when a recording completes. - /// [enableTts] — if true, TTS plays when [feedContent] is called. - /// [onError] — called with a human-readable message on failure. Future enterVoiceMode({ required Future Function(String transcript) onTranscript, bool enableTts = false, @@ -160,27 +206,23 @@ class VoiceNotifier extends Notifier { return; } - // Check server availability — STT is required, TTS is optional. try { final status = await ref.read(voiceRepositoryProvider).checkStatus(); if (!status.enabled || !status.stt) { onError('Speech-to-text not available on this server'); return; } - // Downgrade to STT-only when TTS is unavailable if (!status.tts) enableTts = false; } catch (_) { onError('Could not reach voice service'); return; } - // Check microphone permission var permStatus = await Permission.microphone.request(); if (permStatus == PermissionStatus.permanentlyDenied) { onError('Microphone blocked — opening settings'); final opened = await openAppSettings(); if (!opened) return; - // Re-check after user returns from settings permStatus = await Permission.microphone.status; } if (!permStatus.isGranted) { @@ -198,27 +240,11 @@ class VoiceNotifier extends Notifier { await _startListening(); } - /// Exit voice mode, stop all recording and TTS. void exitVoiceMode() { - _amplitudeSubscription?.cancel(); - _amplitudeSubscription = null; - _recorder?.stop(); - _stopVad(); - _player?.stop(); - _ttsQueue.clear(); - _ttsPlaying = false; - _sentenceBuffer = ''; - _lastSeenLength = 0; - _streamComplete = false; - _speechDetected = false; - _onTranscript = null; - _onError = null; - state = const VoiceState(); + _cleanup(); + if (!_disposed) state = const VoiceState(); } - /// Feed streaming assistant content for TTS synthesis. - /// Call from screens with the full [fullContent] string on each update. - /// Set [isComplete] to true when the stream has finished. void feedContent(String fullContent, {required bool isComplete}) { if (!state.voiceModeActive || !_enableTts) return; @@ -237,112 +263,131 @@ class VoiceNotifier extends Notifier { } } + // ── Internal helpers ──────────────────────────────────────────────────────── + + void _cleanup() { + _cancelSubscriptions(); + final handler = _vadHandler; + _vadHandler = null; + handler?.dispose(); + _player?.stop(); + _ttsQueue.clear(); + _ttsPlaying = false; + _sentenceBuffer = ''; + _lastSeenLength = 0; + _streamComplete = false; + _speechDetected = false; + _onTranscript = null; + _onError = null; + } + + void _cancelSubscriptions() { + _vadSpeechStartSub?.cancel(); + _vadSpeechStartSub = null; + _vadSpeechEndSub?.cancel(); + _vadSpeechEndSub = null; + _vadFrameSub?.cancel(); + _vadFrameSub = null; + _vadErrorSub?.cancel(); + _vadErrorSub = null; + } + // ── Internal recording ────────────────────────────────────────────────────── Future _startListening() async { - if (!state.voiceModeActive) return; + if (_disposed || !state.voiceModeActive) return; _speechDetected = false; _speechStartMs = 0; - state = state.copyWith(mode: VoiceMode.recording); - - final dir = _tempDir ?? await getTemporaryDirectory(); - final path = - '${dir.path}/voice_rec_${DateTime.now().millisecondsSinceEpoch}.m4a'; try { - _recorder?.dispose(); - _recorder = AudioRecorder(); - - if (!await _recorder!.hasPermission()) { - _onError?.call('Microphone permission was revoked'); - exitVoiceMode(); - return; - } - - await _recorder!.start( - const RecordConfig(encoder: AudioEncoder.aacLc, sampleRate: 16000), - path: path, - ); - - _amplitudeSubscription?.cancel(); - _amplitudeSubscription = _recorder! - .onAmplitudeChanged(const Duration(milliseconds: 200)) - .listen(_onAmplitude); - - // VAD for speech detection — uses its own AudioRecorder internally await _stopVad(); _vadHandler = VadHandler.create(); + _vadSpeechStartSub = _vadHandler!.onSpeechStart.listen((_) { + if (_disposed || !state.voiceModeActive) return; if (!_speechDetected) { _speechDetected = true; _speechStartMs = DateTime.now().millisecondsSinceEpoch; } }); - _vadSpeechEndSub = _vadHandler!.onSpeechEnd.listen((_) { - if (!state.voiceModeActive) return; + + _vadSpeechEndSub = _vadHandler!.onSpeechEnd.listen((audioSamples) { + if (_disposed || !state.voiceModeActive) return; final now = DateTime.now().millisecondsSinceEpoch; final sinceStart = _speechStartMs > 0 ? now - _speechStartMs : 0; if (_speechDetected && sinceStart >= _vadGraceMs) { - _amplitudeSubscription?.cancel(); - _amplitudeSubscription = null; _stopVad(); - _handleSilence(); + _handleSpeechEnd(audioSamples); } }); + + _vadFrameSub = _vadHandler!.onFrameProcessed.listen((event) { + if (_disposed || !state.voiceModeActive) return; + final frame = event.frame; + if (frame.isEmpty) return; + double sumSq = 0; + for (final s in frame) { + sumSq += s * s; + } + final rms = sqrt(sumSq / frame.length); + final norm = (rms * 4.0).clamp(0.0, 1.0); + if ((norm - state.amplitude).abs() > 0.02) { + state = state.copyWith(amplitude: norm); + } + }); + + _vadErrorSub = _vadHandler!.onError.listen((msg) { + if (_disposed) return; + _onError?.call('VAD error: $msg'); + }); + await _vadHandler!.startListening(model: 'v5'); + + // Only show recording UI after the mic is actually open. + if (!_disposed && state.voiceModeActive) { + state = state.copyWith(mode: VoiceMode.recording, amplitude: 0.0); + } } catch (e) { _onError?.call('Microphone error: $e'); - exitVoiceMode(); + _cleanup(); + if (!_disposed) state = const VoiceState(); } } Future _stopVad() async { - await _vadSpeechStartSub?.cancel(); - _vadSpeechStartSub = null; - await _vadSpeechEndSub?.cancel(); - _vadSpeechEndSub = null; + _cancelSubscriptions(); if (_vadHandler != null) { - await _vadHandler!.dispose(); + final handler = _vadHandler!; _vadHandler = null; + await handler.dispose(); } } - void _onAmplitude(Amplitude event) { - if (!state.voiceModeActive) return; - final db = event.current; - if (db.isNaN || db.isInfinite) return; - final norm = ((db + 60.0) / 60.0).clamp(0.0, 1.0); - if ((norm - state.amplitude).abs() > 0.02) { - state = state.copyWith(amplitude: norm); - } - } - - Future _handleSilence() async { - if (!state.voiceModeActive) return; + Future _handleSpeechEnd(List audioSamples) async { + if (_disposed || !state.voiceModeActive) return; state = state.copyWith(mode: VoiceMode.transcribing); - final path = await _recorder!.stop(); - if (path == null || !state.voiceModeActive) return; - try { - final bytes = await File(path).readAsBytes(); - await File(path).delete().catchError((_) => File(path)); + final wavBytes = encodeWav(audioSamples); - if (!state.voiceModeActive) return; + if (_disposed || !state.voiceModeActive) return; final transcript = await ref.read(voiceRepositoryProvider).transcribe( - bytes, - context: _lastAssistantContent.isNotEmpty ? _lastAssistantContent : null, + wavBytes, + context: + _lastAssistantContent.isNotEmpty ? _lastAssistantContent : null, ); - if (!state.voiceModeActive) return; + if (_disposed || !state.voiceModeActive) return; if (transcript.isEmpty) { _emptyTranscriptCount++; if (_emptyTranscriptCount >= 3) { _onError?.call('No speech detected — tap the mic to try again'); - exitVoiceMode(); + _cleanup(); + if (!_disposed) state = const VoiceState(); + return; } await _startListening(); @@ -350,24 +395,25 @@ class VoiceNotifier extends Notifier { } _emptyTranscriptCount = 0; - // Reset TTS state for this new turn _sentenceBuffer = ''; _lastSeenLength = 0; _streamComplete = false; - if (_enableTts) { + if (_enableTts && !_disposed) { state = state.copyWith(mode: VoiceMode.playing); } await _onTranscript?.call(transcript); - // If TTS is not enabled, loop immediately - if (!_enableTts && state.voiceModeActive) { - await _startListening(); + // In STT-only mode (no TTS), return to idle after transcript is sent. + // The user taps the mic again to record another message. + if (!_enableTts && !_disposed && state.voiceModeActive) { + state = state.copyWith(mode: VoiceMode.idle); } } catch (e) { _onError?.call('Voice error: transcription failed'); - exitVoiceMode(); + _cleanup(); + if (!_disposed) state = const VoiceState(); } } @@ -395,7 +441,7 @@ class VoiceNotifier extends Notifier { try { final wavBytes = await ref.read(voiceRepositoryProvider).synthesise(text); - if (!state.voiceModeActive) return; + if (_disposed || !state.voiceModeActive) return; _ttsQueue.add(wavBytes); if (!_ttsPlaying) _drainTtsQueue(); } catch (_) { @@ -434,6 +480,7 @@ class VoiceNotifier extends Notifier { } void _checkRestartListening() { + if (_disposed) return; if (_streamComplete && _ttsQueue.isEmpty && !_ttsPlaying && diff --git a/lib/screens/briefing/briefing_screen.dart b/lib/screens/briefing/briefing_screen.dart index d2d3080..41678d0 100644 --- a/lib/screens/briefing/briefing_screen.dart +++ b/lib/screens/briefing/briefing_screen.dart @@ -76,7 +76,6 @@ class _BriefingScreenState extends ConsumerState WidgetsBinding.instance.removeObserver(this); _controller.dispose(); _scrollController.dispose(); - ref.read(voiceProvider.notifier).exitVoiceMode(); super.dispose(); } diff --git a/lib/screens/chat/chat_screen.dart b/lib/screens/chat/chat_screen.dart index 89347f9..af341f5 100644 --- a/lib/screens/chat/chat_screen.dart +++ b/lib/screens/chat/chat_screen.dart @@ -68,8 +68,6 @@ class _ChatScreenState extends ConsumerState WidgetsBinding.instance.removeObserver(this); _controller.dispose(); _scrollController.dispose(); - // Exit voice mode if the user navigates away mid-session. - ref.read(voiceProvider.notifier).exitVoiceMode(); super.dispose(); } diff --git a/linux/flutter/generated_plugins.cmake b/linux/flutter/generated_plugins.cmake index 641253b..61405b3 100644 --- a/linux/flutter/generated_plugins.cmake +++ b/linux/flutter/generated_plugins.cmake @@ -10,6 +10,7 @@ list(APPEND FLUTTER_PLUGIN_LIST ) list(APPEND FLUTTER_FFI_PLUGIN_LIST + vad ) set(PLUGIN_BUNDLED_LIBRARIES) diff --git a/macos/Flutter/GeneratedPluginRegistrant.swift b/macos/Flutter/GeneratedPluginRegistrant.swift index 27878f5..74ec380 100644 --- a/macos/Flutter/GeneratedPluginRegistrant.swift +++ b/macos/Flutter/GeneratedPluginRegistrant.swift @@ -11,7 +11,7 @@ import flutter_timezone import just_audio import open_file_mac import package_info_plus -import record_darwin +import record_macos import shared_preferences_foundation import url_launcher_macos @@ -22,7 +22,7 @@ func RegisterGeneratedPlugins(registry: FlutterPluginRegistry) { JustAudioPlugin.register(with: registry.registrar(forPlugin: "JustAudioPlugin")) OpenFilePlugin.register(with: registry.registrar(forPlugin: "OpenFilePlugin")) FPPPackageInfoPlusPlugin.register(with: registry.registrar(forPlugin: "FPPPackageInfoPlusPlugin")) - RecordPlugin.register(with: registry.registrar(forPlugin: "RecordPlugin")) + RecordMacOsPlugin.register(with: registry.registrar(forPlugin: "RecordMacOsPlugin")) SharedPreferencesPlugin.register(with: registry.registrar(forPlugin: "SharedPreferencesPlugin")) UrlLauncherPlugin.register(with: registry.registrar(forPlugin: "UrlLauncherPlugin")) } diff --git a/windows/flutter/generated_plugins.cmake b/windows/flutter/generated_plugins.cmake index f53b4aa..c57f7ba 100644 --- a/windows/flutter/generated_plugins.cmake +++ b/windows/flutter/generated_plugins.cmake @@ -11,6 +11,7 @@ list(APPEND FLUTTER_PLUGIN_LIST ) list(APPEND FLUTTER_FFI_PLUGIN_LIST + vad ) set(PLUGIN_BUNDLED_LIBRARIES)