import 'dart:async'; import 'dart:collection'; import 'dart:io'; import 'dart:typed_data'; import 'package:flutter_riverpod/flutter_riverpod.dart'; import 'package:just_audio/just_audio.dart'; import 'package:path_provider/path_provider.dart'; import 'package:permission_handler/permission_handler.dart'; import 'package:record/record.dart'; import 'api_client_provider.dart'; // ── Public helpers (also used by tests) ────────────────────────────────────── class SentenceResult { final List sentences; final String remainder; const SentenceResult({required this.sentences, required this.remainder}); } /// Extract completed sentences from [text] at `.`, `!`, `?` boundaries. /// Returns the completed sentences and the unconsumed remainder. SentenceResult extractSentences(String text) { final boundary = RegExp(r'[.!?]+(?=\s|$)'); final sentences = []; var remaining = text; RegExpMatch? match; while ((match = boundary.firstMatch(remaining)) != null) { final end = match!.end; final sentence = remaining.substring(0, end).trim(); if (sentence.isNotEmpty) sentences.add(sentence); remaining = remaining.substring(end).trimLeft(); } return SentenceResult(sentences: sentences, remainder: remaining); } /// Strip markdown formatting before sending text to TTS. String stripMarkdownForTts(String text) { return text .replaceAll(RegExp(r'```[\s\S]*?```'), '') // fenced code blocks .replaceAllMapped(RegExp(r'`([^`]+)`'), (m) => m[1]!) // inline code .replaceAll(RegExp(r'#{1,6}\s+'), '') // headings .replaceAllMapped(RegExp(r'\*\*([^*]+)\*\*'), (m) => m[1]!) // bold .replaceAllMapped(RegExp(r'\*([^*]+)\*'), (m) => m[1]!) // italic .replaceAllMapped( RegExp(r'\[([^\]]+)\]\([^)]+\)'), (m) => m[1]!) // links → text .replaceAll(RegExp(r'^\s*[-*+]\s+', multiLine: true), '') // list markers .replaceAll(RegExp(r'\n{2,}'), ' ') // multiple newlines → space .replaceAll('\n', ' ') .trim(); } // ── State ───────────────────────────────────────────────────────────────────── enum VoiceMode { idle, recording, transcribing, playing } class VoiceState { final VoiceMode mode; final bool voiceModeActive; final bool available; const VoiceState({ this.mode = VoiceMode.idle, this.voiceModeActive = false, this.available = true, }); VoiceState copyWith({ VoiceMode? mode, bool? voiceModeActive, bool? available, }) => VoiceState( mode: mode ?? this.mode, voiceModeActive: voiceModeActive ?? this.voiceModeActive, available: available ?? this.available, ); } // ── Provider ────────────────────────────────────────────────────────────────── final voiceProvider = NotifierProvider(VoiceNotifier.new); // ── Notifier ────────────────────────────────────────────────────────────────── class VoiceNotifier extends Notifier { // Audio I/O AudioRecorder? _recorder; AudioPlayer? _player; StreamSubscription? _amplitudeSubscription; // Recording / silence detection int _recordingStartMs = 0; int _silenceMs = 0; static const _silenceThresholdDb = -40.0; static const _silenceDurationMs = 1500; static const _minRecordingMs = 300; // Voice mode callbacks Future Function(String transcript)? _onTranscript; void Function(String message)? _onError; bool _enableTts = false; // Streaming TTS state String _sentenceBuffer = ''; int _lastSeenLength = 0; bool _streamComplete = false; // Last complete assistant response — passed to Whisper as initial_prompt // to reduce STT mishearings of domain-specific words. String _lastAssistantContent = ''; // TTS playback queue final _ttsQueue = Queue(); bool _ttsPlaying = false; int _ttsCounter = 0; Directory? _tempDir; @override VoiceState build() { _recorder = AudioRecorder(); _player = AudioPlayer(); ref.onDispose(() { _amplitudeSubscription?.cancel(); _recorder?.dispose(); _player?.dispose(); }); return const VoiceState(); } // ── Public API ────────────────────────────────────────────────────────────── /// Enter voice mode. Checks server availability and mic permission first. /// [onTranscript] is called with the transcript when a recording completes. /// [enableTts] — if true, TTS plays when [feedContent] is called. /// [onError] — called with a human-readable message on failure. Future enterVoiceMode({ required Future Function(String transcript) onTranscript, bool enableTts = false, required void Function(String message) onError, }) async { if (state.voiceModeActive) { exitVoiceMode(); return; } // Check server availability try { final status = await ref.read(voiceRepositoryProvider).checkStatus(); if (!status.fullyAvailable) { onError('Voice not available on this server'); return; } } catch (_) { onError('Voice not available on this server'); return; } // Check microphone permission final permStatus = await Permission.microphone.request(); if (permStatus == PermissionStatus.denied || permStatus == PermissionStatus.permanentlyDenied) { onError('Microphone permission required'); if (permStatus == PermissionStatus.permanentlyDenied) { await openAppSettings(); } return; } _onTranscript = onTranscript; _onError = onError; _enableTts = enableTts; _tempDir = await getTemporaryDirectory(); state = state.copyWith(voiceModeActive: true, available: true); await _startListening(); } /// Exit voice mode, stop all recording and TTS. void exitVoiceMode() { _amplitudeSubscription?.cancel(); _amplitudeSubscription = null; _recorder?.stop(); _player?.stop(); _ttsQueue.clear(); _ttsPlaying = false; _sentenceBuffer = ''; _lastSeenLength = 0; _streamComplete = false; _onTranscript = null; _onError = null; state = const VoiceState(); } /// Feed streaming assistant content for TTS synthesis. /// Call from screens with the full [fullContent] string on each update. /// Set [isComplete] to true when the stream has finished. void feedContent(String fullContent, {required bool isComplete}) { if (!state.voiceModeActive || !_enableTts) return; final delta = fullContent.length > _lastSeenLength ? fullContent.substring(_lastSeenLength) : ''; _lastSeenLength = fullContent.length; _sentenceBuffer += delta; _dispatchSentences(flush: isComplete); if (isComplete) { _lastAssistantContent = fullContent; _streamComplete = true; _checkRestartListening(); } } // ── Internal recording ────────────────────────────────────────────────────── Future _startListening() async { if (!state.voiceModeActive) return; _silenceMs = 0; _recordingStartMs = DateTime.now().millisecondsSinceEpoch; state = state.copyWith(mode: VoiceMode.recording); final dir = _tempDir ?? await getTemporaryDirectory(); // AAC/M4A is reliably supported on Android (unlike WebM/Opus which can // produce OGG bytes in a .webm file, confusing server-side decoders). final path = '${dir.path}/voice_rec_${DateTime.now().millisecondsSinceEpoch}.m4a'; try { await _recorder!.start( const RecordConfig(encoder: AudioEncoder.aacLc, sampleRate: 16000), path: path, ); _amplitudeSubscription?.cancel(); _amplitudeSubscription = _recorder! .onAmplitudeChanged(const Duration(milliseconds: 200)) .listen(_onAmplitude); } catch (e) { _onError?.call('Microphone error: could not start recording'); exitVoiceMode(); } } void _onAmplitude(Amplitude event) { if (!state.voiceModeActive) return; final elapsed = DateTime.now().millisecondsSinceEpoch - _recordingStartMs; if (elapsed < _minRecordingMs) return; final db = event.current; // Guard against NaN / ±Infinity which can arrive on some Android devices // when the recorder is initialising. Treat invalid readings as silence. final isSilent = db.isNaN || db.isInfinite || db < _silenceThresholdDb; if (isSilent) { _silenceMs += 200; if (_silenceMs >= _silenceDurationMs) { _amplitudeSubscription?.cancel(); _amplitudeSubscription = null; _handleSilence(); } } else { _silenceMs = 0; } } Future _handleSilence() async { if (!state.voiceModeActive) return; state = state.copyWith(mode: VoiceMode.transcribing); final path = await _recorder!.stop(); if (path == null || !state.voiceModeActive) return; try { final bytes = await File(path).readAsBytes(); await File(path).delete().catchError((_) => File(path)); if (!state.voiceModeActive) return; final transcript = await ref.read(voiceRepositoryProvider).transcribe( bytes, context: _lastAssistantContent.isNotEmpty ? _lastAssistantContent : null, ); if (!state.voiceModeActive) return; if (transcript.isEmpty) { // Empty transcript — restart silently await _startListening(); return; } // Reset TTS state for this new turn _sentenceBuffer = ''; _lastSeenLength = 0; _streamComplete = false; if (_enableTts) { state = state.copyWith(mode: VoiceMode.playing); } await _onTranscript?.call(transcript); // If TTS is not enabled, loop immediately if (!_enableTts && state.voiceModeActive) { await _startListening(); } } catch (e) { _onError?.call('Voice error: transcription failed'); exitVoiceMode(); } } // ── Internal TTS ──────────────────────────────────────────────────────────── void _dispatchSentences({required bool flush}) { final result = extractSentences(_sentenceBuffer); _sentenceBuffer = flush ? '' : result.remainder; for (final sentence in result.sentences) { _enqueueSentence(sentence); } if (flush && result.remainder.trim().length >= 3) { _enqueueSentence(result.remainder.trim()); } } void _enqueueSentence(String sentence) { final stripped = stripMarkdownForTts(sentence); if (stripped.length < 3) return; _synthesiseSentence(stripped); } Future _synthesiseSentence(String text) async { try { final wavBytes = await ref.read(voiceRepositoryProvider).synthesise(text); if (!state.voiceModeActive) return; _ttsQueue.add(wavBytes); if (!_ttsPlaying) _drainTtsQueue(); } catch (_) { // Skip failed sentence — TTS errors are non-fatal } } Future _drainTtsQueue() async { if (_ttsPlaying) return; _ttsPlaying = true; final dir = _tempDir ?? await getTemporaryDirectory(); try { while (_ttsQueue.isNotEmpty && state.voiceModeActive) { final wavBytes = _ttsQueue.removeFirst(); final path = '${dir.path}/tts_${_ttsCounter++}.wav'; final file = File(path); await file.writeAsBytes(wavBytes); try { await _player!.setFilePath(path); await _player!.play(); await _player!.processingStateStream.firstWhere( (s) => s == ProcessingState.completed || s == ProcessingState.idle, ); } finally { await file.delete().catchError((_) => file); } } } finally { _ttsPlaying = false; } _checkRestartListening(); } void _checkRestartListening() { if (_streamComplete && _ttsQueue.isEmpty && !_ttsPlaying && state.voiceModeActive) { _streamComplete = false; _lastSeenLength = 0; _sentenceBuffer = ''; _startListening(); } } }