import 'dart:async'; import 'dart:collection'; import 'dart:io'; import 'dart:math'; import 'dart:typed_data'; import 'package:flutter_riverpod/flutter_riverpod.dart'; import 'package:just_audio/just_audio.dart'; import 'package:path_provider/path_provider.dart'; import 'package:permission_handler/permission_handler.dart'; import 'package:vad/vad.dart'; import 'api_client_provider.dart'; // ── Public helpers (also used by tests) ────────────────────────────────────── class SentenceResult { final List sentences; final String remainder; const SentenceResult({required this.sentences, required this.remainder}); } /// Extract completed sentences from [text] at `.`, `!`, `?` boundaries. /// Returns the completed sentences and the unconsumed remainder. SentenceResult extractSentences(String text) { final boundary = RegExp(r'[.!?]+(?=\s|$)'); final sentences = []; var remaining = text; RegExpMatch? match; while ((match = boundary.firstMatch(remaining)) != null) { final end = match!.end; final sentence = remaining.substring(0, end).trim(); if (sentence.isNotEmpty) sentences.add(sentence); remaining = remaining.substring(end).trimLeft(); } return SentenceResult(sentences: sentences, remainder: remaining); } /// Strip markdown formatting before sending text to TTS. String stripMarkdownForTts(String text) { return text .replaceAll(RegExp(r'```[\s\S]*?```'), '') // fenced code blocks .replaceAllMapped(RegExp(r'`([^`]+)`'), (m) => m[1]!) // inline code .replaceAll(RegExp(r'#{1,6}\s+'), '') // headings .replaceAllMapped(RegExp(r'\*\*([^*]+)\*\*'), (m) => m[1]!) // bold .replaceAllMapped(RegExp(r'\*([^*]+)\*'), (m) => m[1]!) // italic .replaceAllMapped( RegExp(r'\[([^\]]+)\]\([^)]+\)'), (m) => m[1]!) // links → text .replaceAll(RegExp(r'^\s*[-*+]\s+', multiLine: true), '') // list markers .replaceAll(RegExp(r'\n{2,}'), ' ') // multiple newlines → space .replaceAll('\n', ' ') .trim(); } /// Encode float PCM samples (-1..1) as a 16-bit mono WAV at 16 kHz. Uint8List encodeWav(List samples, {int sampleRate = 16000}) { final numSamples = samples.length; final dataSize = numSamples * 2; final fileSize = 44 + dataSize; final buf = ByteData(fileSize); // RIFF header buf.setUint8(0, 0x52); // R buf.setUint8(1, 0x49); // I buf.setUint8(2, 0x46); // F buf.setUint8(3, 0x46); // F buf.setUint32(4, fileSize - 8, Endian.little); buf.setUint8(8, 0x57); // W buf.setUint8(9, 0x41); // A buf.setUint8(10, 0x56); // V buf.setUint8(11, 0x45); // E // fmt chunk buf.setUint8(12, 0x66); // f buf.setUint8(13, 0x6D); // m buf.setUint8(14, 0x74); // t buf.setUint8(15, 0x20); // (space) buf.setUint32(16, 16, Endian.little); // chunk size buf.setUint16(20, 1, Endian.little); // PCM format buf.setUint16(22, 1, Endian.little); // mono buf.setUint32(24, sampleRate, Endian.little); buf.setUint32(28, sampleRate * 2, Endian.little); // byte rate buf.setUint16(32, 2, Endian.little); // block align buf.setUint16(34, 16, Endian.little); // bits per sample // data chunk buf.setUint8(36, 0x64); // d buf.setUint8(37, 0x61); // a buf.setUint8(38, 0x74); // t buf.setUint8(39, 0x61); // a buf.setUint32(40, dataSize, Endian.little); for (var i = 0; i < numSamples; i++) { final clamped = samples[i].clamp(-1.0, 1.0); final int16 = (clamped * 32767).round().clamp(-32768, 32767); buf.setInt16(44 + i * 2, int16, Endian.little); } return buf.buffer.asUint8List(); } // ── State ───────────────────────────────────────────────────────────────────── enum VoiceMode { idle, recording, transcribing, playing } class VoiceState { final VoiceMode mode; final bool voiceModeActive; final bool available; /// Normalized mic amplitude 0.0–1.0 while recording. final double amplitude; const VoiceState({ this.mode = VoiceMode.idle, this.voiceModeActive = false, this.available = true, this.amplitude = 0.0, }); VoiceState copyWith({ VoiceMode? mode, bool? voiceModeActive, bool? available, double? amplitude, }) => VoiceState( mode: mode ?? this.mode, voiceModeActive: voiceModeActive ?? this.voiceModeActive, available: available ?? this.available, amplitude: amplitude ?? this.amplitude, ); } // ── Provider ────────────────────────────────────────────────────────────────── final voiceProvider = NotifierProvider.autoDispose(VoiceNotifier.new); // ── Notifier ────────────────────────────────────────────────────────────────── class VoiceNotifier extends Notifier { // Audio playback AudioPlayer? _player; // VAD — sole owner of the microphone VadHandler? _vadHandler; StreamSubscription? _vadSpeechStartSub; StreamSubscription>? _vadSpeechEndSub; StreamSubscription<({double isSpeech, double notSpeech, List frame})>? _vadFrameSub; StreamSubscription? _vadErrorSub; bool _speechDetected = false; int _speechStartMs = 0; static const _vadGraceMs = 1500; bool _disposed = false; // Voice mode callbacks Future Function(String transcript)? _onTranscript; void Function(String message)? _onError; bool _enableTts = false; // Streaming TTS state String _sentenceBuffer = ''; int _lastSeenLength = 0; bool _streamComplete = false; // Whisper context hint String _lastAssistantContent = ''; // Empty transcript counter int _emptyTranscriptCount = 0; // TTS playback queue final _ttsQueue = Queue(); bool _ttsPlaying = false; int _ttsCounter = 0; Directory? _tempDir; @override VoiceState build() { _disposed = false; _player = AudioPlayer(); ref.onDispose(() { _disposed = true; _cancelSubscriptions(); _vadHandler?.dispose(); _vadHandler = null; _player?.dispose(); _player = null; }); return const VoiceState(); } // ── Public API ────────────────────────────────────────────────────────────── Future enterVoiceMode({ required Future Function(String transcript) onTranscript, bool enableTts = false, required void Function(String message) onError, }) async { if (state.voiceModeActive) { if (state.mode == VoiceMode.recording && !_speechDetected) { onError('No speech detected'); } exitVoiceMode(); return; } try { final status = await ref.read(voiceRepositoryProvider).checkStatus(); if (!status.enabled || !status.stt) { onError('Speech-to-text not available on this server'); return; } if (!status.tts) enableTts = false; } catch (_) { onError('Could not reach voice service'); return; } var permStatus = await Permission.microphone.request(); if (permStatus == PermissionStatus.permanentlyDenied) { onError('Microphone blocked — opening settings'); final opened = await openAppSettings(); if (!opened) return; permStatus = await Permission.microphone.status; } if (!permStatus.isGranted) { onError('Microphone permission required'); return; } _onTranscript = onTranscript; _onError = onError; _enableTts = enableTts; _emptyTranscriptCount = 0; _tempDir = await getTemporaryDirectory(); state = state.copyWith(voiceModeActive: true, available: true); await _startListening(); } void exitVoiceMode() { _cleanup(); if (!_disposed) state = const VoiceState(); } void feedContent(String fullContent, {required bool isComplete}) { if (!state.voiceModeActive || !_enableTts) return; final delta = fullContent.length > _lastSeenLength ? fullContent.substring(_lastSeenLength) : ''; _lastSeenLength = fullContent.length; _sentenceBuffer += delta; _dispatchSentences(flush: isComplete); if (isComplete) { _lastAssistantContent = fullContent; _streamComplete = true; _checkRestartListening(); } } // ── Internal helpers ──────────────────────────────────────────────────────── void _cleanup() { _cancelSubscriptions(); final handler = _vadHandler; _vadHandler = null; handler?.dispose(); _player?.stop(); _ttsQueue.clear(); _ttsPlaying = false; _sentenceBuffer = ''; _lastSeenLength = 0; _streamComplete = false; _speechDetected = false; _onTranscript = null; _onError = null; } void _cancelSubscriptions() { _vadSpeechStartSub?.cancel(); _vadSpeechStartSub = null; _vadSpeechEndSub?.cancel(); _vadSpeechEndSub = null; _vadFrameSub?.cancel(); _vadFrameSub = null; _vadErrorSub?.cancel(); _vadErrorSub = null; } // ── Internal recording ────────────────────────────────────────────────────── Future _startListening() async { if (_disposed || !state.voiceModeActive) return; _speechDetected = false; _speechStartMs = 0; try { await _stopVad(); _vadHandler = VadHandler.create(); _vadSpeechStartSub = _vadHandler!.onSpeechStart.listen((_) { if (_disposed || !state.voiceModeActive) return; if (!_speechDetected) { _speechDetected = true; _speechStartMs = DateTime.now().millisecondsSinceEpoch; } }); _vadSpeechEndSub = _vadHandler!.onSpeechEnd.listen((audioSamples) { if (_disposed || !state.voiceModeActive) return; final now = DateTime.now().millisecondsSinceEpoch; final sinceStart = _speechStartMs > 0 ? now - _speechStartMs : 0; if (_speechDetected && sinceStart >= _vadGraceMs) { _stopVad(); _handleSpeechEnd(audioSamples); } }); _vadFrameSub = _vadHandler!.onFrameProcessed.listen((event) { if (_disposed || !state.voiceModeActive) return; final frame = event.frame; if (frame.isEmpty) return; double sumSq = 0; for (final s in frame) { sumSq += s * s; } final rms = sqrt(sumSq / frame.length); final norm = (rms * 4.0).clamp(0.0, 1.0); if ((norm - state.amplitude).abs() > 0.02) { state = state.copyWith(amplitude: norm); } }); _vadErrorSub = _vadHandler!.onError.listen((msg) { if (_disposed) return; _onError?.call('VAD error: $msg'); }); await _vadHandler!.startListening(model: 'v5'); // Only show recording UI after the mic is actually open. if (!_disposed && state.voiceModeActive) { state = state.copyWith(mode: VoiceMode.recording, amplitude: 0.0); } } catch (e) { _onError?.call('Microphone error: $e'); _cleanup(); if (!_disposed) state = const VoiceState(); } } Future _stopVad() async { _cancelSubscriptions(); if (_vadHandler != null) { final handler = _vadHandler!; _vadHandler = null; await handler.dispose(); } } Future _handleSpeechEnd(List audioSamples) async { if (_disposed || !state.voiceModeActive) return; state = state.copyWith(mode: VoiceMode.transcribing); try { final wavBytes = encodeWav(audioSamples); if (_disposed || !state.voiceModeActive) return; final transcript = await ref.read(voiceRepositoryProvider).transcribe( wavBytes, context: _lastAssistantContent.isNotEmpty ? _lastAssistantContent : null, ); if (_disposed || !state.voiceModeActive) return; if (transcript.isEmpty) { _emptyTranscriptCount++; if (_emptyTranscriptCount >= 3) { _onError?.call('No speech detected — tap the mic to try again'); _cleanup(); if (!_disposed) state = const VoiceState(); return; } await _startListening(); return; } _emptyTranscriptCount = 0; _sentenceBuffer = ''; _lastSeenLength = 0; _streamComplete = false; if (_enableTts && !_disposed) { state = state.copyWith(mode: VoiceMode.playing); } await _onTranscript?.call(transcript); // In STT-only mode (no TTS), return to idle after transcript is sent. // The user taps the mic again to record another message. if (!_enableTts && !_disposed && state.voiceModeActive) { state = state.copyWith(mode: VoiceMode.idle); } } catch (e) { _onError?.call('Voice error: transcription failed'); _cleanup(); if (!_disposed) state = const VoiceState(); } } // ── Internal TTS ──────────────────────────────────────────────────────────── void _dispatchSentences({required bool flush}) { final result = extractSentences(_sentenceBuffer); _sentenceBuffer = flush ? '' : result.remainder; for (final sentence in result.sentences) { _enqueueSentence(sentence); } if (flush && result.remainder.trim().length >= 3) { _enqueueSentence(result.remainder.trim()); } } void _enqueueSentence(String sentence) { final stripped = stripMarkdownForTts(sentence); if (stripped.length < 3) return; _synthesiseSentence(stripped); } Future _synthesiseSentence(String text) async { try { final wavBytes = await ref.read(voiceRepositoryProvider).synthesise(text); if (_disposed || !state.voiceModeActive) return; _ttsQueue.add(wavBytes); if (!_ttsPlaying) _drainTtsQueue(); } catch (_) { // Skip failed sentence — TTS errors are non-fatal } } Future _drainTtsQueue() async { if (_ttsPlaying) return; _ttsPlaying = true; final dir = _tempDir ?? await getTemporaryDirectory(); try { while (_ttsQueue.isNotEmpty && state.voiceModeActive) { final wavBytes = _ttsQueue.removeFirst(); final path = '${dir.path}/tts_${_ttsCounter++}.wav'; final file = File(path); await file.writeAsBytes(wavBytes); try { await _player!.setFilePath(path); await _player!.play(); await _player!.processingStateStream.firstWhere( (s) => s == ProcessingState.completed || s == ProcessingState.idle, ); } finally { await file.delete().catchError((_) => file); } } } finally { _ttsPlaying = false; } _checkRestartListening(); } void _checkRestartListening() { if (_disposed) return; if (_streamComplete && _ttsQueue.isEmpty && !_ttsPlaying && state.voiceModeActive) { _streamComplete = false; _lastSeenLength = 0; _sentenceBuffer = ''; _startListening(); } } }