From 2cb566336be65944cfd4af10e397c90bbcd03266 Mon Sep 17 00:00:00 2001 From: bvandeusen Date: Sun, 5 Apr 2026 14:44:21 -0400 Subject: [PATCH] feat(voice): add VoiceNotifier with recording, silence detection, and streaming TTS Co-Authored-By: Claude Sonnet 4.6 --- lib/providers/voice_provider.dart | 371 ++++++++++++++++++++++++++++++ test/widget_test.dart | 48 ++++ 2 files changed, 419 insertions(+) create mode 100644 lib/providers/voice_provider.dart diff --git a/lib/providers/voice_provider.dart b/lib/providers/voice_provider.dart new file mode 100644 index 0000000..5f1002a --- /dev/null +++ b/lib/providers/voice_provider.dart @@ -0,0 +1,371 @@ +import 'dart:async'; +import 'dart:collection'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:flutter_riverpod/flutter_riverpod.dart'; +import 'package:just_audio/just_audio.dart'; +import 'package:path_provider/path_provider.dart'; +import 'package:permission_handler/permission_handler.dart'; +import 'package:record/record.dart'; + +import 'api_client_provider.dart'; + +// ── Public helpers (also used by tests) ────────────────────────────────────── + +class SentenceResult { + final List sentences; + final String remainder; + const SentenceResult({required this.sentences, required this.remainder}); +} + +/// Extract completed sentences from [text] at `.`, `!`, `?` boundaries. +/// Returns the completed sentences and the unconsumed remainder. +SentenceResult extractSentences(String text) { + final boundary = RegExp(r'[.!?]+(?=\s|$)'); + final sentences = []; + var remaining = text; + RegExpMatch? match; + while ((match = boundary.firstMatch(remaining)) != null) { + final end = match!.end; + final sentence = remaining.substring(0, end).trim(); + if (sentence.isNotEmpty) sentences.add(sentence); + remaining = remaining.substring(end).trimLeft(); + } + return SentenceResult(sentences: sentences, remainder: remaining); +} + +/// Strip markdown formatting before sending text to TTS. +String stripMarkdownForTts(String text) { + return text + .replaceAll(RegExp(r'```[\s\S]*?```'), '') // fenced code blocks + .replaceAllMapped(RegExp(r'`([^`]+)`'), (m) => m[1]!) // inline code + .replaceAll(RegExp(r'#{1,6}\s+'), '') // headings + .replaceAllMapped(RegExp(r'\*\*([^*]+)\*\*'), (m) => m[1]!) // bold + .replaceAllMapped(RegExp(r'\*([^*]+)\*'), (m) => m[1]!) // italic + .replaceAllMapped( + RegExp(r'\[([^\]]+)\]\([^)]+\)'), (m) => m[1]!) // links → text + .replaceAll(RegExp(r'^\s*[-*+]\s+', multiLine: true), '') // list markers + .replaceAll(RegExp(r'\n{2,}'), ' ') // multiple newlines → space + .replaceAll('\n', ' ') + .trim(); +} + +// ── State ───────────────────────────────────────────────────────────────────── + +enum VoiceMode { idle, recording, transcribing, playing } + +class VoiceState { + final VoiceMode mode; + final bool voiceModeActive; + final bool available; + + const VoiceState({ + this.mode = VoiceMode.idle, + this.voiceModeActive = false, + this.available = true, + }); + + VoiceState copyWith({ + VoiceMode? mode, + bool? voiceModeActive, + bool? available, + }) => + VoiceState( + mode: mode ?? this.mode, + voiceModeActive: voiceModeActive ?? this.voiceModeActive, + available: available ?? this.available, + ); +} + +// ── Provider ────────────────────────────────────────────────────────────────── + +final voiceProvider = + NotifierProvider(VoiceNotifier.new); + +// ── Notifier ────────────────────────────────────────────────────────────────── + +class VoiceNotifier extends Notifier { + // Audio I/O + AudioRecorder? _recorder; + AudioPlayer? _player; + StreamSubscription? _amplitudeSubscription; + + // Recording / silence detection + int _recordingStartMs = 0; + int _silenceMs = 0; + static const _silenceThresholdDb = -40.0; + static const _silenceDurationMs = 1500; + static const _minRecordingMs = 300; + + // Voice mode callbacks + Future Function(String transcript)? _onTranscript; + bool _enableTts = false; + + // Streaming TTS state + String _sentenceBuffer = ''; + int _lastSeenLength = 0; + bool _streamComplete = false; + + // TTS playback queue + final _ttsQueue = Queue(); + bool _ttsPlaying = false; + int _ttsCounter = 0; + Directory? _tempDir; + + @override + VoiceState build() { + _recorder = AudioRecorder(); + _player = AudioPlayer(); + ref.onDispose(() { + _amplitudeSubscription?.cancel(); + _recorder?.dispose(); + _player?.dispose(); + }); + return const VoiceState(); + } + + // ── Public API ────────────────────────────────────────────────────────────── + + /// Enter voice mode. Checks server availability and mic permission first. + /// [onTranscript] is called with the transcript when a recording completes. + /// [enableTts] — if true, TTS plays when [feedContent] is called. + /// [onError] — called with a human-readable message on failure. + Future enterVoiceMode({ + required Future Function(String transcript) onTranscript, + bool enableTts = false, + required void Function(String message) onError, + }) async { + if (state.voiceModeActive) { + exitVoiceMode(); + return; + } + + // Check server availability + try { + final status = await ref.read(voiceRepositoryProvider).checkStatus(); + if (!status.fullyAvailable) { + onError('Voice not available on this server'); + return; + } + } catch (_) { + onError('Voice not available on this server'); + return; + } + + // Check microphone permission + final permStatus = await Permission.microphone.request(); + if (permStatus == PermissionStatus.denied || + permStatus == PermissionStatus.permanentlyDenied) { + onError('Microphone permission required'); + if (permStatus == PermissionStatus.permanentlyDenied) { + await openAppSettings(); + } + return; + } + + _onTranscript = onTranscript; + _enableTts = enableTts; + _tempDir = await getTemporaryDirectory(); + + state = state.copyWith(voiceModeActive: true, available: true); + await _startListening(); + } + + /// Exit voice mode, stop all recording and TTS. + void exitVoiceMode() { + _amplitudeSubscription?.cancel(); + _amplitudeSubscription = null; + _recorder?.stop(); + _player?.stop(); + _ttsQueue.clear(); + _ttsPlaying = false; + _sentenceBuffer = ''; + _lastSeenLength = 0; + _streamComplete = false; + _onTranscript = null; + state = const VoiceState(); + } + + /// Feed streaming assistant content for TTS synthesis. + /// Call from screens with the full [fullContent] string on each update. + /// Set [isComplete] to true when the stream has finished. + void feedContent(String fullContent, {required bool isComplete}) { + if (!state.voiceModeActive || !_enableTts) return; + + final delta = fullContent.length > _lastSeenLength + ? fullContent.substring(_lastSeenLength) + : ''; + _lastSeenLength = fullContent.length; + _sentenceBuffer += delta; + + _dispatchSentences(flush: isComplete); + + if (isComplete) { + _streamComplete = true; + _checkRestartListening(); + } + } + + // ── Internal recording ────────────────────────────────────────────────────── + + Future _startListening() async { + if (!state.voiceModeActive) return; + + _silenceMs = 0; + _recordingStartMs = DateTime.now().millisecondsSinceEpoch; + state = state.copyWith(mode: VoiceMode.recording); + + final dir = _tempDir ?? await getTemporaryDirectory(); + final path = + '${dir.path}/voice_rec_${DateTime.now().millisecondsSinceEpoch}.webm'; + + await _recorder!.start( + const RecordConfig(encoder: AudioEncoder.opus, sampleRate: 16000), + path: path, + ); + + _amplitudeSubscription?.cancel(); + _amplitudeSubscription = _recorder! + .onAmplitudeChanged(const Duration(milliseconds: 200)) + .listen(_onAmplitude); + } + + void _onAmplitude(Amplitude event) { + if (!state.voiceModeActive) return; + + final elapsed = + DateTime.now().millisecondsSinceEpoch - _recordingStartMs; + if (elapsed < _minRecordingMs) return; + + if (event.current < _silenceThresholdDb) { + _silenceMs += 200; + if (_silenceMs >= _silenceDurationMs) { + _amplitudeSubscription?.cancel(); + _amplitudeSubscription = null; + _handleSilence(); + } + } else { + _silenceMs = 0; + } + } + + Future _handleSilence() async { + if (!state.voiceModeActive) return; + state = state.copyWith(mode: VoiceMode.transcribing); + + final path = await _recorder!.stop(); + if (path == null || !state.voiceModeActive) return; + + try { + final bytes = await File(path).readAsBytes(); + await File(path).delete().catchError((_) => File(path)); + + if (!state.voiceModeActive) return; + + final transcript = + await ref.read(voiceRepositoryProvider).transcribe(bytes); + + if (!state.voiceModeActive) return; + + if (transcript.isEmpty) { + // Empty transcript — restart silently + await _startListening(); + return; + } + + // Reset TTS state for this new turn + _sentenceBuffer = ''; + _lastSeenLength = 0; + _streamComplete = false; + + if (_enableTts) { + state = state.copyWith(mode: VoiceMode.playing); + } + + await _onTranscript?.call(transcript); + + // If TTS is not enabled, loop immediately + if (!_enableTts && state.voiceModeActive) { + await _startListening(); + } + } catch (_) { + // Network/API error — exit voice mode + exitVoiceMode(); + } + } + + // ── Internal TTS ──────────────────────────────────────────────────────────── + + void _dispatchSentences({required bool flush}) { + final result = extractSentences(_sentenceBuffer); + _sentenceBuffer = flush ? '' : result.remainder; + + for (final sentence in result.sentences) { + _enqueueSentence(sentence); + } + if (flush && result.remainder.trim().length >= 3) { + _enqueueSentence(result.remainder.trim()); + } + } + + void _enqueueSentence(String sentence) { + final stripped = stripMarkdownForTts(sentence); + if (stripped.length < 3) return; + _synthesiseSentence(stripped); + } + + Future _synthesiseSentence(String text) async { + try { + final wavBytes = + await ref.read(voiceRepositoryProvider).synthesise(text); + if (!state.voiceModeActive) return; + _ttsQueue.add(wavBytes); + if (!_ttsPlaying) _drainTtsQueue(); + } catch (_) { + // Skip failed sentence — TTS errors are non-fatal + } + } + + Future _drainTtsQueue() async { + if (_ttsPlaying) return; + _ttsPlaying = true; + final dir = _tempDir ?? await getTemporaryDirectory(); + + try { + while (_ttsQueue.isNotEmpty && state.voiceModeActive) { + final wavBytes = _ttsQueue.removeFirst(); + final path = '${dir.path}/tts_${_ttsCounter++}.wav'; + final file = File(path); + await file.writeAsBytes(wavBytes); + + try { + await _player!.setFilePath(path); + await _player!.play(); + await _player!.processingStateStream.firstWhere( + (s) => + s == ProcessingState.completed || s == ProcessingState.idle, + ); + } finally { + await file.delete().catchError((_) => file); + } + } + } finally { + _ttsPlaying = false; + } + + _checkRestartListening(); + } + + void _checkRestartListening() { + if (_streamComplete && + _ttsQueue.isEmpty && + !_ttsPlaying && + state.voiceModeActive) { + _streamComplete = false; + _lastSeenLength = 0; + _sentenceBuffer = ''; + _startListening(); + } + } +} diff --git a/test/widget_test.dart b/test/widget_test.dart index bdfc282..667fdd2 100644 --- a/test/widget_test.dart +++ b/test/widget_test.dart @@ -1,4 +1,5 @@ import 'package:fabled_app/data/api/voice_api.dart'; +import 'package:fabled_app/providers/voice_provider.dart'; import 'package:fabled_app/data/models/knowledge_item.dart'; import 'package:fabled_app/data/models/note.dart'; import 'package:flutter_test/flutter_test.dart'; @@ -98,4 +99,51 @@ void main() { expect(status.fullyAvailable, isFalse); }); }); + + group('VoiceNotifier sentence extraction', () { + test('extracts complete sentences at full stops', () { + final result = extractSentences('Hello world. How are you? I am fine!'); + expect(result.sentences, + equals(['Hello world.', 'How are you?', 'I am fine!'])); + expect(result.remainder, equals('')); + }); + + test('leaves incomplete fragment in remainder', () { + final result = extractSentences('Hello world. Incomplete'); + expect(result.sentences, equals(['Hello world.'])); + expect(result.remainder, equals('Incomplete')); + }); + + test('returns empty sentences and full text when no boundary', () { + final result = extractSentences('No boundary here'); + expect(result.sentences, isEmpty); + expect(result.remainder, equals('No boundary here')); + }); + }); + + group('VoiceNotifier markdown stripping', () { + test('strips code fences', () { + expect(stripMarkdownForTts('Before\n```dart\ncode\n```\nAfter'), + equals('Before After')); + }); + + test('strips bold and italic markers', () { + expect(stripMarkdownForTts('**bold** and *italic*'), + equals('bold and italic')); + }); + + test('strips headers', () { + expect(stripMarkdownForTts('## Section title'), equals('Section title')); + }); + + test('keeps link text, removes URL', () { + expect(stripMarkdownForTts('[click here](https://example.com)'), + equals('click here')); + }); + + test('strips list markers', () { + expect(stripMarkdownForTts('- item one\n- item two'), + equals('item one item two')); + }); + }); }