feat(voice): add VoiceNotifier with recording, silence detection, and streaming TTS

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-04-05 14:44:21 -04:00
parent 6e33d74178
commit 2cb566336b
2 changed files with 419 additions and 0 deletions
@@ -0,0 +1,371 @@
+import 'dart:async';
+import 'dart:collection';
+import 'dart:io';
+import 'dart:typed_data';
+
+import 'package:flutter_riverpod/flutter_riverpod.dart';
+import 'package:just_audio/just_audio.dart';
+import 'package:path_provider/path_provider.dart';
+import 'package:permission_handler/permission_handler.dart';
+import 'package:record/record.dart';
+
+import 'api_client_provider.dart';
+
+// ── Public helpers (also used by tests) ──────────────────────────────────────
+
+class SentenceResult {
+  final List<String> sentences;
+  final String remainder;
+  const SentenceResult({required this.sentences, required this.remainder});
+}
+
+/// Extract completed sentences from [text] at `.`, `!`, `?` boundaries.
+/// Returns the completed sentences and the unconsumed remainder.
+SentenceResult extractSentences(String text) {
+  final boundary = RegExp(r'[.!?]+(?=\s|$)');
+  final sentences = <String>[];
+  var remaining = text;
+  RegExpMatch? match;
+  while ((match = boundary.firstMatch(remaining)) != null) {
+    final end = match!.end;
+    final sentence = remaining.substring(0, end).trim();
+    if (sentence.isNotEmpty) sentences.add(sentence);
+    remaining = remaining.substring(end).trimLeft();
+  }
+  return SentenceResult(sentences: sentences, remainder: remaining);
+}
+
+/// Strip markdown formatting before sending text to TTS.
+String stripMarkdownForTts(String text) {
+  return text
+      .replaceAll(RegExp(r'```[\s\S]*?```'), '') // fenced code blocks
+      .replaceAllMapped(RegExp(r'`([^`]+)`'), (m) => m[1]!) // inline code
+      .replaceAll(RegExp(r'#{1,6}\s+'), '') // headings
+      .replaceAllMapped(RegExp(r'\*\*([^*]+)\*\*'), (m) => m[1]!) // bold
+      .replaceAllMapped(RegExp(r'\*([^*]+)\*'), (m) => m[1]!) // italic
+      .replaceAllMapped(
+          RegExp(r'\[([^\]]+)\]\([^)]+\)'), (m) => m[1]!) // links → text
+      .replaceAll(RegExp(r'^\s*[-*+]\s+', multiLine: true), '') // list markers
+      .replaceAll(RegExp(r'\n{2,}'), ' ') // multiple newlines → space
+      .replaceAll('\n', ' ')
+      .trim();
+}
+
+// ── State ─────────────────────────────────────────────────────────────────────
+
+enum VoiceMode { idle, recording, transcribing, playing }
+
+class VoiceState {
+  final VoiceMode mode;
+  final bool voiceModeActive;
+  final bool available;
+
+  const VoiceState({
+    this.mode = VoiceMode.idle,
+    this.voiceModeActive = false,
+    this.available = true,
+  });
+
+  VoiceState copyWith({
+    VoiceMode? mode,
+    bool? voiceModeActive,
+    bool? available,
+  }) =>
+      VoiceState(
+        mode: mode ?? this.mode,
+        voiceModeActive: voiceModeActive ?? this.voiceModeActive,
+        available: available ?? this.available,
+      );
+}
+
+// ── Provider ──────────────────────────────────────────────────────────────────
+
+final voiceProvider =
+    NotifierProvider<VoiceNotifier, VoiceState>(VoiceNotifier.new);
+
+// ── Notifier ──────────────────────────────────────────────────────────────────
+
+class VoiceNotifier extends Notifier<VoiceState> {
+  // Audio I/O
+  AudioRecorder? _recorder;
+  AudioPlayer? _player;
+  StreamSubscription<Amplitude>? _amplitudeSubscription;
+
+  // Recording / silence detection
+  int _recordingStartMs = 0;
+  int _silenceMs = 0;
+  static const _silenceThresholdDb = -40.0;
+  static const _silenceDurationMs = 1500;
+  static const _minRecordingMs = 300;
+
+  // Voice mode callbacks
+  Future<void> Function(String transcript)? _onTranscript;
+  bool _enableTts = false;
+
+  // Streaming TTS state
+  String _sentenceBuffer = '';
+  int _lastSeenLength = 0;
+  bool _streamComplete = false;
+
+  // TTS playback queue
+  final _ttsQueue = Queue<Uint8List>();
+  bool _ttsPlaying = false;
+  int _ttsCounter = 0;
+  Directory? _tempDir;
+
+  @override
+  VoiceState build() {
+    _recorder = AudioRecorder();
+    _player = AudioPlayer();
+    ref.onDispose(() {
+      _amplitudeSubscription?.cancel();
+      _recorder?.dispose();
+      _player?.dispose();
+    });
+    return const VoiceState();
+  }
+
+  // ── Public API ──────────────────────────────────────────────────────────────
+
+  /// Enter voice mode. Checks server availability and mic permission first.
+  /// [onTranscript] is called with the transcript when a recording completes.
+  /// [enableTts] — if true, TTS plays when [feedContent] is called.
+  /// [onError] — called with a human-readable message on failure.
+  Future<void> enterVoiceMode({
+    required Future<void> Function(String transcript) onTranscript,
+    bool enableTts = false,
+    required void Function(String message) onError,
+  }) async {
+    if (state.voiceModeActive) {
+      exitVoiceMode();
+      return;
+    }
+
+    // Check server availability
+    try {
+      final status = await ref.read(voiceRepositoryProvider).checkStatus();
+      if (!status.fullyAvailable) {
+        onError('Voice not available on this server');
+        return;
+      }
+    } catch (_) {
+      onError('Voice not available on this server');
+      return;
+    }
+
+    // Check microphone permission
+    final permStatus = await Permission.microphone.request();
+    if (permStatus == PermissionStatus.denied ||
+        permStatus == PermissionStatus.permanentlyDenied) {
+      onError('Microphone permission required');
+      if (permStatus == PermissionStatus.permanentlyDenied) {
+        await openAppSettings();
+      }
+      return;
+    }
+
+    _onTranscript = onTranscript;
+    _enableTts = enableTts;
+    _tempDir = await getTemporaryDirectory();
+
+    state = state.copyWith(voiceModeActive: true, available: true);
+    await _startListening();
+  }
+
+  /// Exit voice mode, stop all recording and TTS.
+  void exitVoiceMode() {
+    _amplitudeSubscription?.cancel();
+    _amplitudeSubscription = null;
+    _recorder?.stop();
+    _player?.stop();
+    _ttsQueue.clear();
+    _ttsPlaying = false;
+    _sentenceBuffer = '';
+    _lastSeenLength = 0;
+    _streamComplete = false;
+    _onTranscript = null;
+    state = const VoiceState();
+  }
+
+  /// Feed streaming assistant content for TTS synthesis.
+  /// Call from screens with the full [fullContent] string on each update.
+  /// Set [isComplete] to true when the stream has finished.
+  void feedContent(String fullContent, {required bool isComplete}) {
+    if (!state.voiceModeActive || !_enableTts) return;
+
+    final delta = fullContent.length > _lastSeenLength
+        ? fullContent.substring(_lastSeenLength)
+        : '';
+    _lastSeenLength = fullContent.length;
+    _sentenceBuffer += delta;
+
+    _dispatchSentences(flush: isComplete);
+
+    if (isComplete) {
+      _streamComplete = true;
+      _checkRestartListening();
+    }
+  }
+
+  // ── Internal recording ──────────────────────────────────────────────────────
+
+  Future<void> _startListening() async {
+    if (!state.voiceModeActive) return;
+
+    _silenceMs = 0;
+    _recordingStartMs = DateTime.now().millisecondsSinceEpoch;
+    state = state.copyWith(mode: VoiceMode.recording);
+
+    final dir = _tempDir ?? await getTemporaryDirectory();
+    final path =
+        '${dir.path}/voice_rec_${DateTime.now().millisecondsSinceEpoch}.webm';
+
+    await _recorder!.start(
+      const RecordConfig(encoder: AudioEncoder.opus, sampleRate: 16000),
+      path: path,
+    );
+
+    _amplitudeSubscription?.cancel();
+    _amplitudeSubscription = _recorder!
+        .onAmplitudeChanged(const Duration(milliseconds: 200))
+        .listen(_onAmplitude);
+  }
+
+  void _onAmplitude(Amplitude event) {
+    if (!state.voiceModeActive) return;
+
+    final elapsed =
+        DateTime.now().millisecondsSinceEpoch - _recordingStartMs;
+    if (elapsed < _minRecordingMs) return;
+
+    if (event.current < _silenceThresholdDb) {
+      _silenceMs += 200;
+      if (_silenceMs >= _silenceDurationMs) {
+        _amplitudeSubscription?.cancel();
+        _amplitudeSubscription = null;
+        _handleSilence();
+      }
+    } else {
+      _silenceMs = 0;
+    }
+  }
+
+  Future<void> _handleSilence() async {
+    if (!state.voiceModeActive) return;
+    state = state.copyWith(mode: VoiceMode.transcribing);
+
+    final path = await _recorder!.stop();
+    if (path == null || !state.voiceModeActive) return;
+
+    try {
+      final bytes = await File(path).readAsBytes();
+      await File(path).delete().catchError((_) => File(path));
+
+      if (!state.voiceModeActive) return;
+
+      final transcript =
+          await ref.read(voiceRepositoryProvider).transcribe(bytes);
+
+      if (!state.voiceModeActive) return;
+
+      if (transcript.isEmpty) {
+        // Empty transcript — restart silently
+        await _startListening();
+        return;
+      }
+
+      // Reset TTS state for this new turn
+      _sentenceBuffer = '';
+      _lastSeenLength = 0;
+      _streamComplete = false;
+
+      if (_enableTts) {
+        state = state.copyWith(mode: VoiceMode.playing);
+      }
+
+      await _onTranscript?.call(transcript);
+
+      // If TTS is not enabled, loop immediately
+      if (!_enableTts && state.voiceModeActive) {
+        await _startListening();
+      }
+    } catch (_) {
+      // Network/API error — exit voice mode
+      exitVoiceMode();
+    }
+  }
+
+  // ── Internal TTS ────────────────────────────────────────────────────────────
+
+  void _dispatchSentences({required bool flush}) {
+    final result = extractSentences(_sentenceBuffer);
+    _sentenceBuffer = flush ? '' : result.remainder;
+
+    for (final sentence in result.sentences) {
+      _enqueueSentence(sentence);
+    }
+    if (flush && result.remainder.trim().length >= 3) {
+      _enqueueSentence(result.remainder.trim());
+    }
+  }
+
+  void _enqueueSentence(String sentence) {
+    final stripped = stripMarkdownForTts(sentence);
+    if (stripped.length < 3) return;
+    _synthesiseSentence(stripped);
+  }
+
+  Future<void> _synthesiseSentence(String text) async {
+    try {
+      final wavBytes =
+          await ref.read(voiceRepositoryProvider).synthesise(text);
+      if (!state.voiceModeActive) return;
+      _ttsQueue.add(wavBytes);
+      if (!_ttsPlaying) _drainTtsQueue();
+    } catch (_) {
+      // Skip failed sentence — TTS errors are non-fatal
+    }
+  }
+
+  Future<void> _drainTtsQueue() async {
+    if (_ttsPlaying) return;
+    _ttsPlaying = true;
+    final dir = _tempDir ?? await getTemporaryDirectory();
+
+    try {
+      while (_ttsQueue.isNotEmpty && state.voiceModeActive) {
+        final wavBytes = _ttsQueue.removeFirst();
+        final path = '${dir.path}/tts_${_ttsCounter++}.wav';
+        final file = File(path);
+        await file.writeAsBytes(wavBytes);
+
+        try {
+          await _player!.setFilePath(path);
+          await _player!.play();
+          await _player!.processingStateStream.firstWhere(
+            (s) =>
+                s == ProcessingState.completed || s == ProcessingState.idle,
+          );
+        } finally {
+          await file.delete().catchError((_) => file);
+        }
+      }
+    } finally {
+      _ttsPlaying = false;
+    }
+
+    _checkRestartListening();
+  }
+
+  void _checkRestartListening() {
+    if (_streamComplete &&
+        _ttsQueue.isEmpty &&
+        !_ttsPlaying &&
+        state.voiceModeActive) {
+      _streamComplete = false;
+      _lastSeenLength = 0;
+      _sentenceBuffer = '';
+      _startListening();
+    }
+  }
+}