This repository has been archived on 2026-06-02. You can view files and clone it. You cannot open issues or pull requests or push a commit.
Files
FabledApp/lib/providers/voice_provider.dart
T
bvandeusen fc6c9648f9 fix(voice): eliminate dual-recorder conflict and defunct element crashes
VadHandler is now the sole mic owner — removed separate AudioRecorder that
caused audio focus contention on Android. Audio from onSpeechEnd is encoded
as WAV for Whisper. Provider switched to autoDispose to match widget lifecycle,
preventing defunct element assertions. Recording UI deferred until mic is open.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-04-17 08:04:03 -04:00

495 lines
15 KiB
Dart
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import 'dart:async';
import 'dart:collection';
import 'dart:io';
import 'dart:math';
import 'dart:typed_data';
import 'package:flutter_riverpod/flutter_riverpod.dart';
import 'package:just_audio/just_audio.dart';
import 'package:path_provider/path_provider.dart';
import 'package:permission_handler/permission_handler.dart';
import 'package:vad/vad.dart';
import 'api_client_provider.dart';
// ── Public helpers (also used by tests) ──────────────────────────────────────
class SentenceResult {
final List<String> sentences;
final String remainder;
const SentenceResult({required this.sentences, required this.remainder});
}
/// Extract completed sentences from [text] at `.`, `!`, `?` boundaries.
/// Returns the completed sentences and the unconsumed remainder.
SentenceResult extractSentences(String text) {
final boundary = RegExp(r'[.!?]+(?=\s|$)');
final sentences = <String>[];
var remaining = text;
RegExpMatch? match;
while ((match = boundary.firstMatch(remaining)) != null) {
final end = match!.end;
final sentence = remaining.substring(0, end).trim();
if (sentence.isNotEmpty) sentences.add(sentence);
remaining = remaining.substring(end).trimLeft();
}
return SentenceResult(sentences: sentences, remainder: remaining);
}
/// Strip markdown formatting before sending text to TTS.
String stripMarkdownForTts(String text) {
return text
.replaceAll(RegExp(r'```[\s\S]*?```'), '') // fenced code blocks
.replaceAllMapped(RegExp(r'`([^`]+)`'), (m) => m[1]!) // inline code
.replaceAll(RegExp(r'#{1,6}\s+'), '') // headings
.replaceAllMapped(RegExp(r'\*\*([^*]+)\*\*'), (m) => m[1]!) // bold
.replaceAllMapped(RegExp(r'\*([^*]+)\*'), (m) => m[1]!) // italic
.replaceAllMapped(
RegExp(r'\[([^\]]+)\]\([^)]+\)'), (m) => m[1]!) // links → text
.replaceAll(RegExp(r'^\s*[-*+]\s+', multiLine: true), '') // list markers
.replaceAll(RegExp(r'\n{2,}'), ' ') // multiple newlines → space
.replaceAll('\n', ' ')
.trim();
}
/// Encode float PCM samples (-1..1) as a 16-bit mono WAV at 16 kHz.
Uint8List encodeWav(List<double> samples, {int sampleRate = 16000}) {
final numSamples = samples.length;
final dataSize = numSamples * 2;
final fileSize = 44 + dataSize;
final buf = ByteData(fileSize);
// RIFF header
buf.setUint8(0, 0x52); // R
buf.setUint8(1, 0x49); // I
buf.setUint8(2, 0x46); // F
buf.setUint8(3, 0x46); // F
buf.setUint32(4, fileSize - 8, Endian.little);
buf.setUint8(8, 0x57); // W
buf.setUint8(9, 0x41); // A
buf.setUint8(10, 0x56); // V
buf.setUint8(11, 0x45); // E
// fmt chunk
buf.setUint8(12, 0x66); // f
buf.setUint8(13, 0x6D); // m
buf.setUint8(14, 0x74); // t
buf.setUint8(15, 0x20); // (space)
buf.setUint32(16, 16, Endian.little); // chunk size
buf.setUint16(20, 1, Endian.little); // PCM format
buf.setUint16(22, 1, Endian.little); // mono
buf.setUint32(24, sampleRate, Endian.little);
buf.setUint32(28, sampleRate * 2, Endian.little); // byte rate
buf.setUint16(32, 2, Endian.little); // block align
buf.setUint16(34, 16, Endian.little); // bits per sample
// data chunk
buf.setUint8(36, 0x64); // d
buf.setUint8(37, 0x61); // a
buf.setUint8(38, 0x74); // t
buf.setUint8(39, 0x61); // a
buf.setUint32(40, dataSize, Endian.little);
for (var i = 0; i < numSamples; i++) {
final clamped = samples[i].clamp(-1.0, 1.0);
final int16 = (clamped * 32767).round().clamp(-32768, 32767);
buf.setInt16(44 + i * 2, int16, Endian.little);
}
return buf.buffer.asUint8List();
}
// ── State ─────────────────────────────────────────────────────────────────────
enum VoiceMode { idle, recording, transcribing, playing }
class VoiceState {
final VoiceMode mode;
final bool voiceModeActive;
final bool available;
/// Normalized mic amplitude 0.01.0 while recording.
final double amplitude;
const VoiceState({
this.mode = VoiceMode.idle,
this.voiceModeActive = false,
this.available = true,
this.amplitude = 0.0,
});
VoiceState copyWith({
VoiceMode? mode,
bool? voiceModeActive,
bool? available,
double? amplitude,
}) =>
VoiceState(
mode: mode ?? this.mode,
voiceModeActive: voiceModeActive ?? this.voiceModeActive,
available: available ?? this.available,
amplitude: amplitude ?? this.amplitude,
);
}
// ── Provider ──────────────────────────────────────────────────────────────────
final voiceProvider =
NotifierProvider.autoDispose<VoiceNotifier, VoiceState>(VoiceNotifier.new);
// ── Notifier ──────────────────────────────────────────────────────────────────
class VoiceNotifier extends Notifier<VoiceState> {
// Audio playback
AudioPlayer? _player;
// VAD — sole owner of the microphone
VadHandler? _vadHandler;
StreamSubscription<void>? _vadSpeechStartSub;
StreamSubscription<List<double>>? _vadSpeechEndSub;
StreamSubscription<({double isSpeech, double notSpeech, List<double> frame})>?
_vadFrameSub;
StreamSubscription<String>? _vadErrorSub;
bool _speechDetected = false;
int _speechStartMs = 0;
static const _vadGraceMs = 1500;
bool _disposed = false;
// Voice mode callbacks
Future<void> Function(String transcript)? _onTranscript;
void Function(String message)? _onError;
bool _enableTts = false;
// Streaming TTS state
String _sentenceBuffer = '';
int _lastSeenLength = 0;
bool _streamComplete = false;
// Whisper context hint
String _lastAssistantContent = '';
// Empty transcript counter
int _emptyTranscriptCount = 0;
// TTS playback queue
final _ttsQueue = Queue<Uint8List>();
bool _ttsPlaying = false;
int _ttsCounter = 0;
Directory? _tempDir;
@override
VoiceState build() {
_disposed = false;
_player = AudioPlayer();
ref.onDispose(() {
_disposed = true;
_cancelSubscriptions();
_vadHandler?.dispose();
_vadHandler = null;
_player?.dispose();
_player = null;
});
return const VoiceState();
}
// ── Public API ──────────────────────────────────────────────────────────────
Future<void> enterVoiceMode({
required Future<void> Function(String transcript) onTranscript,
bool enableTts = false,
required void Function(String message) onError,
}) async {
if (state.voiceModeActive) {
if (state.mode == VoiceMode.recording && !_speechDetected) {
onError('No speech detected');
}
exitVoiceMode();
return;
}
try {
final status = await ref.read(voiceRepositoryProvider).checkStatus();
if (!status.enabled || !status.stt) {
onError('Speech-to-text not available on this server');
return;
}
if (!status.tts) enableTts = false;
} catch (_) {
onError('Could not reach voice service');
return;
}
var permStatus = await Permission.microphone.request();
if (permStatus == PermissionStatus.permanentlyDenied) {
onError('Microphone blocked — opening settings');
final opened = await openAppSettings();
if (!opened) return;
permStatus = await Permission.microphone.status;
}
if (!permStatus.isGranted) {
onError('Microphone permission required');
return;
}
_onTranscript = onTranscript;
_onError = onError;
_enableTts = enableTts;
_emptyTranscriptCount = 0;
_tempDir = await getTemporaryDirectory();
state = state.copyWith(voiceModeActive: true, available: true);
await _startListening();
}
void exitVoiceMode() {
_cleanup();
if (!_disposed) state = const VoiceState();
}
void feedContent(String fullContent, {required bool isComplete}) {
if (!state.voiceModeActive || !_enableTts) return;
final delta = fullContent.length > _lastSeenLength
? fullContent.substring(_lastSeenLength)
: '';
_lastSeenLength = fullContent.length;
_sentenceBuffer += delta;
_dispatchSentences(flush: isComplete);
if (isComplete) {
_lastAssistantContent = fullContent;
_streamComplete = true;
_checkRestartListening();
}
}
// ── Internal helpers ────────────────────────────────────────────────────────
void _cleanup() {
_cancelSubscriptions();
final handler = _vadHandler;
_vadHandler = null;
handler?.dispose();
_player?.stop();
_ttsQueue.clear();
_ttsPlaying = false;
_sentenceBuffer = '';
_lastSeenLength = 0;
_streamComplete = false;
_speechDetected = false;
_onTranscript = null;
_onError = null;
}
void _cancelSubscriptions() {
_vadSpeechStartSub?.cancel();
_vadSpeechStartSub = null;
_vadSpeechEndSub?.cancel();
_vadSpeechEndSub = null;
_vadFrameSub?.cancel();
_vadFrameSub = null;
_vadErrorSub?.cancel();
_vadErrorSub = null;
}
// ── Internal recording ──────────────────────────────────────────────────────
Future<void> _startListening() async {
if (_disposed || !state.voiceModeActive) return;
_speechDetected = false;
_speechStartMs = 0;
try {
await _stopVad();
_vadHandler = VadHandler.create();
_vadSpeechStartSub = _vadHandler!.onSpeechStart.listen((_) {
if (_disposed || !state.voiceModeActive) return;
if (!_speechDetected) {
_speechDetected = true;
_speechStartMs = DateTime.now().millisecondsSinceEpoch;
}
});
_vadSpeechEndSub = _vadHandler!.onSpeechEnd.listen((audioSamples) {
if (_disposed || !state.voiceModeActive) return;
final now = DateTime.now().millisecondsSinceEpoch;
final sinceStart = _speechStartMs > 0 ? now - _speechStartMs : 0;
if (_speechDetected && sinceStart >= _vadGraceMs) {
_stopVad();
_handleSpeechEnd(audioSamples);
}
});
_vadFrameSub = _vadHandler!.onFrameProcessed.listen((event) {
if (_disposed || !state.voiceModeActive) return;
final frame = event.frame;
if (frame.isEmpty) return;
double sumSq = 0;
for (final s in frame) {
sumSq += s * s;
}
final rms = sqrt(sumSq / frame.length);
final norm = (rms * 4.0).clamp(0.0, 1.0);
if ((norm - state.amplitude).abs() > 0.02) {
state = state.copyWith(amplitude: norm);
}
});
_vadErrorSub = _vadHandler!.onError.listen((msg) {
if (_disposed) return;
_onError?.call('VAD error: $msg');
});
await _vadHandler!.startListening(model: 'v5');
// Only show recording UI after the mic is actually open.
if (!_disposed && state.voiceModeActive) {
state = state.copyWith(mode: VoiceMode.recording, amplitude: 0.0);
}
} catch (e) {
_onError?.call('Microphone error: $e');
_cleanup();
if (!_disposed) state = const VoiceState();
}
}
Future<void> _stopVad() async {
_cancelSubscriptions();
if (_vadHandler != null) {
final handler = _vadHandler!;
_vadHandler = null;
await handler.dispose();
}
}
Future<void> _handleSpeechEnd(List<double> audioSamples) async {
if (_disposed || !state.voiceModeActive) return;
state = state.copyWith(mode: VoiceMode.transcribing);
try {
final wavBytes = encodeWav(audioSamples);
if (_disposed || !state.voiceModeActive) return;
final transcript = await ref.read(voiceRepositoryProvider).transcribe(
wavBytes,
context:
_lastAssistantContent.isNotEmpty ? _lastAssistantContent : null,
);
if (_disposed || !state.voiceModeActive) return;
if (transcript.isEmpty) {
_emptyTranscriptCount++;
if (_emptyTranscriptCount >= 3) {
_onError?.call('No speech detected — tap the mic to try again');
_cleanup();
if (!_disposed) state = const VoiceState();
return;
}
await _startListening();
return;
}
_emptyTranscriptCount = 0;
_sentenceBuffer = '';
_lastSeenLength = 0;
_streamComplete = false;
if (_enableTts && !_disposed) {
state = state.copyWith(mode: VoiceMode.playing);
}
await _onTranscript?.call(transcript);
// In STT-only mode (no TTS), return to idle after transcript is sent.
// The user taps the mic again to record another message.
if (!_enableTts && !_disposed && state.voiceModeActive) {
state = state.copyWith(mode: VoiceMode.idle);
}
} catch (e) {
_onError?.call('Voice error: transcription failed');
_cleanup();
if (!_disposed) state = const VoiceState();
}
}
// ── Internal TTS ────────────────────────────────────────────────────────────
void _dispatchSentences({required bool flush}) {
final result = extractSentences(_sentenceBuffer);
_sentenceBuffer = flush ? '' : result.remainder;
for (final sentence in result.sentences) {
_enqueueSentence(sentence);
}
if (flush && result.remainder.trim().length >= 3) {
_enqueueSentence(result.remainder.trim());
}
}
void _enqueueSentence(String sentence) {
final stripped = stripMarkdownForTts(sentence);
if (stripped.length < 3) return;
_synthesiseSentence(stripped);
}
Future<void> _synthesiseSentence(String text) async {
try {
final wavBytes =
await ref.read(voiceRepositoryProvider).synthesise(text);
if (_disposed || !state.voiceModeActive) return;
_ttsQueue.add(wavBytes);
if (!_ttsPlaying) _drainTtsQueue();
} catch (_) {
// Skip failed sentence — TTS errors are non-fatal
}
}
Future<void> _drainTtsQueue() async {
if (_ttsPlaying) return;
_ttsPlaying = true;
final dir = _tempDir ?? await getTemporaryDirectory();
try {
while (_ttsQueue.isNotEmpty && state.voiceModeActive) {
final wavBytes = _ttsQueue.removeFirst();
final path = '${dir.path}/tts_${_ttsCounter++}.wav';
final file = File(path);
await file.writeAsBytes(wavBytes);
try {
await _player!.setFilePath(path);
await _player!.play();
await _player!.processingStateStream.firstWhere(
(s) =>
s == ProcessingState.completed || s == ProcessingState.idle,
);
} finally {
await file.delete().catchError((_) => file);
}
}
} finally {
_ttsPlaying = false;
}
_checkRestartListening();
}
void _checkRestartListening() {
if (_disposed) return;
if (_streamComplete &&
_ttsQueue.isEmpty &&
!_ttsPlaying &&
state.voiceModeActive) {
_streamComplete = false;
_lastSeenLength = 0;
_sentenceBuffer = '';
_startListening();
}
}
}