bvandeusen/FabledApp
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 32 Wiki Activity

fix(voice): eliminate dual-recorder conflict and defunct element crashes

Browse Source 
VadHandler is now the sole mic owner — removed separate AudioRecorder that
caused audio focus contention on Android. Audio from onSpeechEnd is encoded
as WAV for Whisper. Provider switched to autoDispose to match widget lifecycle,
preventing defunct element assertions. Recording UI deferred until mic is open.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
bvandeusen
2026-04-17 08:04:03 -04:00
parent 413b82f724
commit fc6c9648f9
8 changed files with 160 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/providers/voice_provider.dart
+153 -106
View File
@@ -1,13 +1,13 @@
import 'dart:async';
import 'dart:collection';
import 'dart:io';
import 'dart:math';
import 'dart:typed_data';
import 'package:flutter_riverpod/flutter_riverpod.dart';
import 'package:just_audio/just_audio.dart';
import 'package:path_provider/path_provider.dart';
import 'package:permission_handler/permission_handler.dart';
import 'package:record/record.dart';
import 'package:vad/vad.dart';
import 'api_client_provider.dart';
@@ -52,6 +52,53 @@ String stripMarkdownForTts(String text) {
.trim();
}
/// Encode float PCM samples (-1..1) as a 16-bit mono WAV at 16 kHz.
Uint8List encodeWav(List<double> samples, {int sampleRate = 16000}) {
final numSamples = samples.length;
final dataSize = numSamples * 2;
final fileSize = 44 + dataSize;
final buf = ByteData(fileSize);
// RIFF header
buf.setUint8(0, 0x52); // R
buf.setUint8(1, 0x49); // I
buf.setUint8(2, 0x46); // F
buf.setUint8(3, 0x46); // F
buf.setUint32(4, fileSize - 8, Endian.little);
buf.setUint8(8, 0x57); // W
buf.setUint8(9, 0x41); // A
buf.setUint8(10, 0x56); // V
buf.setUint8(11, 0x45); // E
// fmt chunk
buf.setUint8(12, 0x66); // f
buf.setUint8(13, 0x6D); // m
buf.setUint8(14, 0x74); // t
buf.setUint8(15, 0x20); // (space)
buf.setUint32(16, 16, Endian.little); // chunk size
buf.setUint16(20, 1, Endian.little); // PCM format
buf.setUint16(22, 1, Endian.little); // mono
buf.setUint32(24, sampleRate, Endian.little);
buf.setUint32(28, sampleRate * 2, Endian.little); // byte rate
buf.setUint16(32, 2, Endian.little); // block align
buf.setUint16(34, 16, Endian.little); // bits per sample
// data chunk
buf.setUint8(36, 0x64); // d
buf.setUint8(37, 0x61); // a
buf.setUint8(38, 0x74); // t
buf.setUint8(39, 0x61); // a
buf.setUint32(40, dataSize, Endian.little);
for (var i = 0; i < numSamples; i++) {
final clamped = samples[i].clamp(-1.0, 1.0);
final int16 = (clamped * 32767).round().clamp(-32768, 32767);
buf.setInt16(44 + i * 2, int16, Endian.little);
}
return buf.buffer.asUint8List();
}
// ── State ─────────────────────────────────────────────────────────────────────
enum VoiceMode { idle, recording, transcribing, playing }
@@ -60,9 +107,7 @@ class VoiceState {
final VoiceMode mode;
final bool voiceModeActive;
final bool available;
/// Normalized mic amplitude 0.0–1.0 while recording. Drives the live
/// pulse on VoiceMicButton so the user has obvious feedback that audio
/// is actually being picked up.
/// Normalized mic amplitude 0.0–1.0 while recording.
final double amplitude;
const VoiceState({
@@ -89,23 +134,25 @@ class VoiceState {
// ── Provider ──────────────────────────────────────────────────────────────────
final voiceProvider =
NotifierProvider<VoiceNotifier, VoiceState>(VoiceNotifier.new);
NotifierProvider.autoDispose<VoiceNotifier, VoiceState>(VoiceNotifier.new);
// ── Notifier ──────────────────────────────────────────────────────────────────
class VoiceNotifier extends Notifier<VoiceState> {
// Audio I/O
AudioRecorder? _recorder;
// Audio playback
AudioPlayer? _player;
StreamSubscription<Amplitude>? _amplitudeSubscription;
// VAD-based speech detection
// VAD — sole owner of the microphone
VadHandler? _vadHandler;
StreamSubscription<void>? _vadSpeechStartSub;
StreamSubscription<List<double>>? _vadSpeechEndSub;
StreamSubscription<({double isSpeech, double notSpeech, List<double> frame})>?
_vadFrameSub;
StreamSubscription<String>? _vadErrorSub;
bool _speechDetected = false;
int _speechStartMs = 0;
static const _vadGraceMs = 1500;
bool _disposed = false;
// Voice mode callbacks
Future<void> Function(String transcript)? _onTranscript;
@@ -117,11 +164,10 @@ class VoiceNotifier extends Notifier<VoiceState> {
int _lastSeenLength = 0;
bool _streamComplete = false;
// Last complete assistant response — passed to Whisper as initial_prompt
// to reduce STT mishearings of domain-specific words.
// Whisper context hint
String _lastAssistantContent = '';
// Empty transcript counter — show feedback after consecutive blanks
// Empty transcript counter
int _emptyTranscriptCount = 0;
// TTS playback queue
@@ -132,21 +178,21 @@ class VoiceNotifier extends Notifier<VoiceState> {
@override
VoiceState build() {
_disposed = false;
_player = AudioPlayer();
ref.onDispose(() {
_amplitudeSubscription?.cancel();
_recorder?.dispose();
_disposed = true;
_cancelSubscriptions();
_vadHandler?.dispose();
_vadHandler = null;
_player?.dispose();
_player = null;
});
return const VoiceState();
}
// ── Public API ──────────────────────────────────────────────────────────────
/// Enter voice mode. Checks server availability and mic permission first.
/// [onTranscript] is called with the transcript when a recording completes.
/// [enableTts] — if true, TTS plays when [feedContent] is called.
/// [onError] — called with a human-readable message on failure.
Future<void> enterVoiceMode({
required Future<void> Function(String transcript) onTranscript,
bool enableTts = false,
@@ -160,27 +206,23 @@ class VoiceNotifier extends Notifier<VoiceState> {
return;
}
// Check server availability — STT is required, TTS is optional.
try {
final status = await ref.read(voiceRepositoryProvider).checkStatus();
if (!status.enabled || !status.stt) {
onError('Speech-to-text not available on this server');
return;
}
// Downgrade to STT-only when TTS is unavailable
if (!status.tts) enableTts = false;
} catch (_) {
onError('Could not reach voice service');
return;
}
// Check microphone permission
var permStatus = await Permission.microphone.request();
if (permStatus == PermissionStatus.permanentlyDenied) {
onError('Microphone blocked — opening settings');
final opened = await openAppSettings();
if (!opened) return;
// Re-check after user returns from settings
permStatus = await Permission.microphone.status;
}
if (!permStatus.isGranted) {
@@ -198,27 +240,11 @@ class VoiceNotifier extends Notifier<VoiceState> {
await _startListening();
}
/// Exit voice mode, stop all recording and TTS.
void exitVoiceMode() {
_amplitudeSubscription?.cancel();
_amplitudeSubscription = null;
_recorder?.stop();
_stopVad();
_player?.stop();
_ttsQueue.clear();
_ttsPlaying = false;
_sentenceBuffer = '';
_lastSeenLength = 0;
_streamComplete = false;
_speechDetected = false;
_onTranscript = null;
_onError = null;
state = const VoiceState();
_cleanup();
if (!_disposed) state = const VoiceState();
}
/// Feed streaming assistant content for TTS synthesis.
/// Call from screens with the full [fullContent] string on each update.
/// Set [isComplete] to true when the stream has finished.
void feedContent(String fullContent, {required bool isComplete}) {
if (!state.voiceModeActive || !_enableTts) return;
@@ -237,112 +263,131 @@ class VoiceNotifier extends Notifier<VoiceState> {
}
}
// ── Internal helpers ────────────────────────────────────────────────────────
void _cleanup() {
_cancelSubscriptions();
final handler = _vadHandler;
_vadHandler = null;
handler?.dispose();
_player?.stop();
_ttsQueue.clear();
_ttsPlaying = false;
_sentenceBuffer = '';
_lastSeenLength = 0;
_streamComplete = false;
_speechDetected = false;
_onTranscript = null;
_onError = null;
}
void _cancelSubscriptions() {
_vadSpeechStartSub?.cancel();
_vadSpeechStartSub = null;
_vadSpeechEndSub?.cancel();
_vadSpeechEndSub = null;
_vadFrameSub?.cancel();
_vadFrameSub = null;
_vadErrorSub?.cancel();
_vadErrorSub = null;
}
// ── Internal recording ──────────────────────────────────────────────────────
Future<void> _startListening() async {
if (!state.voiceModeActive) return;
if (_disposed || !state.voiceModeActive) return;
_speechDetected = false;
_speechStartMs = 0;
state = state.copyWith(mode: VoiceMode.recording);
final dir = _tempDir ?? await getTemporaryDirectory();
final path =
'${dir.path}/voice_rec_${DateTime.now().millisecondsSinceEpoch}.m4a';
try {
_recorder?.dispose();
_recorder = AudioRecorder();
if (!await _recorder!.hasPermission()) {
_onError?.call('Microphone permission was revoked');
exitVoiceMode();
return;
}
await _recorder!.start(
const RecordConfig(encoder: AudioEncoder.aacLc, sampleRate: 16000),
path: path,
);
_amplitudeSubscription?.cancel();
_amplitudeSubscription = _recorder!
.onAmplitudeChanged(const Duration(milliseconds: 200))
.listen(_onAmplitude);
// VAD for speech detection — uses its own AudioRecorder internally
await _stopVad();
_vadHandler = VadHandler.create();
_vadSpeechStartSub = _vadHandler!.onSpeechStart.listen((_) {
if (_disposed || !state.voiceModeActive) return;
if (!_speechDetected) {
_speechDetected = true;
_speechStartMs = DateTime.now().millisecondsSinceEpoch;
}
});
_vadSpeechEndSub = _vadHandler!.onSpeechEnd.listen((_) {
if (!state.voiceModeActive) return;
_vadSpeechEndSub = _vadHandler!.onSpeechEnd.listen((audioSamples) {
if (_disposed || !state.voiceModeActive) return;
final now = DateTime.now().millisecondsSinceEpoch;
final sinceStart = _speechStartMs > 0 ? now - _speechStartMs : 0;
if (_speechDetected && sinceStart >= _vadGraceMs) {
_amplitudeSubscription?.cancel();
_amplitudeSubscription = null;
_stopVad();
_handleSilence();
_handleSpeechEnd(audioSamples);
}
});
_vadFrameSub = _vadHandler!.onFrameProcessed.listen((event) {
if (_disposed || !state.voiceModeActive) return;
final frame = event.frame;
if (frame.isEmpty) return;
double sumSq = 0;
for (final s in frame) {
sumSq += s * s;
}
final rms = sqrt(sumSq / frame.length);
final norm = (rms * 4.0).clamp(0.0, 1.0);
if ((norm - state.amplitude).abs() > 0.02) {
state = state.copyWith(amplitude: norm);
}
});
_vadErrorSub = _vadHandler!.onError.listen((msg) {
if (_disposed) return;
_onError?.call('VAD error: $msg');
});
await _vadHandler!.startListening(model: 'v5');
// Only show recording UI after the mic is actually open.
if (!_disposed && state.voiceModeActive) {
state = state.copyWith(mode: VoiceMode.recording, amplitude: 0.0);
}
} catch (e) {
_onError?.call('Microphone error: $e');
exitVoiceMode();
_cleanup();
if (!_disposed) state = const VoiceState();
}
}
Future<void> _stopVad() async {
await _vadSpeechStartSub?.cancel();
_vadSpeechStartSub = null;
await _vadSpeechEndSub?.cancel();
_vadSpeechEndSub = null;
_cancelSubscriptions();
if (_vadHandler != null) {
await _vadHandler!.dispose();
final handler = _vadHandler!;
_vadHandler = null;
await handler.dispose();
}
}
void _onAmplitude(Amplitude event) {
if (!state.voiceModeActive) return;
final db = event.current;
if (db.isNaN || db.isInfinite) return;
final norm = ((db + 60.0) / 60.0).clamp(0.0, 1.0);
if ((norm - state.amplitude).abs() > 0.02) {
state = state.copyWith(amplitude: norm);
}
}
Future<void> _handleSilence() async {
if (!state.voiceModeActive) return;
Future<void> _handleSpeechEnd(List<double> audioSamples) async {
if (_disposed || !state.voiceModeActive) return;
state = state.copyWith(mode: VoiceMode.transcribing);
final path = await _recorder!.stop();
if (path == null || !state.voiceModeActive) return;
try {
final bytes = await File(path).readAsBytes();
await File(path).delete().catchError((_) => File(path));
final wavBytes = encodeWav(audioSamples);
if (!state.voiceModeActive) return;
if (_disposed || !state.voiceModeActive) return;
final transcript = await ref.read(voiceRepositoryProvider).transcribe(
bytes,
context: _lastAssistantContent.isNotEmpty ? _lastAssistantContent : null,
wavBytes,
context:
_lastAssistantContent.isNotEmpty ? _lastAssistantContent : null,
);
if (!state.voiceModeActive) return;
if (_disposed || !state.voiceModeActive) return;
if (transcript.isEmpty) {
_emptyTranscriptCount++;
if (_emptyTranscriptCount >= 3) {
_onError?.call('No speech detected — tap the mic to try again');
exitVoiceMode();
_cleanup();
if (!_disposed) state = const VoiceState();
return;
}
await _startListening();
@@ -350,24 +395,25 @@ class VoiceNotifier extends Notifier<VoiceState> {
}
_emptyTranscriptCount = 0;
// Reset TTS state for this new turn
_sentenceBuffer = '';
_lastSeenLength = 0;
_streamComplete = false;
if (_enableTts) {
if (_enableTts && !_disposed) {
state = state.copyWith(mode: VoiceMode.playing);
}
await _onTranscript?.call(transcript);
// If TTS is not enabled, loop immediately
if (!_enableTts && state.voiceModeActive) {
await _startListening();
// In STT-only mode (no TTS), return to idle after transcript is sent.
// The user taps the mic again to record another message.
if (!_enableTts && !_disposed && state.voiceModeActive) {
state = state.copyWith(mode: VoiceMode.idle);
}
} catch (e) {
_onError?.call('Voice error: transcription failed');
exitVoiceMode();
_cleanup();
if (!_disposed) state = const VoiceState();
}
}
@@ -395,7 +441,7 @@ class VoiceNotifier extends Notifier<VoiceState> {
try {
final wavBytes =
await ref.read(voiceRepositoryProvider).synthesise(text);
if (!state.voiceModeActive) return;
if (_disposed || !state.voiceModeActive) return;
_ttsQueue.add(wavBytes);
if (!_ttsPlaying) _drainTtsQueue();
} catch (_) {
@@ -434,6 +480,7 @@ class VoiceNotifier extends Notifier<VoiceState> {
}
void _checkRestartListening() {
if (_disposed) return;
if (_streamComplete &&
_ttsQueue.isEmpty &&
!_ttsPlaying &&