fix(voice): eliminate dual-recorder conflict and defunct element crashes

VadHandler is now the sole mic owner — removed separate AudioRecorder that
caused audio focus contention on Android. Audio from onSpeechEnd is encoded
as WAV for Whisper. Provider switched to autoDispose to match widget lifecycle,
preventing defunct element assertions. Recording UI deferred until mic is open.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-04-17 08:04:03 -04:00
parent 413b82f724
commit fc6c9648f9
8 changed files with 160 additions and 115 deletions
-1
View File
@@ -529,7 +529,6 @@ class _QuickCaptureBarState extends ConsumerState<_QuickCaptureBar> {
@override @override
void dispose() { void dispose() {
_controller.dispose(); _controller.dispose();
ref.read(voiceProvider.notifier).exitVoiceMode();
super.dispose(); super.dispose();
} }
+3 -3
View File
@@ -40,7 +40,7 @@ class VoiceApi {
} }
} }
/// POST WebM/Opus audio bytes and return the transcript string. /// POST audio bytes (WAV) and return the transcript string.
/// [context] is optional recent conversation text passed as initial_prompt /// [context] is optional recent conversation text passed as initial_prompt
/// to Whisper, reducing mishearings of domain-specific words. /// to Whisper, reducing mishearings of domain-specific words.
/// Returns empty string on empty or error response. /// Returns empty string on empty or error response.
@@ -49,8 +49,8 @@ class VoiceApi {
final fields = <String, dynamic>{ final fields = <String, dynamic>{
'audio': MultipartFile.fromBytes( 'audio': MultipartFile.fromBytes(
audioBytes, audioBytes,
filename: 'audio.m4a', filename: 'audio.wav',
contentType: DioMediaType('audio', 'mp4'), contentType: DioMediaType('audio', 'wav'),
), ),
if (context != null && context.isNotEmpty) 'context': context, if (context != null && context.isNotEmpty) 'context': context,
}; };
+153 -106
View File
@@ -1,13 +1,13 @@
import 'dart:async'; import 'dart:async';
import 'dart:collection'; import 'dart:collection';
import 'dart:io'; import 'dart:io';
import 'dart:math';
import 'dart:typed_data'; import 'dart:typed_data';
import 'package:flutter_riverpod/flutter_riverpod.dart'; import 'package:flutter_riverpod/flutter_riverpod.dart';
import 'package:just_audio/just_audio.dart'; import 'package:just_audio/just_audio.dart';
import 'package:path_provider/path_provider.dart'; import 'package:path_provider/path_provider.dart';
import 'package:permission_handler/permission_handler.dart'; import 'package:permission_handler/permission_handler.dart';
import 'package:record/record.dart';
import 'package:vad/vad.dart'; import 'package:vad/vad.dart';
import 'api_client_provider.dart'; import 'api_client_provider.dart';
@@ -52,6 +52,53 @@ String stripMarkdownForTts(String text) {
.trim(); .trim();
} }
/// Encode float PCM samples (-1..1) as a 16-bit mono WAV at 16 kHz.
Uint8List encodeWav(List<double> samples, {int sampleRate = 16000}) {
final numSamples = samples.length;
final dataSize = numSamples * 2;
final fileSize = 44 + dataSize;
final buf = ByteData(fileSize);
// RIFF header
buf.setUint8(0, 0x52); // R
buf.setUint8(1, 0x49); // I
buf.setUint8(2, 0x46); // F
buf.setUint8(3, 0x46); // F
buf.setUint32(4, fileSize - 8, Endian.little);
buf.setUint8(8, 0x57); // W
buf.setUint8(9, 0x41); // A
buf.setUint8(10, 0x56); // V
buf.setUint8(11, 0x45); // E
// fmt chunk
buf.setUint8(12, 0x66); // f
buf.setUint8(13, 0x6D); // m
buf.setUint8(14, 0x74); // t
buf.setUint8(15, 0x20); // (space)
buf.setUint32(16, 16, Endian.little); // chunk size
buf.setUint16(20, 1, Endian.little); // PCM format
buf.setUint16(22, 1, Endian.little); // mono
buf.setUint32(24, sampleRate, Endian.little);
buf.setUint32(28, sampleRate * 2, Endian.little); // byte rate
buf.setUint16(32, 2, Endian.little); // block align
buf.setUint16(34, 16, Endian.little); // bits per sample
// data chunk
buf.setUint8(36, 0x64); // d
buf.setUint8(37, 0x61); // a
buf.setUint8(38, 0x74); // t
buf.setUint8(39, 0x61); // a
buf.setUint32(40, dataSize, Endian.little);
for (var i = 0; i < numSamples; i++) {
final clamped = samples[i].clamp(-1.0, 1.0);
final int16 = (clamped * 32767).round().clamp(-32768, 32767);
buf.setInt16(44 + i * 2, int16, Endian.little);
}
return buf.buffer.asUint8List();
}
// ── State ───────────────────────────────────────────────────────────────────── // ── State ─────────────────────────────────────────────────────────────────────
enum VoiceMode { idle, recording, transcribing, playing } enum VoiceMode { idle, recording, transcribing, playing }
@@ -60,9 +107,7 @@ class VoiceState {
final VoiceMode mode; final VoiceMode mode;
final bool voiceModeActive; final bool voiceModeActive;
final bool available; final bool available;
/// Normalized mic amplitude 0.01.0 while recording. Drives the live /// Normalized mic amplitude 0.01.0 while recording.
/// pulse on VoiceMicButton so the user has obvious feedback that audio
/// is actually being picked up.
final double amplitude; final double amplitude;
const VoiceState({ const VoiceState({
@@ -89,23 +134,25 @@ class VoiceState {
// ── Provider ────────────────────────────────────────────────────────────────── // ── Provider ──────────────────────────────────────────────────────────────────
final voiceProvider = final voiceProvider =
NotifierProvider<VoiceNotifier, VoiceState>(VoiceNotifier.new); NotifierProvider.autoDispose<VoiceNotifier, VoiceState>(VoiceNotifier.new);
// ── Notifier ────────────────────────────────────────────────────────────────── // ── Notifier ──────────────────────────────────────────────────────────────────
class VoiceNotifier extends Notifier<VoiceState> { class VoiceNotifier extends Notifier<VoiceState> {
// Audio I/O // Audio playback
AudioRecorder? _recorder;
AudioPlayer? _player; AudioPlayer? _player;
StreamSubscription<Amplitude>? _amplitudeSubscription;
// VAD-based speech detection // VAD — sole owner of the microphone
VadHandler? _vadHandler; VadHandler? _vadHandler;
StreamSubscription<void>? _vadSpeechStartSub; StreamSubscription<void>? _vadSpeechStartSub;
StreamSubscription<List<double>>? _vadSpeechEndSub; StreamSubscription<List<double>>? _vadSpeechEndSub;
StreamSubscription<({double isSpeech, double notSpeech, List<double> frame})>?
_vadFrameSub;
StreamSubscription<String>? _vadErrorSub;
bool _speechDetected = false; bool _speechDetected = false;
int _speechStartMs = 0; int _speechStartMs = 0;
static const _vadGraceMs = 1500; static const _vadGraceMs = 1500;
bool _disposed = false;
// Voice mode callbacks // Voice mode callbacks
Future<void> Function(String transcript)? _onTranscript; Future<void> Function(String transcript)? _onTranscript;
@@ -117,11 +164,10 @@ class VoiceNotifier extends Notifier<VoiceState> {
int _lastSeenLength = 0; int _lastSeenLength = 0;
bool _streamComplete = false; bool _streamComplete = false;
// Last complete assistant response — passed to Whisper as initial_prompt // Whisper context hint
// to reduce STT mishearings of domain-specific words.
String _lastAssistantContent = ''; String _lastAssistantContent = '';
// Empty transcript counter — show feedback after consecutive blanks // Empty transcript counter
int _emptyTranscriptCount = 0; int _emptyTranscriptCount = 0;
// TTS playback queue // TTS playback queue
@@ -132,21 +178,21 @@ class VoiceNotifier extends Notifier<VoiceState> {
@override @override
VoiceState build() { VoiceState build() {
_disposed = false;
_player = AudioPlayer(); _player = AudioPlayer();
ref.onDispose(() { ref.onDispose(() {
_amplitudeSubscription?.cancel(); _disposed = true;
_recorder?.dispose(); _cancelSubscriptions();
_vadHandler?.dispose();
_vadHandler = null;
_player?.dispose(); _player?.dispose();
_player = null;
}); });
return const VoiceState(); return const VoiceState();
} }
// ── Public API ────────────────────────────────────────────────────────────── // ── Public API ──────────────────────────────────────────────────────────────
/// Enter voice mode. Checks server availability and mic permission first.
/// [onTranscript] is called with the transcript when a recording completes.
/// [enableTts] — if true, TTS plays when [feedContent] is called.
/// [onError] — called with a human-readable message on failure.
Future<void> enterVoiceMode({ Future<void> enterVoiceMode({
required Future<void> Function(String transcript) onTranscript, required Future<void> Function(String transcript) onTranscript,
bool enableTts = false, bool enableTts = false,
@@ -160,27 +206,23 @@ class VoiceNotifier extends Notifier<VoiceState> {
return; return;
} }
// Check server availability — STT is required, TTS is optional.
try { try {
final status = await ref.read(voiceRepositoryProvider).checkStatus(); final status = await ref.read(voiceRepositoryProvider).checkStatus();
if (!status.enabled || !status.stt) { if (!status.enabled || !status.stt) {
onError('Speech-to-text not available on this server'); onError('Speech-to-text not available on this server');
return; return;
} }
// Downgrade to STT-only when TTS is unavailable
if (!status.tts) enableTts = false; if (!status.tts) enableTts = false;
} catch (_) { } catch (_) {
onError('Could not reach voice service'); onError('Could not reach voice service');
return; return;
} }
// Check microphone permission
var permStatus = await Permission.microphone.request(); var permStatus = await Permission.microphone.request();
if (permStatus == PermissionStatus.permanentlyDenied) { if (permStatus == PermissionStatus.permanentlyDenied) {
onError('Microphone blocked — opening settings'); onError('Microphone blocked — opening settings');
final opened = await openAppSettings(); final opened = await openAppSettings();
if (!opened) return; if (!opened) return;
// Re-check after user returns from settings
permStatus = await Permission.microphone.status; permStatus = await Permission.microphone.status;
} }
if (!permStatus.isGranted) { if (!permStatus.isGranted) {
@@ -198,27 +240,11 @@ class VoiceNotifier extends Notifier<VoiceState> {
await _startListening(); await _startListening();
} }
/// Exit voice mode, stop all recording and TTS.
void exitVoiceMode() { void exitVoiceMode() {
_amplitudeSubscription?.cancel(); _cleanup();
_amplitudeSubscription = null; if (!_disposed) state = const VoiceState();
_recorder?.stop();
_stopVad();
_player?.stop();
_ttsQueue.clear();
_ttsPlaying = false;
_sentenceBuffer = '';
_lastSeenLength = 0;
_streamComplete = false;
_speechDetected = false;
_onTranscript = null;
_onError = null;
state = const VoiceState();
} }
/// Feed streaming assistant content for TTS synthesis.
/// Call from screens with the full [fullContent] string on each update.
/// Set [isComplete] to true when the stream has finished.
void feedContent(String fullContent, {required bool isComplete}) { void feedContent(String fullContent, {required bool isComplete}) {
if (!state.voiceModeActive || !_enableTts) return; if (!state.voiceModeActive || !_enableTts) return;
@@ -237,112 +263,131 @@ class VoiceNotifier extends Notifier<VoiceState> {
} }
} }
// ── Internal helpers ────────────────────────────────────────────────────────
void _cleanup() {
_cancelSubscriptions();
final handler = _vadHandler;
_vadHandler = null;
handler?.dispose();
_player?.stop();
_ttsQueue.clear();
_ttsPlaying = false;
_sentenceBuffer = '';
_lastSeenLength = 0;
_streamComplete = false;
_speechDetected = false;
_onTranscript = null;
_onError = null;
}
void _cancelSubscriptions() {
_vadSpeechStartSub?.cancel();
_vadSpeechStartSub = null;
_vadSpeechEndSub?.cancel();
_vadSpeechEndSub = null;
_vadFrameSub?.cancel();
_vadFrameSub = null;
_vadErrorSub?.cancel();
_vadErrorSub = null;
}
// ── Internal recording ────────────────────────────────────────────────────── // ── Internal recording ──────────────────────────────────────────────────────
Future<void> _startListening() async { Future<void> _startListening() async {
if (!state.voiceModeActive) return; if (_disposed || !state.voiceModeActive) return;
_speechDetected = false; _speechDetected = false;
_speechStartMs = 0; _speechStartMs = 0;
state = state.copyWith(mode: VoiceMode.recording);
final dir = _tempDir ?? await getTemporaryDirectory();
final path =
'${dir.path}/voice_rec_${DateTime.now().millisecondsSinceEpoch}.m4a';
try { try {
_recorder?.dispose();
_recorder = AudioRecorder();
if (!await _recorder!.hasPermission()) {
_onError?.call('Microphone permission was revoked');
exitVoiceMode();
return;
}
await _recorder!.start(
const RecordConfig(encoder: AudioEncoder.aacLc, sampleRate: 16000),
path: path,
);
_amplitudeSubscription?.cancel();
_amplitudeSubscription = _recorder!
.onAmplitudeChanged(const Duration(milliseconds: 200))
.listen(_onAmplitude);
// VAD for speech detection — uses its own AudioRecorder internally
await _stopVad(); await _stopVad();
_vadHandler = VadHandler.create(); _vadHandler = VadHandler.create();
_vadSpeechStartSub = _vadHandler!.onSpeechStart.listen((_) { _vadSpeechStartSub = _vadHandler!.onSpeechStart.listen((_) {
if (_disposed || !state.voiceModeActive) return;
if (!_speechDetected) { if (!_speechDetected) {
_speechDetected = true; _speechDetected = true;
_speechStartMs = DateTime.now().millisecondsSinceEpoch; _speechStartMs = DateTime.now().millisecondsSinceEpoch;
} }
}); });
_vadSpeechEndSub = _vadHandler!.onSpeechEnd.listen((_) {
if (!state.voiceModeActive) return; _vadSpeechEndSub = _vadHandler!.onSpeechEnd.listen((audioSamples) {
if (_disposed || !state.voiceModeActive) return;
final now = DateTime.now().millisecondsSinceEpoch; final now = DateTime.now().millisecondsSinceEpoch;
final sinceStart = _speechStartMs > 0 ? now - _speechStartMs : 0; final sinceStart = _speechStartMs > 0 ? now - _speechStartMs : 0;
if (_speechDetected && sinceStart >= _vadGraceMs) { if (_speechDetected && sinceStart >= _vadGraceMs) {
_amplitudeSubscription?.cancel();
_amplitudeSubscription = null;
_stopVad(); _stopVad();
_handleSilence(); _handleSpeechEnd(audioSamples);
} }
}); });
_vadFrameSub = _vadHandler!.onFrameProcessed.listen((event) {
if (_disposed || !state.voiceModeActive) return;
final frame = event.frame;
if (frame.isEmpty) return;
double sumSq = 0;
for (final s in frame) {
sumSq += s * s;
}
final rms = sqrt(sumSq / frame.length);
final norm = (rms * 4.0).clamp(0.0, 1.0);
if ((norm - state.amplitude).abs() > 0.02) {
state = state.copyWith(amplitude: norm);
}
});
_vadErrorSub = _vadHandler!.onError.listen((msg) {
if (_disposed) return;
_onError?.call('VAD error: $msg');
});
await _vadHandler!.startListening(model: 'v5'); await _vadHandler!.startListening(model: 'v5');
// Only show recording UI after the mic is actually open.
if (!_disposed && state.voiceModeActive) {
state = state.copyWith(mode: VoiceMode.recording, amplitude: 0.0);
}
} catch (e) { } catch (e) {
_onError?.call('Microphone error: $e'); _onError?.call('Microphone error: $e');
exitVoiceMode(); _cleanup();
if (!_disposed) state = const VoiceState();
} }
} }
Future<void> _stopVad() async { Future<void> _stopVad() async {
await _vadSpeechStartSub?.cancel(); _cancelSubscriptions();
_vadSpeechStartSub = null;
await _vadSpeechEndSub?.cancel();
_vadSpeechEndSub = null;
if (_vadHandler != null) { if (_vadHandler != null) {
await _vadHandler!.dispose(); final handler = _vadHandler!;
_vadHandler = null; _vadHandler = null;
await handler.dispose();
} }
} }
void _onAmplitude(Amplitude event) { Future<void> _handleSpeechEnd(List<double> audioSamples) async {
if (!state.voiceModeActive) return; if (_disposed || !state.voiceModeActive) return;
final db = event.current;
if (db.isNaN || db.isInfinite) return;
final norm = ((db + 60.0) / 60.0).clamp(0.0, 1.0);
if ((norm - state.amplitude).abs() > 0.02) {
state = state.copyWith(amplitude: norm);
}
}
Future<void> _handleSilence() async {
if (!state.voiceModeActive) return;
state = state.copyWith(mode: VoiceMode.transcribing); state = state.copyWith(mode: VoiceMode.transcribing);
final path = await _recorder!.stop();
if (path == null || !state.voiceModeActive) return;
try { try {
final bytes = await File(path).readAsBytes(); final wavBytes = encodeWav(audioSamples);
await File(path).delete().catchError((_) => File(path));
if (!state.voiceModeActive) return; if (_disposed || !state.voiceModeActive) return;
final transcript = await ref.read(voiceRepositoryProvider).transcribe( final transcript = await ref.read(voiceRepositoryProvider).transcribe(
bytes, wavBytes,
context: _lastAssistantContent.isNotEmpty ? _lastAssistantContent : null, context:
_lastAssistantContent.isNotEmpty ? _lastAssistantContent : null,
); );
if (!state.voiceModeActive) return; if (_disposed || !state.voiceModeActive) return;
if (transcript.isEmpty) { if (transcript.isEmpty) {
_emptyTranscriptCount++; _emptyTranscriptCount++;
if (_emptyTranscriptCount >= 3) { if (_emptyTranscriptCount >= 3) {
_onError?.call('No speech detected — tap the mic to try again'); _onError?.call('No speech detected — tap the mic to try again');
exitVoiceMode(); _cleanup();
if (!_disposed) state = const VoiceState();
return; return;
} }
await _startListening(); await _startListening();
@@ -350,24 +395,25 @@ class VoiceNotifier extends Notifier<VoiceState> {
} }
_emptyTranscriptCount = 0; _emptyTranscriptCount = 0;
// Reset TTS state for this new turn
_sentenceBuffer = ''; _sentenceBuffer = '';
_lastSeenLength = 0; _lastSeenLength = 0;
_streamComplete = false; _streamComplete = false;
if (_enableTts) { if (_enableTts && !_disposed) {
state = state.copyWith(mode: VoiceMode.playing); state = state.copyWith(mode: VoiceMode.playing);
} }
await _onTranscript?.call(transcript); await _onTranscript?.call(transcript);
// If TTS is not enabled, loop immediately // In STT-only mode (no TTS), return to idle after transcript is sent.
if (!_enableTts && state.voiceModeActive) { // The user taps the mic again to record another message.
await _startListening(); if (!_enableTts && !_disposed && state.voiceModeActive) {
state = state.copyWith(mode: VoiceMode.idle);
} }
} catch (e) { } catch (e) {
_onError?.call('Voice error: transcription failed'); _onError?.call('Voice error: transcription failed');
exitVoiceMode(); _cleanup();
if (!_disposed) state = const VoiceState();
} }
} }
@@ -395,7 +441,7 @@ class VoiceNotifier extends Notifier<VoiceState> {
try { try {
final wavBytes = final wavBytes =
await ref.read(voiceRepositoryProvider).synthesise(text); await ref.read(voiceRepositoryProvider).synthesise(text);
if (!state.voiceModeActive) return; if (_disposed || !state.voiceModeActive) return;
_ttsQueue.add(wavBytes); _ttsQueue.add(wavBytes);
if (!_ttsPlaying) _drainTtsQueue(); if (!_ttsPlaying) _drainTtsQueue();
} catch (_) { } catch (_) {
@@ -434,6 +480,7 @@ class VoiceNotifier extends Notifier<VoiceState> {
} }
void _checkRestartListening() { void _checkRestartListening() {
if (_disposed) return;
if (_streamComplete && if (_streamComplete &&
_ttsQueue.isEmpty && _ttsQueue.isEmpty &&
!_ttsPlaying && !_ttsPlaying &&
@@ -76,7 +76,6 @@ class _BriefingScreenState extends ConsumerState<BriefingScreen>
WidgetsBinding.instance.removeObserver(this); WidgetsBinding.instance.removeObserver(this);
_controller.dispose(); _controller.dispose();
_scrollController.dispose(); _scrollController.dispose();
ref.read(voiceProvider.notifier).exitVoiceMode();
super.dispose(); super.dispose();
} }
-2
View File
@@ -68,8 +68,6 @@ class _ChatScreenState extends ConsumerState<ChatScreen>
WidgetsBinding.instance.removeObserver(this); WidgetsBinding.instance.removeObserver(this);
_controller.dispose(); _controller.dispose();
_scrollController.dispose(); _scrollController.dispose();
// Exit voice mode if the user navigates away mid-session.
ref.read(voiceProvider.notifier).exitVoiceMode();
super.dispose(); super.dispose();
} }
+1
View File
@@ -10,6 +10,7 @@ list(APPEND FLUTTER_PLUGIN_LIST
) )
list(APPEND FLUTTER_FFI_PLUGIN_LIST list(APPEND FLUTTER_FFI_PLUGIN_LIST
vad
) )
set(PLUGIN_BUNDLED_LIBRARIES) set(PLUGIN_BUNDLED_LIBRARIES)
@@ -11,7 +11,7 @@ import flutter_timezone
import just_audio import just_audio
import open_file_mac import open_file_mac
import package_info_plus import package_info_plus
import record_darwin import record_macos
import shared_preferences_foundation import shared_preferences_foundation
import url_launcher_macos import url_launcher_macos
@@ -22,7 +22,7 @@ func RegisterGeneratedPlugins(registry: FlutterPluginRegistry) {
JustAudioPlugin.register(with: registry.registrar(forPlugin: "JustAudioPlugin")) JustAudioPlugin.register(with: registry.registrar(forPlugin: "JustAudioPlugin"))
OpenFilePlugin.register(with: registry.registrar(forPlugin: "OpenFilePlugin")) OpenFilePlugin.register(with: registry.registrar(forPlugin: "OpenFilePlugin"))
FPPPackageInfoPlusPlugin.register(with: registry.registrar(forPlugin: "FPPPackageInfoPlusPlugin")) FPPPackageInfoPlusPlugin.register(with: registry.registrar(forPlugin: "FPPPackageInfoPlusPlugin"))
RecordPlugin.register(with: registry.registrar(forPlugin: "RecordPlugin")) RecordMacOsPlugin.register(with: registry.registrar(forPlugin: "RecordMacOsPlugin"))
SharedPreferencesPlugin.register(with: registry.registrar(forPlugin: "SharedPreferencesPlugin")) SharedPreferencesPlugin.register(with: registry.registrar(forPlugin: "SharedPreferencesPlugin"))
UrlLauncherPlugin.register(with: registry.registrar(forPlugin: "UrlLauncherPlugin")) UrlLauncherPlugin.register(with: registry.registrar(forPlugin: "UrlLauncherPlugin"))
} }
+1
View File
@@ -11,6 +11,7 @@ list(APPEND FLUTTER_PLUGIN_LIST
) )
list(APPEND FLUTTER_FFI_PLUGIN_LIST list(APPEND FLUTTER_FFI_PLUGIN_LIST
vad
) )
set(PLUGIN_BUNDLED_LIBRARIES) set(PLUGIN_BUNDLED_LIBRARIES)