@@ -1,13 +1,13 @@
import ' dart:async ' ;
import ' dart:collection ' ;
import ' dart:io ' ;
import ' dart:math ' ;
import ' dart:typed_data ' ;
import ' package:flutter_riverpod/flutter_riverpod.dart ' ;
import ' package:just_audio/just_audio.dart ' ;
import ' package:path_provider/path_provider.dart ' ;
import ' package:permission_handler/permission_handler.dart ' ;
import ' package:record/record.dart ' ;
import ' package:vad/vad.dart ' ;
import ' api_client_provider.dart ' ;
@@ -52,6 +52,53 @@ String stripMarkdownForTts(String text) {
. trim ( ) ;
}
/// Encode float PCM samples (-1..1) as a 16-bit mono WAV at 16 kHz.
Uint8List encodeWav ( List < double > samples , { int sampleRate = 16000 } ) {
final numSamples = samples . length ;
final dataSize = numSamples * 2 ;
final fileSize = 44 + dataSize ;
final buf = ByteData ( fileSize ) ;
// RIFF header
buf . setUint8 ( 0 , 0x52 ) ; // R
buf . setUint8 ( 1 , 0x49 ) ; // I
buf . setUint8 ( 2 , 0x46 ) ; // F
buf . setUint8 ( 3 , 0x46 ) ; // F
buf . setUint32 ( 4 , fileSize - 8 , Endian . little ) ;
buf . setUint8 ( 8 , 0x57 ) ; // W
buf . setUint8 ( 9 , 0x41 ) ; // A
buf . setUint8 ( 10 , 0x56 ) ; // V
buf . setUint8 ( 11 , 0x45 ) ; // E
// fmt chunk
buf . setUint8 ( 12 , 0x66 ) ; // f
buf . setUint8 ( 13 , 0x6D ) ; // m
buf . setUint8 ( 14 , 0x74 ) ; // t
buf . setUint8 ( 15 , 0x20 ) ; // (space)
buf . setUint32 ( 16 , 16 , Endian . little ) ; // chunk size
buf . setUint16 ( 20 , 1 , Endian . little ) ; // PCM format
buf . setUint16 ( 22 , 1 , Endian . little ) ; // mono
buf . setUint32 ( 24 , sampleRate , Endian . little ) ;
buf . setUint32 ( 28 , sampleRate * 2 , Endian . little ) ; // byte rate
buf . setUint16 ( 32 , 2 , Endian . little ) ; // block align
buf . setUint16 ( 34 , 16 , Endian . little ) ; // bits per sample
// data chunk
buf . setUint8 ( 36 , 0x64 ) ; // d
buf . setUint8 ( 37 , 0x61 ) ; // a
buf . setUint8 ( 38 , 0x74 ) ; // t
buf . setUint8 ( 39 , 0x61 ) ; // a
buf . setUint32 ( 40 , dataSize , Endian . little ) ;
for ( var i = 0 ; i < numSamples ; i + + ) {
final clamped = samples [ i ] . clamp ( - 1.0 , 1.0 ) ;
final int16 = ( clamped * 32767 ) . round ( ) . clamp ( - 32768 , 32767 ) ;
buf . setInt16 ( 44 + i * 2 , int16 , Endian . little ) ;
}
return buf . buffer . asUint8List ( ) ;
}
// ── State ─────────────────────────────────────────────────────────────────────
enum VoiceMode { idle , recording , transcribing , playing }
@@ -60,9 +107,7 @@ class VoiceState {
final VoiceMode mode ;
final bool voiceModeActive ;
final bool available ;
/// Normalized mic amplitude 0.0– 1.0 while recording. Drives the live
/// pulse on VoiceMicButton so the user has obvious feedback that audio
/// is actually being picked up.
/// Normalized mic amplitude 0.0– 1.0 while recording.
final double amplitude ;
const VoiceState ( {
@@ -89,23 +134,25 @@ class VoiceState {
// ── Provider ──────────────────────────────────────────────────────────────────
final voiceProvider =
NotifierProvider < VoiceNotifier , VoiceState > ( VoiceNotifier . new ) ;
NotifierProvider . autoDispose < VoiceNotifier , VoiceState > ( VoiceNotifier . new ) ;
// ── Notifier ──────────────────────────────────────────────────────────────────
class VoiceNotifier extends Notifier < VoiceState > {
// Audio I/O
AudioRecorder ? _recorder ;
// Audio playback
AudioPlayer ? _player ;
StreamSubscription < Amplitude > ? _amplitudeSubscription ;
// VAD-based speech detecti on
// VAD — sole owner of the microph one
VadHandler ? _vadHandler ;
StreamSubscription < void > ? _vadSpeechStartSub ;
StreamSubscription < List < double > > ? _vadSpeechEndSub ;
StreamSubscription < ( { double isSpeech , double notSpeech , List < double > frame } ) > ?
_vadFrameSub ;
StreamSubscription < String > ? _vadErrorSub ;
bool _speechDetected = false ;
int _speechStartMs = 0 ;
static const _vadGraceMs = 1500 ;
bool _disposed = false ;
// Voice mode callbacks
Future < void > Function ( String transcript ) ? _onTranscript ;
@@ -117,11 +164,10 @@ class VoiceNotifier extends Notifier<VoiceState> {
int _lastSeenLength = 0 ;
bool _streamComplete = false ;
// Last complete assistant response — passed to Whisper as initial_promp t
// to reduce STT mishearings of domain-specific words.
// Whisper context hin t
String _lastAssistantContent = ' ' ;
// Empty transcript counter — show feedback after consecutive blanks
// Empty transcript counter
int _emptyTranscriptCount = 0 ;
// TTS playback queue
@@ -132,21 +178,21 @@ class VoiceNotifier extends Notifier<VoiceState> {
@ override
VoiceState build ( ) {
_disposed = false ;
_player = AudioPlayer ( ) ;
ref . onDispose ( ( ) {
_amplitudeSubscription ? . cancel ( ) ;
_recorder ? . dispose ( ) ;
_disposed = true ;
_cancelSubscriptions ( ) ;
_vadHandler ? . dispose ( ) ;
_vadHandler = null ;
_player ? . dispose ( ) ;
_player = null ;
} ) ;
return const VoiceState ( ) ;
}
// ── Public API ──────────────────────────────────────────────────────────────
/// Enter voice mode. Checks server availability and mic permission first.
/// [onTranscript] is called with the transcript when a recording completes.
/// [enableTts] — if true, TTS plays when [feedContent] is called.
/// [onError] — called with a human-readable message on failure.
Future < void > enterVoiceMode ( {
required Future < void > Function ( String transcript ) onTranscript ,
bool enableTts = false ,
@@ -160,27 +206,23 @@ class VoiceNotifier extends Notifier<VoiceState> {
return ;
}
// Check server availability — STT is required, TTS is optional.
try {
final status = await ref . read ( voiceRepositoryProvider ) . checkStatus ( ) ;
if ( ! status . enabled | | ! status . stt ) {
onError ( ' Speech-to-text not available on this server ' ) ;
return ;
}
// Downgrade to STT-only when TTS is unavailable
if ( ! status . tts ) enableTts = false ;
} catch ( _ ) {
onError ( ' Could not reach voice service ' ) ;
return ;
}
// Check microphone permission
var permStatus = await Permission . microphone . request ( ) ;
if ( permStatus = = PermissionStatus . permanentlyDenied ) {
onError ( ' Microphone blocked — opening settings ' ) ;
final opened = await openAppSettings ( ) ;
if ( ! opened ) return ;
// Re-check after user returns from settings
permStatus = await Permission . microphone . status ;
}
if ( ! permStatus . isGranted ) {
@@ -198,27 +240,11 @@ class VoiceNotifier extends Notifier<VoiceState> {
await _startListening ( ) ;
}
/// Exit voice mode, stop all recording and TTS.
void exitVoiceMode ( ) {
_amplitudeSubscription ? . cancel ( ) ;
_amplitudeSubscription = null ;
_recorder ? . stop ( ) ;
_stopVad ( ) ;
_player ? . stop ( ) ;
_ttsQueue . clear ( ) ;
_ttsPlaying = false ;
_sentenceBuffer = ' ' ;
_lastSeenLength = 0 ;
_streamComplete = false ;
_speechDetected = false ;
_onTranscript = null ;
_onError = null ;
state = const VoiceState ( ) ;
_cleanup ( ) ;
if ( ! _disposed ) state = const VoiceState ( ) ;
}
/// Feed streaming assistant content for TTS synthesis.
/// Call from screens with the full [fullContent] string on each update.
/// Set [isComplete] to true when the stream has finished.
void feedContent ( String fullContent , { required bool isComplete } ) {
if ( ! state . voiceModeActive | | ! _enableTts ) return ;
@@ -237,112 +263,131 @@ class VoiceNotifier extends Notifier<VoiceState> {
}
}
// ── Internal helpers ────────────────────────────────────────────────────────
void _cleanup ( ) {
_cancelSubscriptions ( ) ;
final handler = _vadHandler ;
_vadHandler = null ;
handler ? . dispose ( ) ;
_player ? . stop ( ) ;
_ttsQueue . clear ( ) ;
_ttsPlaying = false ;
_sentenceBuffer = ' ' ;
_lastSeenLength = 0 ;
_streamComplete = false ;
_speechDetected = false ;
_onTranscript = null ;
_onError = null ;
}
void _cancelSubscriptions ( ) {
_vadSpeechStartSub ? . cancel ( ) ;
_vadSpeechStartSub = null ;
_vadSpeechEndSub ? . cancel ( ) ;
_vadSpeechEndSub = null ;
_vadFrameSub ? . cancel ( ) ;
_vadFrameSub = null ;
_vadErrorSub ? . cancel ( ) ;
_vadErrorSub = null ;
}
// ── Internal recording ──────────────────────────────────────────────────────
Future < void > _startListening ( ) async {
if ( ! state . voiceModeActive ) return ;
if ( _disposed | | ! state . voiceModeActive ) return ;
_speechDetected = false ;
_speechStartMs = 0 ;
state = state . copyWith ( mode: VoiceMode . recording ) ;
final dir = _tempDir ? ? await getTemporaryDirectory ( ) ;
final path =
' ${ dir . path } /voice_rec_ ${ DateTime . now ( ) . millisecondsSinceEpoch } .m4a ' ;
try {
_recorder ? . dispose ( ) ;
_recorder = AudioRecorder ( ) ;
if ( ! await _recorder ! . hasPermission ( ) ) {
_onError ? . call ( ' Microphone permission was revoked ' ) ;
exitVoiceMode ( ) ;
return ;
}
await _recorder ! . start (
const RecordConfig ( encoder: AudioEncoder . aacLc , sampleRate: 16000 ) ,
path: path ,
) ;
_amplitudeSubscription ? . cancel ( ) ;
_amplitudeSubscription = _recorder !
. onAmplitudeChanged ( const Duration ( milliseconds: 200 ) )
. listen ( _onAmplitude ) ;
// VAD for speech detection — uses its own AudioRecorder internally
await _stopVad ( ) ;
_vadHandler = VadHandler . create ( ) ;
_vadSpeechStartSub = _vadHandler ! . onSpeechStart . listen ( ( _ ) {
if ( _disposed | | ! state . voiceModeActive ) return ;
if ( ! _speechDetected ) {
_speechDetected = true ;
_speechStartMs = DateTime . now ( ) . millisecondsSinceEpoch ;
}
} ) ;
_vadSpeechEndSub = _vadHandler ! . onSpeechEnd . listen ( ( _ ) {
if ( ! state . voiceModeActive ) return ;
_vadSpeechEndSub = _vadHandler ! . onSpeechEnd . listen ( ( audioSamples ) {
if ( _disposed | | ! state . voiceModeActive ) return ;
final now = DateTime . now ( ) . millisecondsSinceEpoch ;
final sinceStart = _speechStartMs > 0 ? now - _speechStartMs : 0 ;
if ( _speechDetected & & sinceStart > = _vadGraceMs ) {
_amplitudeSubscription ? . cancel ( ) ;
_amplitudeSubscription = null ;
_stopVad ( ) ;
_handleSilence ( ) ;
_handleSpeechEnd ( audioSamples ) ;
}
} ) ;
_vadFrameSub = _vadHandler ! . onFrameProcessed . listen ( ( event ) {
if ( _disposed | | ! state . voiceModeActive ) return ;
final frame = event . frame ;
if ( frame . isEmpty ) return ;
double sumSq = 0 ;
for ( final s in frame ) {
sumSq + = s * s ;
}
final rms = sqrt ( sumSq / frame . length ) ;
final norm = ( rms * 4.0 ) . clamp ( 0.0 , 1.0 ) ;
if ( ( norm - state . amplitude ) . abs ( ) > 0.02 ) {
state = state . copyWith ( amplitude: norm ) ;
}
} ) ;
_vadErrorSub = _vadHandler ! . onError . listen ( ( msg ) {
if ( _disposed ) return ;
_onError ? . call ( ' VAD error: $ msg ' ) ;
} ) ;
await _vadHandler ! . startListening ( model: ' v5 ' ) ;
// Only show recording UI after the mic is actually open.
if ( ! _disposed & & state . voiceModeActive ) {
state = state . copyWith ( mode: VoiceMode . recording , amplitude: 0.0 ) ;
}
} catch ( e ) {
_onError ? . call ( ' Microphone error: $ e ' ) ;
exitVoiceMode ( ) ;
_cleanup ( ) ;
if ( ! _disposed ) state = const VoiceState ( ) ;
}
}
Future < void > _stopVad ( ) async {
await _vadSpeechStartSub ? . cancel ( ) ;
_vadSpeechStartSub = null ;
await _vadSpeechEndSub ? . cancel ( ) ;
_vadSpeechEndSub = null ;
_cancelSubscriptions ( ) ;
if ( _vadHandler ! = null ) {
await _vadHandler ! . dispose ( ) ;
final handler = _vadHandler ! ;
_vadHandler = null ;
await handler . dispose ( ) ;
}
}
void _onAmplitude ( Amplitude event ) {
if ( ! state . voiceModeActive ) return ;
final db = event . current ;
if ( db . isNaN | | db . isInfinite ) return ;
final norm = ( ( db + 60.0 ) / 60.0 ) . clamp ( 0.0 , 1.0 ) ;
if ( ( norm - state . amplitude ) . abs ( ) > 0.02 ) {
state = state . copyWith ( amplitude: norm ) ;
}
}
Future < void > _handleSilence ( ) async {
if ( ! state . voiceModeActive ) return ;
Future < void > _handleSpeechEnd ( List < double > audioSamples ) async {
if ( _disposed | | ! state . voiceModeActive ) return ;
state = state . copyWith ( mode: VoiceMode . transcribing ) ;
final path = await _recorder ! . stop ( ) ;
if ( path = = null | | ! state . voiceModeActive ) return ;
try {
final bytes = await File ( path ) . readAsBytes ( ) ;
await File ( path ) . delete ( ) . catchError ( ( _ ) = > File ( path ) ) ;
final wavBytes = encodeWav ( audioSamples ) ;
if ( ! state . voiceModeActive ) return ;
if ( _disposed | | ! state . voiceModeActive ) return ;
final transcript = await ref . read ( voiceRepositoryProvider ) . transcribe (
bytes ,
context: _lastAssistantContent . isNotEmpty ? _lastAssistantContent : null ,
wavBytes ,
context:
_lastAssistantContent . isNotEmpty ? _lastAssistantContent : null ,
) ;
if ( ! state . voiceModeActive ) return ;
if ( _disposed | | ! state . voiceModeActive ) return ;
if ( transcript . isEmpty ) {
_emptyTranscriptCount + + ;
if ( _emptyTranscriptCount > = 3 ) {
_onError ? . call ( ' No speech detected — tap the mic to try again ' ) ;
exitVoiceMode ( ) ;
_cleanup ( ) ;
if ( ! _disposed ) state = const VoiceState ( ) ;
return ;
}
await _startListening ( ) ;
@@ -350,24 +395,25 @@ class VoiceNotifier extends Notifier<VoiceState> {
}
_emptyTranscriptCount = 0 ;
// Reset TTS state for this new turn
_sentenceBuffer = ' ' ;
_lastSeenLength = 0 ;
_streamComplete = false ;
if ( _enableTts ) {
if ( _enableTts & & ! _disposed ) {
state = state . copyWith ( mode: VoiceMode . playing ) ;
}
await _onTranscript ? . call ( transcript ) ;
// If TTS is not enabled, loop immediately
if ( ! _enableTts & & state . voiceModeActive ) {
await _startListening ( ) ;
// In STT-only mode (no TTS), return to idle after transcript is sent.
// The user taps the mic again to record another message.
if ( ! _enableTts & & ! _disposed & & state . voiceModeActive ) {
state = state . copyWith ( mode: VoiceMode . idle ) ;
}
} catch ( e ) {
_onError ? . call ( ' Voice error: transcription failed ' ) ;
exitVoiceMode ( ) ;
_cleanup ( ) ;
if ( ! _disposed ) state = const VoiceState ( ) ;
}
}
@@ -395,7 +441,7 @@ class VoiceNotifier extends Notifier<VoiceState> {
try {
final wavBytes =
await ref . read ( voiceRepositoryProvider ) . synthesise ( text ) ;
if ( ! state . voiceModeActive ) return ;
if ( _disposed | | ! state . voiceModeActive ) return ;
_ttsQueue . add ( wavBytes ) ;
if ( ! _ttsPlaying ) _drainTtsQueue ( ) ;
} catch ( _ ) {
@@ -434,6 +480,7 @@ class VoiceNotifier extends Notifier<VoiceState> {
}
void _checkRestartListening ( ) {
if ( _disposed ) return ;
if ( _streamComplete & &
_ttsQueue . isEmpty & &
! _ttsPlaying & &