This repository has been archived on 2026-06-02. You can view files and clone it. You cannot open issues or pull requests or push a commit.
Files
FabledApp/lib/providers/voice_provider.dart
T
bvandeusen bdaa5210f0 feat(voice): dynamic silence threshold
The previous -40 dBFS static threshold sat right on top of typical
phone mic ambient, so silence detection rarely fired and the user
always had to tap stop manually.

Silence threshold is now dynamic: track the session peak dBFS and
treat "silent" as 15 dB below peak. Auto-calibrates per mic and
environment rather than assuming a fixed ambient level.

- Grace period (1500 ms) at start so the user has time to begin
  speaking before checks arm.
- Static -35 dB fallback until the peak clears -20 dB so a dead-
  silent session doesn't spin forever.
- Silence duration bumped 1500 → 2000 ms for breathing room.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-04-15 00:28:04 -04:00

458 lines
15 KiB
Dart
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import 'dart:async';
import 'dart:collection';
import 'dart:io';
import 'dart:typed_data';
import 'package:flutter_riverpod/flutter_riverpod.dart';
import 'package:just_audio/just_audio.dart';
import 'package:path_provider/path_provider.dart';
import 'package:permission_handler/permission_handler.dart';
import 'package:record/record.dart';
import 'api_client_provider.dart';
// ── Public helpers (also used by tests) ──────────────────────────────────────
class SentenceResult {
final List<String> sentences;
final String remainder;
const SentenceResult({required this.sentences, required this.remainder});
}
/// Extract completed sentences from [text] at `.`, `!`, `?` boundaries.
/// Returns the completed sentences and the unconsumed remainder.
SentenceResult extractSentences(String text) {
final boundary = RegExp(r'[.!?]+(?=\s|$)');
final sentences = <String>[];
var remaining = text;
RegExpMatch? match;
while ((match = boundary.firstMatch(remaining)) != null) {
final end = match!.end;
final sentence = remaining.substring(0, end).trim();
if (sentence.isNotEmpty) sentences.add(sentence);
remaining = remaining.substring(end).trimLeft();
}
return SentenceResult(sentences: sentences, remainder: remaining);
}
/// Strip markdown formatting before sending text to TTS.
String stripMarkdownForTts(String text) {
return text
.replaceAll(RegExp(r'```[\s\S]*?```'), '') // fenced code blocks
.replaceAllMapped(RegExp(r'`([^`]+)`'), (m) => m[1]!) // inline code
.replaceAll(RegExp(r'#{1,6}\s+'), '') // headings
.replaceAllMapped(RegExp(r'\*\*([^*]+)\*\*'), (m) => m[1]!) // bold
.replaceAllMapped(RegExp(r'\*([^*]+)\*'), (m) => m[1]!) // italic
.replaceAllMapped(
RegExp(r'\[([^\]]+)\]\([^)]+\)'), (m) => m[1]!) // links → text
.replaceAll(RegExp(r'^\s*[-*+]\s+', multiLine: true), '') // list markers
.replaceAll(RegExp(r'\n{2,}'), ' ') // multiple newlines → space
.replaceAll('\n', ' ')
.trim();
}
// ── State ─────────────────────────────────────────────────────────────────────
enum VoiceMode { idle, recording, transcribing, playing }
class VoiceState {
final VoiceMode mode;
final bool voiceModeActive;
final bool available;
/// Normalized mic amplitude 0.01.0 while recording. Drives the live
/// pulse on VoiceMicButton so the user has obvious feedback that audio
/// is actually being picked up.
final double amplitude;
const VoiceState({
this.mode = VoiceMode.idle,
this.voiceModeActive = false,
this.available = true,
this.amplitude = 0.0,
});
VoiceState copyWith({
VoiceMode? mode,
bool? voiceModeActive,
bool? available,
double? amplitude,
}) =>
VoiceState(
mode: mode ?? this.mode,
voiceModeActive: voiceModeActive ?? this.voiceModeActive,
available: available ?? this.available,
amplitude: amplitude ?? this.amplitude,
);
}
// ── Provider ──────────────────────────────────────────────────────────────────
final voiceProvider =
NotifierProvider<VoiceNotifier, VoiceState>(VoiceNotifier.new);
// ── Notifier ──────────────────────────────────────────────────────────────────
class VoiceNotifier extends Notifier<VoiceState> {
// Audio I/O
AudioRecorder? _recorder;
AudioPlayer? _player;
StreamSubscription<Amplitude>? _amplitudeSubscription;
// Recording / silence detection
//
// Silence threshold is dynamic: we track the session peak dBFS and treat
// "silent" as "current level is at least _dropFromPeakDb below peak."
// This auto-calibrates to whatever mic + room the user is on rather than
// assuming a fixed ambient level. Until the peak climbs above
// _dynamicArmDb we fall back to a conservative static threshold so a
// quiet room doesn't spin forever. A grace period at the start of the
// recording gives the user time to begin speaking before silence checks
// arm.
int _recordingStartMs = 0;
int _silenceMs = 0;
double _peakDb = -100.0;
static const _fallbackThresholdDb = -35.0;
static const _dropFromPeakDb = 15.0;
static const _dynamicArmDb = -20.0;
static const _graceMs = 1500;
static const _silenceDurationMs = 2000;
static const _minRecordingMs = 300;
// Voice mode callbacks
Future<void> Function(String transcript)? _onTranscript;
void Function(String message)? _onError;
bool _enableTts = false;
// Streaming TTS state
String _sentenceBuffer = '';
int _lastSeenLength = 0;
bool _streamComplete = false;
// Last complete assistant response — passed to Whisper as initial_prompt
// to reduce STT mishearings of domain-specific words.
String _lastAssistantContent = '';
// Empty transcript counter — show feedback after consecutive blanks
int _emptyTranscriptCount = 0;
// TTS playback queue
final _ttsQueue = Queue<Uint8List>();
bool _ttsPlaying = false;
int _ttsCounter = 0;
Directory? _tempDir;
@override
VoiceState build() {
_player = AudioPlayer();
ref.onDispose(() {
_amplitudeSubscription?.cancel();
_recorder?.dispose();
_player?.dispose();
});
return const VoiceState();
}
// ── Public API ──────────────────────────────────────────────────────────────
/// Enter voice mode. Checks server availability and mic permission first.
/// [onTranscript] is called with the transcript when a recording completes.
/// [enableTts] — if true, TTS plays when [feedContent] is called.
/// [onError] — called with a human-readable message on failure.
Future<void> enterVoiceMode({
required Future<void> Function(String transcript) onTranscript,
bool enableTts = false,
required void Function(String message) onError,
}) async {
if (state.voiceModeActive) {
exitVoiceMode();
return;
}
// Check server availability — STT is required, TTS is optional.
try {
final status = await ref.read(voiceRepositoryProvider).checkStatus();
if (!status.enabled || !status.stt) {
onError('Speech-to-text not available on this server');
return;
}
// Downgrade to STT-only when TTS is unavailable
if (!status.tts) enableTts = false;
} catch (_) {
onError('Could not reach voice service');
return;
}
// Check microphone permission
var permStatus = await Permission.microphone.request();
if (permStatus == PermissionStatus.permanentlyDenied) {
onError('Microphone blocked — opening settings');
final opened = await openAppSettings();
if (!opened) return;
// Re-check after user returns from settings
permStatus = await Permission.microphone.status;
}
if (!permStatus.isGranted) {
onError('Microphone permission required');
return;
}
_onTranscript = onTranscript;
_onError = onError;
_enableTts = enableTts;
_emptyTranscriptCount = 0;
_tempDir = await getTemporaryDirectory();
state = state.copyWith(voiceModeActive: true, available: true);
await _startListening();
}
/// Exit voice mode, stop all recording and TTS.
void exitVoiceMode() {
_amplitudeSubscription?.cancel();
_amplitudeSubscription = null;
_recorder?.stop();
_player?.stop();
_ttsQueue.clear();
_ttsPlaying = false;
_sentenceBuffer = '';
_lastSeenLength = 0;
_streamComplete = false;
_onTranscript = null;
_onError = null;
state = const VoiceState();
}
/// Feed streaming assistant content for TTS synthesis.
/// Call from screens with the full [fullContent] string on each update.
/// Set [isComplete] to true when the stream has finished.
void feedContent(String fullContent, {required bool isComplete}) {
if (!state.voiceModeActive || !_enableTts) return;
final delta = fullContent.length > _lastSeenLength
? fullContent.substring(_lastSeenLength)
: '';
_lastSeenLength = fullContent.length;
_sentenceBuffer += delta;
_dispatchSentences(flush: isComplete);
if (isComplete) {
_lastAssistantContent = fullContent;
_streamComplete = true;
_checkRestartListening();
}
}
// ── Internal recording ──────────────────────────────────────────────────────
Future<void> _startListening() async {
if (!state.voiceModeActive) return;
_silenceMs = 0;
_peakDb = -100.0;
_recordingStartMs = DateTime.now().millisecondsSinceEpoch;
state = state.copyWith(mode: VoiceMode.recording);
final dir = _tempDir ?? await getTemporaryDirectory();
final path =
'${dir.path}/voice_rec_${DateTime.now().millisecondsSinceEpoch}.m4a';
try {
// Recreate the recorder each session — the record package can leave
// the native AudioRecord in a bad state after stop/error cycles,
// and a stale instance is the most common cause of "could not start".
_recorder?.dispose();
_recorder = AudioRecorder();
if (!await _recorder!.hasPermission()) {
_onError?.call('Microphone permission was revoked');
exitVoiceMode();
return;
}
await _recorder!.start(
const RecordConfig(encoder: AudioEncoder.aacLc, sampleRate: 16000),
path: path,
);
_amplitudeSubscription?.cancel();
_amplitudeSubscription = _recorder!
.onAmplitudeChanged(const Duration(milliseconds: 200))
.listen(_onAmplitude);
} catch (e) {
_onError?.call('Microphone error: $e');
exitVoiceMode();
}
}
void _onAmplitude(Amplitude event) {
if (!state.voiceModeActive) return;
final db = event.current;
final validDb = !(db.isNaN || db.isInfinite);
// Normalize dB to 0..1 for the pulse animation. -60dB is dead-quiet,
// 0dB is peak; we clamp and bias so the button visibly breathes even
// on soft speech without exploding on loud input.
if (validDb) {
final norm = ((db + 60.0) / 60.0).clamp(0.0, 1.0);
if ((norm - state.amplitude).abs() > 0.02) {
state = state.copyWith(amplitude: norm);
}
if (db > _peakDb) _peakDb = db;
}
final elapsed =
DateTime.now().millisecondsSinceEpoch - _recordingStartMs;
// Grace period at the start — let the user begin speaking before we
// start silence-counting. Also suppresses any false triggers while
// the native recorder is still warming up.
if (elapsed < _graceMs) return;
if (elapsed < _minRecordingMs) return;
// Dynamic threshold once we've seen real speech; static fallback
// before that so a dead-silent session doesn't spin forever.
final threshold =
_peakDb > _dynamicArmDb ? _peakDb - _dropFromPeakDb : _fallbackThresholdDb;
final isSilent = !validDb || db < threshold;
if (isSilent) {
_silenceMs += 200;
if (_silenceMs >= _silenceDurationMs) {
_amplitudeSubscription?.cancel();
_amplitudeSubscription = null;
_handleSilence();
}
} else {
_silenceMs = 0;
}
}
Future<void> _handleSilence() async {
if (!state.voiceModeActive) return;
state = state.copyWith(mode: VoiceMode.transcribing);
final path = await _recorder!.stop();
if (path == null || !state.voiceModeActive) return;
try {
final bytes = await File(path).readAsBytes();
await File(path).delete().catchError((_) => File(path));
if (!state.voiceModeActive) return;
final transcript = await ref.read(voiceRepositoryProvider).transcribe(
bytes,
context: _lastAssistantContent.isNotEmpty ? _lastAssistantContent : null,
);
if (!state.voiceModeActive) return;
if (transcript.isEmpty) {
_emptyTranscriptCount++;
if (_emptyTranscriptCount >= 3) {
_onError?.call('No speech detected — tap the mic to try again');
exitVoiceMode();
return;
}
await _startListening();
return;
}
_emptyTranscriptCount = 0;
// Reset TTS state for this new turn
_sentenceBuffer = '';
_lastSeenLength = 0;
_streamComplete = false;
if (_enableTts) {
state = state.copyWith(mode: VoiceMode.playing);
}
await _onTranscript?.call(transcript);
// If TTS is not enabled, loop immediately
if (!_enableTts && state.voiceModeActive) {
await _startListening();
}
} catch (e) {
_onError?.call('Voice error: transcription failed');
exitVoiceMode();
}
}
// ── Internal TTS ────────────────────────────────────────────────────────────
void _dispatchSentences({required bool flush}) {
final result = extractSentences(_sentenceBuffer);
_sentenceBuffer = flush ? '' : result.remainder;
for (final sentence in result.sentences) {
_enqueueSentence(sentence);
}
if (flush && result.remainder.trim().length >= 3) {
_enqueueSentence(result.remainder.trim());
}
}
void _enqueueSentence(String sentence) {
final stripped = stripMarkdownForTts(sentence);
if (stripped.length < 3) return;
_synthesiseSentence(stripped);
}
Future<void> _synthesiseSentence(String text) async {
try {
final wavBytes =
await ref.read(voiceRepositoryProvider).synthesise(text);
if (!state.voiceModeActive) return;
_ttsQueue.add(wavBytes);
if (!_ttsPlaying) _drainTtsQueue();
} catch (_) {
// Skip failed sentence — TTS errors are non-fatal
}
}
Future<void> _drainTtsQueue() async {
if (_ttsPlaying) return;
_ttsPlaying = true;
final dir = _tempDir ?? await getTemporaryDirectory();
try {
while (_ttsQueue.isNotEmpty && state.voiceModeActive) {
final wavBytes = _ttsQueue.removeFirst();
final path = '${dir.path}/tts_${_ttsCounter++}.wav';
final file = File(path);
await file.writeAsBytes(wavBytes);
try {
await _player!.setFilePath(path);
await _player!.play();
await _player!.processingStateStream.firstWhere(
(s) =>
s == ProcessingState.completed || s == ProcessingState.idle,
);
} finally {
await file.delete().catchError((_) => file);
}
}
} finally {
_ttsPlaying = false;
}
_checkRestartListening();
}
void _checkRestartListening() {
if (_streamComplete &&
_ttsQueue.isEmpty &&
!_ttsPlaying &&
state.voiceModeActive) {
_streamComplete = false;
_lastSeenLength = 0;
_sentenceBuffer = '';
_startListening();
}
}
}