This repository has been archived on 2026-06-02. You can view files and clone it. You cannot open issues or pull requests or push a commit.
Files
FabledApp/lib/providers/voice_provider.dart
T
bvandeusen d441dcf954 fix(voice): fix silent STT failure — use AAC/M4A, store onError, guard NaN amplitude
Three bugs causing silent STT failure on Android:

1. AudioEncoder.opus produces an OGG container on Android but the file
   was named .webm — faster-whisper rejected it due to format mismatch.
   Changed to AudioEncoder.aacLc + .m4a (reliable on all Android versions).
   Updated voice_api.dart to send audio/mp4 MIME type accordingly.

2. onError callback was never stored, so errors in _startListening() and
   _handleSilence() were silently swallowed. Now stored as _onError and
   called before exitVoiceMode() on any failure.

3. Amplitude stream can emit NaN or ±Infinity on some Android devices
   during recorder initialisation. NaN < -40.0 is false, so silence was
   never detected. Now treated explicitly as silence.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-04-06 17:39:21 -04:00

387 lines
12 KiB
Dart

import 'dart:async';
import 'dart:collection';
import 'dart:io';
import 'dart:typed_data';
import 'package:flutter_riverpod/flutter_riverpod.dart';
import 'package:just_audio/just_audio.dart';
import 'package:path_provider/path_provider.dart';
import 'package:permission_handler/permission_handler.dart';
import 'package:record/record.dart';
import 'api_client_provider.dart';
// ── Public helpers (also used by tests) ──────────────────────────────────────
class SentenceResult {
final List<String> sentences;
final String remainder;
const SentenceResult({required this.sentences, required this.remainder});
}
/// Extract completed sentences from [text] at `.`, `!`, `?` boundaries.
/// Returns the completed sentences and the unconsumed remainder.
SentenceResult extractSentences(String text) {
final boundary = RegExp(r'[.!?]+(?=\s|$)');
final sentences = <String>[];
var remaining = text;
RegExpMatch? match;
while ((match = boundary.firstMatch(remaining)) != null) {
final end = match!.end;
final sentence = remaining.substring(0, end).trim();
if (sentence.isNotEmpty) sentences.add(sentence);
remaining = remaining.substring(end).trimLeft();
}
return SentenceResult(sentences: sentences, remainder: remaining);
}
/// Strip markdown formatting before sending text to TTS.
String stripMarkdownForTts(String text) {
return text
.replaceAll(RegExp(r'```[\s\S]*?```'), '') // fenced code blocks
.replaceAllMapped(RegExp(r'`([^`]+)`'), (m) => m[1]!) // inline code
.replaceAll(RegExp(r'#{1,6}\s+'), '') // headings
.replaceAllMapped(RegExp(r'\*\*([^*]+)\*\*'), (m) => m[1]!) // bold
.replaceAllMapped(RegExp(r'\*([^*]+)\*'), (m) => m[1]!) // italic
.replaceAllMapped(
RegExp(r'\[([^\]]+)\]\([^)]+\)'), (m) => m[1]!) // links → text
.replaceAll(RegExp(r'^\s*[-*+]\s+', multiLine: true), '') // list markers
.replaceAll(RegExp(r'\n{2,}'), ' ') // multiple newlines → space
.replaceAll('\n', ' ')
.trim();
}
// ── State ─────────────────────────────────────────────────────────────────────
enum VoiceMode { idle, recording, transcribing, playing }
class VoiceState {
final VoiceMode mode;
final bool voiceModeActive;
final bool available;
const VoiceState({
this.mode = VoiceMode.idle,
this.voiceModeActive = false,
this.available = true,
});
VoiceState copyWith({
VoiceMode? mode,
bool? voiceModeActive,
bool? available,
}) =>
VoiceState(
mode: mode ?? this.mode,
voiceModeActive: voiceModeActive ?? this.voiceModeActive,
available: available ?? this.available,
);
}
// ── Provider ──────────────────────────────────────────────────────────────────
final voiceProvider =
NotifierProvider<VoiceNotifier, VoiceState>(VoiceNotifier.new);
// ── Notifier ──────────────────────────────────────────────────────────────────
class VoiceNotifier extends Notifier<VoiceState> {
// Audio I/O
AudioRecorder? _recorder;
AudioPlayer? _player;
StreamSubscription<Amplitude>? _amplitudeSubscription;
// Recording / silence detection
int _recordingStartMs = 0;
int _silenceMs = 0;
static const _silenceThresholdDb = -40.0;
static const _silenceDurationMs = 1500;
static const _minRecordingMs = 300;
// Voice mode callbacks
Future<void> Function(String transcript)? _onTranscript;
void Function(String message)? _onError;
bool _enableTts = false;
// Streaming TTS state
String _sentenceBuffer = '';
int _lastSeenLength = 0;
bool _streamComplete = false;
// TTS playback queue
final _ttsQueue = Queue<Uint8List>();
bool _ttsPlaying = false;
int _ttsCounter = 0;
Directory? _tempDir;
@override
VoiceState build() {
_recorder = AudioRecorder();
_player = AudioPlayer();
ref.onDispose(() {
_amplitudeSubscription?.cancel();
_recorder?.dispose();
_player?.dispose();
});
return const VoiceState();
}
// ── Public API ──────────────────────────────────────────────────────────────
/// Enter voice mode. Checks server availability and mic permission first.
/// [onTranscript] is called with the transcript when a recording completes.
/// [enableTts] — if true, TTS plays when [feedContent] is called.
/// [onError] — called with a human-readable message on failure.
Future<void> enterVoiceMode({
required Future<void> Function(String transcript) onTranscript,
bool enableTts = false,
required void Function(String message) onError,
}) async {
if (state.voiceModeActive) {
exitVoiceMode();
return;
}
// Check server availability
try {
final status = await ref.read(voiceRepositoryProvider).checkStatus();
if (!status.fullyAvailable) {
onError('Voice not available on this server');
return;
}
} catch (_) {
onError('Voice not available on this server');
return;
}
// Check microphone permission
final permStatus = await Permission.microphone.request();
if (permStatus == PermissionStatus.denied ||
permStatus == PermissionStatus.permanentlyDenied) {
onError('Microphone permission required');
if (permStatus == PermissionStatus.permanentlyDenied) {
await openAppSettings();
}
return;
}
_onTranscript = onTranscript;
_onError = onError;
_enableTts = enableTts;
_tempDir = await getTemporaryDirectory();
state = state.copyWith(voiceModeActive: true, available: true);
await _startListening();
}
/// Exit voice mode, stop all recording and TTS.
void exitVoiceMode() {
_amplitudeSubscription?.cancel();
_amplitudeSubscription = null;
_recorder?.stop();
_player?.stop();
_ttsQueue.clear();
_ttsPlaying = false;
_sentenceBuffer = '';
_lastSeenLength = 0;
_streamComplete = false;
_onTranscript = null;
_onError = null;
state = const VoiceState();
}
/// Feed streaming assistant content for TTS synthesis.
/// Call from screens with the full [fullContent] string on each update.
/// Set [isComplete] to true when the stream has finished.
void feedContent(String fullContent, {required bool isComplete}) {
if (!state.voiceModeActive || !_enableTts) return;
final delta = fullContent.length > _lastSeenLength
? fullContent.substring(_lastSeenLength)
: '';
_lastSeenLength = fullContent.length;
_sentenceBuffer += delta;
_dispatchSentences(flush: isComplete);
if (isComplete) {
_streamComplete = true;
_checkRestartListening();
}
}
// ── Internal recording ──────────────────────────────────────────────────────
Future<void> _startListening() async {
if (!state.voiceModeActive) return;
_silenceMs = 0;
_recordingStartMs = DateTime.now().millisecondsSinceEpoch;
state = state.copyWith(mode: VoiceMode.recording);
final dir = _tempDir ?? await getTemporaryDirectory();
// AAC/M4A is reliably supported on Android (unlike WebM/Opus which can
// produce OGG bytes in a .webm file, confusing server-side decoders).
final path =
'${dir.path}/voice_rec_${DateTime.now().millisecondsSinceEpoch}.m4a';
try {
await _recorder!.start(
const RecordConfig(encoder: AudioEncoder.aacLc, sampleRate: 16000),
path: path,
);
_amplitudeSubscription?.cancel();
_amplitudeSubscription = _recorder!
.onAmplitudeChanged(const Duration(milliseconds: 200))
.listen(_onAmplitude);
} catch (e) {
_onError?.call('Microphone error: could not start recording');
exitVoiceMode();
}
}
void _onAmplitude(Amplitude event) {
if (!state.voiceModeActive) return;
final elapsed =
DateTime.now().millisecondsSinceEpoch - _recordingStartMs;
if (elapsed < _minRecordingMs) return;
final db = event.current;
// Guard against NaN / ±Infinity which can arrive on some Android devices
// when the recorder is initialising. Treat invalid readings as silence.
final isSilent = db.isNaN || db.isInfinite || db < _silenceThresholdDb;
if (isSilent) {
_silenceMs += 200;
if (_silenceMs >= _silenceDurationMs) {
_amplitudeSubscription?.cancel();
_amplitudeSubscription = null;
_handleSilence();
}
} else {
_silenceMs = 0;
}
}
Future<void> _handleSilence() async {
if (!state.voiceModeActive) return;
state = state.copyWith(mode: VoiceMode.transcribing);
final path = await _recorder!.stop();
if (path == null || !state.voiceModeActive) return;
try {
final bytes = await File(path).readAsBytes();
await File(path).delete().catchError((_) => File(path));
if (!state.voiceModeActive) return;
final transcript =
await ref.read(voiceRepositoryProvider).transcribe(bytes);
if (!state.voiceModeActive) return;
if (transcript.isEmpty) {
// Empty transcript — restart silently
await _startListening();
return;
}
// Reset TTS state for this new turn
_sentenceBuffer = '';
_lastSeenLength = 0;
_streamComplete = false;
if (_enableTts) {
state = state.copyWith(mode: VoiceMode.playing);
}
await _onTranscript?.call(transcript);
// If TTS is not enabled, loop immediately
if (!_enableTts && state.voiceModeActive) {
await _startListening();
}
} catch (e) {
_onError?.call('Voice error: transcription failed');
exitVoiceMode();
}
}
// ── Internal TTS ────────────────────────────────────────────────────────────
void _dispatchSentences({required bool flush}) {
final result = extractSentences(_sentenceBuffer);
_sentenceBuffer = flush ? '' : result.remainder;
for (final sentence in result.sentences) {
_enqueueSentence(sentence);
}
if (flush && result.remainder.trim().length >= 3) {
_enqueueSentence(result.remainder.trim());
}
}
void _enqueueSentence(String sentence) {
final stripped = stripMarkdownForTts(sentence);
if (stripped.length < 3) return;
_synthesiseSentence(stripped);
}
Future<void> _synthesiseSentence(String text) async {
try {
final wavBytes =
await ref.read(voiceRepositoryProvider).synthesise(text);
if (!state.voiceModeActive) return;
_ttsQueue.add(wavBytes);
if (!_ttsPlaying) _drainTtsQueue();
} catch (_) {
// Skip failed sentence — TTS errors are non-fatal
}
}
Future<void> _drainTtsQueue() async {
if (_ttsPlaying) return;
_ttsPlaying = true;
final dir = _tempDir ?? await getTemporaryDirectory();
try {
while (_ttsQueue.isNotEmpty && state.voiceModeActive) {
final wavBytes = _ttsQueue.removeFirst();
final path = '${dir.path}/tts_${_ttsCounter++}.wav';
final file = File(path);
await file.writeAsBytes(wavBytes);
try {
await _player!.setFilePath(path);
await _player!.play();
await _player!.processingStateStream.firstWhere(
(s) =>
s == ProcessingState.completed || s == ProcessingState.idle,
);
} finally {
await file.delete().catchError((_) => file);
}
}
} finally {
_ttsPlaying = false;
}
_checkRestartListening();
}
void _checkRestartListening() {
if (_streamComplete &&
_ttsQueue.isEmpty &&
!_ttsPlaying &&
state.voiceModeActive) {
_streamComplete = false;
_lastSeenLength = 0;
_sentenceBuffer = '';
_startListening();
}
}
}