This repository has been archived on 2026-06-02. You can view files and clone it. You cannot open issues or pull requests or push a commit.
Files
FabledApp/lib/providers/voice_provider.dart
T
bvandeusen 4240c90d55 fix(voice,knowledge): mic recording + pagination + STT-only mode
Voice: recreate AudioRecorder each session to avoid stale native
state, improve permission handling (re-check after settings), show
feedback after 3 consecutive empty transcripts, allow STT-only mode
when TTS is unavailable.

Knowledge: hydrate IDs immediately after loading more pages so
scroll-based pagination doesn't stall at the bottom.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-04-12 11:34:00 -04:00

418 lines
13 KiB
Dart

import 'dart:async';
import 'dart:collection';
import 'dart:io';
import 'dart:typed_data';
import 'package:flutter_riverpod/flutter_riverpod.dart';
import 'package:just_audio/just_audio.dart';
import 'package:path_provider/path_provider.dart';
import 'package:permission_handler/permission_handler.dart';
import 'package:record/record.dart';
import 'api_client_provider.dart';
// ── Public helpers (also used by tests) ──────────────────────────────────────
class SentenceResult {
final List<String> sentences;
final String remainder;
const SentenceResult({required this.sentences, required this.remainder});
}
/// Extract completed sentences from [text] at `.`, `!`, `?` boundaries.
/// Returns the completed sentences and the unconsumed remainder.
SentenceResult extractSentences(String text) {
final boundary = RegExp(r'[.!?]+(?=\s|$)');
final sentences = <String>[];
var remaining = text;
RegExpMatch? match;
while ((match = boundary.firstMatch(remaining)) != null) {
final end = match!.end;
final sentence = remaining.substring(0, end).trim();
if (sentence.isNotEmpty) sentences.add(sentence);
remaining = remaining.substring(end).trimLeft();
}
return SentenceResult(sentences: sentences, remainder: remaining);
}
/// Strip markdown formatting before sending text to TTS.
String stripMarkdownForTts(String text) {
return text
.replaceAll(RegExp(r'```[\s\S]*?```'), '') // fenced code blocks
.replaceAllMapped(RegExp(r'`([^`]+)`'), (m) => m[1]!) // inline code
.replaceAll(RegExp(r'#{1,6}\s+'), '') // headings
.replaceAllMapped(RegExp(r'\*\*([^*]+)\*\*'), (m) => m[1]!) // bold
.replaceAllMapped(RegExp(r'\*([^*]+)\*'), (m) => m[1]!) // italic
.replaceAllMapped(
RegExp(r'\[([^\]]+)\]\([^)]+\)'), (m) => m[1]!) // links → text
.replaceAll(RegExp(r'^\s*[-*+]\s+', multiLine: true), '') // list markers
.replaceAll(RegExp(r'\n{2,}'), ' ') // multiple newlines → space
.replaceAll('\n', ' ')
.trim();
}
// ── State ─────────────────────────────────────────────────────────────────────
enum VoiceMode { idle, recording, transcribing, playing }
class VoiceState {
final VoiceMode mode;
final bool voiceModeActive;
final bool available;
const VoiceState({
this.mode = VoiceMode.idle,
this.voiceModeActive = false,
this.available = true,
});
VoiceState copyWith({
VoiceMode? mode,
bool? voiceModeActive,
bool? available,
}) =>
VoiceState(
mode: mode ?? this.mode,
voiceModeActive: voiceModeActive ?? this.voiceModeActive,
available: available ?? this.available,
);
}
// ── Provider ──────────────────────────────────────────────────────────────────
final voiceProvider =
NotifierProvider<VoiceNotifier, VoiceState>(VoiceNotifier.new);
// ── Notifier ──────────────────────────────────────────────────────────────────
class VoiceNotifier extends Notifier<VoiceState> {
// Audio I/O
AudioRecorder? _recorder;
AudioPlayer? _player;
StreamSubscription<Amplitude>? _amplitudeSubscription;
// Recording / silence detection
int _recordingStartMs = 0;
int _silenceMs = 0;
static const _silenceThresholdDb = -40.0;
static const _silenceDurationMs = 1500;
static const _minRecordingMs = 300;
// Voice mode callbacks
Future<void> Function(String transcript)? _onTranscript;
void Function(String message)? _onError;
bool _enableTts = false;
// Streaming TTS state
String _sentenceBuffer = '';
int _lastSeenLength = 0;
bool _streamComplete = false;
// Last complete assistant response — passed to Whisper as initial_prompt
// to reduce STT mishearings of domain-specific words.
String _lastAssistantContent = '';
// Empty transcript counter — show feedback after consecutive blanks
int _emptyTranscriptCount = 0;
// TTS playback queue
final _ttsQueue = Queue<Uint8List>();
bool _ttsPlaying = false;
int _ttsCounter = 0;
Directory? _tempDir;
@override
VoiceState build() {
_player = AudioPlayer();
ref.onDispose(() {
_amplitudeSubscription?.cancel();
_recorder?.dispose();
_player?.dispose();
});
return const VoiceState();
}
// ── Public API ──────────────────────────────────────────────────────────────
/// Enter voice mode. Checks server availability and mic permission first.
/// [onTranscript] is called with the transcript when a recording completes.
/// [enableTts] — if true, TTS plays when [feedContent] is called.
/// [onError] — called with a human-readable message on failure.
Future<void> enterVoiceMode({
required Future<void> Function(String transcript) onTranscript,
bool enableTts = false,
required void Function(String message) onError,
}) async {
if (state.voiceModeActive) {
exitVoiceMode();
return;
}
// Check server availability — STT is required, TTS is optional.
try {
final status = await ref.read(voiceRepositoryProvider).checkStatus();
if (!status.enabled || !status.stt) {
onError('Speech-to-text not available on this server');
return;
}
// Downgrade to STT-only when TTS is unavailable
if (!status.tts) enableTts = false;
} catch (_) {
onError('Could not reach voice service');
return;
}
// Check microphone permission
var permStatus = await Permission.microphone.request();
if (permStatus == PermissionStatus.permanentlyDenied) {
onError('Microphone blocked — opening settings');
final opened = await openAppSettings();
if (!opened) return;
// Re-check after user returns from settings
permStatus = await Permission.microphone.status;
}
if (!permStatus.isGranted) {
onError('Microphone permission required');
return;
}
_onTranscript = onTranscript;
_onError = onError;
_enableTts = enableTts;
_emptyTranscriptCount = 0;
_tempDir = await getTemporaryDirectory();
state = state.copyWith(voiceModeActive: true, available: true);
await _startListening();
}
/// Exit voice mode, stop all recording and TTS.
void exitVoiceMode() {
_amplitudeSubscription?.cancel();
_amplitudeSubscription = null;
_recorder?.stop();
_player?.stop();
_ttsQueue.clear();
_ttsPlaying = false;
_sentenceBuffer = '';
_lastSeenLength = 0;
_streamComplete = false;
_onTranscript = null;
_onError = null;
state = const VoiceState();
}
/// Feed streaming assistant content for TTS synthesis.
/// Call from screens with the full [fullContent] string on each update.
/// Set [isComplete] to true when the stream has finished.
void feedContent(String fullContent, {required bool isComplete}) {
if (!state.voiceModeActive || !_enableTts) return;
final delta = fullContent.length > _lastSeenLength
? fullContent.substring(_lastSeenLength)
: '';
_lastSeenLength = fullContent.length;
_sentenceBuffer += delta;
_dispatchSentences(flush: isComplete);
if (isComplete) {
_lastAssistantContent = fullContent;
_streamComplete = true;
_checkRestartListening();
}
}
// ── Internal recording ──────────────────────────────────────────────────────
Future<void> _startListening() async {
if (!state.voiceModeActive) return;
_silenceMs = 0;
_recordingStartMs = DateTime.now().millisecondsSinceEpoch;
state = state.copyWith(mode: VoiceMode.recording);
final dir = _tempDir ?? await getTemporaryDirectory();
final path =
'${dir.path}/voice_rec_${DateTime.now().millisecondsSinceEpoch}.m4a';
try {
// Recreate the recorder each session — the record package can leave
// the native AudioRecord in a bad state after stop/error cycles,
// and a stale instance is the most common cause of "could not start".
_recorder?.dispose();
_recorder = AudioRecorder();
if (!await _recorder!.hasPermission()) {
_onError?.call('Microphone permission was revoked');
exitVoiceMode();
return;
}
await _recorder!.start(
const RecordConfig(encoder: AudioEncoder.aacLc, sampleRate: 16000),
path: path,
);
_amplitudeSubscription?.cancel();
_amplitudeSubscription = _recorder!
.onAmplitudeChanged(const Duration(milliseconds: 200))
.listen(_onAmplitude);
} catch (e) {
_onError?.call('Microphone error: $e');
exitVoiceMode();
}
}
void _onAmplitude(Amplitude event) {
if (!state.voiceModeActive) return;
final elapsed =
DateTime.now().millisecondsSinceEpoch - _recordingStartMs;
if (elapsed < _minRecordingMs) return;
final db = event.current;
// Guard against NaN / ±Infinity which can arrive on some Android devices
// when the recorder is initialising. Treat invalid readings as silence.
final isSilent = db.isNaN || db.isInfinite || db < _silenceThresholdDb;
if (isSilent) {
_silenceMs += 200;
if (_silenceMs >= _silenceDurationMs) {
_amplitudeSubscription?.cancel();
_amplitudeSubscription = null;
_handleSilence();
}
} else {
_silenceMs = 0;
}
}
Future<void> _handleSilence() async {
if (!state.voiceModeActive) return;
state = state.copyWith(mode: VoiceMode.transcribing);
final path = await _recorder!.stop();
if (path == null || !state.voiceModeActive) return;
try {
final bytes = await File(path).readAsBytes();
await File(path).delete().catchError((_) => File(path));
if (!state.voiceModeActive) return;
final transcript = await ref.read(voiceRepositoryProvider).transcribe(
bytes,
context: _lastAssistantContent.isNotEmpty ? _lastAssistantContent : null,
);
if (!state.voiceModeActive) return;
if (transcript.isEmpty) {
_emptyTranscriptCount++;
if (_emptyTranscriptCount >= 3) {
_onError?.call('No speech detected — tap the mic to try again');
exitVoiceMode();
return;
}
await _startListening();
return;
}
_emptyTranscriptCount = 0;
// Reset TTS state for this new turn
_sentenceBuffer = '';
_lastSeenLength = 0;
_streamComplete = false;
if (_enableTts) {
state = state.copyWith(mode: VoiceMode.playing);
}
await _onTranscript?.call(transcript);
// If TTS is not enabled, loop immediately
if (!_enableTts && state.voiceModeActive) {
await _startListening();
}
} catch (e) {
_onError?.call('Voice error: transcription failed');
exitVoiceMode();
}
}
// ── Internal TTS ────────────────────────────────────────────────────────────
void _dispatchSentences({required bool flush}) {
final result = extractSentences(_sentenceBuffer);
_sentenceBuffer = flush ? '' : result.remainder;
for (final sentence in result.sentences) {
_enqueueSentence(sentence);
}
if (flush && result.remainder.trim().length >= 3) {
_enqueueSentence(result.remainder.trim());
}
}
void _enqueueSentence(String sentence) {
final stripped = stripMarkdownForTts(sentence);
if (stripped.length < 3) return;
_synthesiseSentence(stripped);
}
Future<void> _synthesiseSentence(String text) async {
try {
final wavBytes =
await ref.read(voiceRepositoryProvider).synthesise(text);
if (!state.voiceModeActive) return;
_ttsQueue.add(wavBytes);
if (!_ttsPlaying) _drainTtsQueue();
} catch (_) {
// Skip failed sentence — TTS errors are non-fatal
}
}
Future<void> _drainTtsQueue() async {
if (_ttsPlaying) return;
_ttsPlaying = true;
final dir = _tempDir ?? await getTemporaryDirectory();
try {
while (_ttsQueue.isNotEmpty && state.voiceModeActive) {
final wavBytes = _ttsQueue.removeFirst();
final path = '${dir.path}/tts_${_ttsCounter++}.wav';
final file = File(path);
await file.writeAsBytes(wavBytes);
try {
await _player!.setFilePath(path);
await _player!.play();
await _player!.processingStateStream.firstWhere(
(s) =>
s == ProcessingState.completed || s == ProcessingState.idle,
);
} finally {
await file.delete().catchError((_) => file);
}
}
} finally {
_ttsPlaying = false;
}
_checkRestartListening();
}
void _checkRestartListening() {
if (_streamComplete &&
_ttsQueue.isEmpty &&
!_ttsPlaying &&
state.voiceModeActive) {
_streamComplete = false;
_lastSeenLength = 0;
_sentenceBuffer = '';
_startListening();
}
}
}