This repository has been archived on 2026-06-02. You can view files and clone it. You cannot open issues or pull requests or push a commit.
Files
FabledApp/lib/providers/voice_provider.dart
T
bvandeusen 48c134ce6a feat(voice): pulse mic button with live amplitude
VoiceState now carries a normalized mic amplitude (0..1) updated
from the existing onAmplitudeChanged subscription, with a 0.02
change threshold so we don't spam rebuilds.

VoiceMicButton swaps the constant-rate AnimationController for an
AnimatedScale + animated glow driven by the live amplitude. 0.1
floor keeps the button breathing on silence; 120ms ease-out smooths
between 200ms samples.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-04-14 22:46:48 -04:00

435 lines
14 KiB
Dart
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import 'dart:async';
import 'dart:collection';
import 'dart:io';
import 'dart:typed_data';
import 'package:flutter_riverpod/flutter_riverpod.dart';
import 'package:just_audio/just_audio.dart';
import 'package:path_provider/path_provider.dart';
import 'package:permission_handler/permission_handler.dart';
import 'package:record/record.dart';
import 'api_client_provider.dart';
// ── Public helpers (also used by tests) ──────────────────────────────────────
class SentenceResult {
final List<String> sentences;
final String remainder;
const SentenceResult({required this.sentences, required this.remainder});
}
/// Extract completed sentences from [text] at `.`, `!`, `?` boundaries.
/// Returns the completed sentences and the unconsumed remainder.
SentenceResult extractSentences(String text) {
final boundary = RegExp(r'[.!?]+(?=\s|$)');
final sentences = <String>[];
var remaining = text;
RegExpMatch? match;
while ((match = boundary.firstMatch(remaining)) != null) {
final end = match!.end;
final sentence = remaining.substring(0, end).trim();
if (sentence.isNotEmpty) sentences.add(sentence);
remaining = remaining.substring(end).trimLeft();
}
return SentenceResult(sentences: sentences, remainder: remaining);
}
/// Strip markdown formatting before sending text to TTS.
String stripMarkdownForTts(String text) {
return text
.replaceAll(RegExp(r'```[\s\S]*?```'), '') // fenced code blocks
.replaceAllMapped(RegExp(r'`([^`]+)`'), (m) => m[1]!) // inline code
.replaceAll(RegExp(r'#{1,6}\s+'), '') // headings
.replaceAllMapped(RegExp(r'\*\*([^*]+)\*\*'), (m) => m[1]!) // bold
.replaceAllMapped(RegExp(r'\*([^*]+)\*'), (m) => m[1]!) // italic
.replaceAllMapped(
RegExp(r'\[([^\]]+)\]\([^)]+\)'), (m) => m[1]!) // links → text
.replaceAll(RegExp(r'^\s*[-*+]\s+', multiLine: true), '') // list markers
.replaceAll(RegExp(r'\n{2,}'), ' ') // multiple newlines → space
.replaceAll('\n', ' ')
.trim();
}
// ── State ─────────────────────────────────────────────────────────────────────
enum VoiceMode { idle, recording, transcribing, playing }
class VoiceState {
final VoiceMode mode;
final bool voiceModeActive;
final bool available;
/// Normalized mic amplitude 0.01.0 while recording. Drives the live
/// pulse on VoiceMicButton so the user has obvious feedback that audio
/// is actually being picked up.
final double amplitude;
const VoiceState({
this.mode = VoiceMode.idle,
this.voiceModeActive = false,
this.available = true,
this.amplitude = 0.0,
});
VoiceState copyWith({
VoiceMode? mode,
bool? voiceModeActive,
bool? available,
double? amplitude,
}) =>
VoiceState(
mode: mode ?? this.mode,
voiceModeActive: voiceModeActive ?? this.voiceModeActive,
available: available ?? this.available,
amplitude: amplitude ?? this.amplitude,
);
}
// ── Provider ──────────────────────────────────────────────────────────────────
final voiceProvider =
NotifierProvider<VoiceNotifier, VoiceState>(VoiceNotifier.new);
// ── Notifier ──────────────────────────────────────────────────────────────────
class VoiceNotifier extends Notifier<VoiceState> {
// Audio I/O
AudioRecorder? _recorder;
AudioPlayer? _player;
StreamSubscription<Amplitude>? _amplitudeSubscription;
// Recording / silence detection
int _recordingStartMs = 0;
int _silenceMs = 0;
static const _silenceThresholdDb = -40.0;
static const _silenceDurationMs = 1500;
static const _minRecordingMs = 300;
// Voice mode callbacks
Future<void> Function(String transcript)? _onTranscript;
void Function(String message)? _onError;
bool _enableTts = false;
// Streaming TTS state
String _sentenceBuffer = '';
int _lastSeenLength = 0;
bool _streamComplete = false;
// Last complete assistant response — passed to Whisper as initial_prompt
// to reduce STT mishearings of domain-specific words.
String _lastAssistantContent = '';
// Empty transcript counter — show feedback after consecutive blanks
int _emptyTranscriptCount = 0;
// TTS playback queue
final _ttsQueue = Queue<Uint8List>();
bool _ttsPlaying = false;
int _ttsCounter = 0;
Directory? _tempDir;
@override
VoiceState build() {
_player = AudioPlayer();
ref.onDispose(() {
_amplitudeSubscription?.cancel();
_recorder?.dispose();
_player?.dispose();
});
return const VoiceState();
}
// ── Public API ──────────────────────────────────────────────────────────────
/// Enter voice mode. Checks server availability and mic permission first.
/// [onTranscript] is called with the transcript when a recording completes.
/// [enableTts] — if true, TTS plays when [feedContent] is called.
/// [onError] — called with a human-readable message on failure.
Future<void> enterVoiceMode({
required Future<void> Function(String transcript) onTranscript,
bool enableTts = false,
required void Function(String message) onError,
}) async {
if (state.voiceModeActive) {
exitVoiceMode();
return;
}
// Check server availability — STT is required, TTS is optional.
try {
final status = await ref.read(voiceRepositoryProvider).checkStatus();
if (!status.enabled || !status.stt) {
onError('Speech-to-text not available on this server');
return;
}
// Downgrade to STT-only when TTS is unavailable
if (!status.tts) enableTts = false;
} catch (_) {
onError('Could not reach voice service');
return;
}
// Check microphone permission
var permStatus = await Permission.microphone.request();
if (permStatus == PermissionStatus.permanentlyDenied) {
onError('Microphone blocked — opening settings');
final opened = await openAppSettings();
if (!opened) return;
// Re-check after user returns from settings
permStatus = await Permission.microphone.status;
}
if (!permStatus.isGranted) {
onError('Microphone permission required');
return;
}
_onTranscript = onTranscript;
_onError = onError;
_enableTts = enableTts;
_emptyTranscriptCount = 0;
_tempDir = await getTemporaryDirectory();
state = state.copyWith(voiceModeActive: true, available: true);
await _startListening();
}
/// Exit voice mode, stop all recording and TTS.
void exitVoiceMode() {
_amplitudeSubscription?.cancel();
_amplitudeSubscription = null;
_recorder?.stop();
_player?.stop();
_ttsQueue.clear();
_ttsPlaying = false;
_sentenceBuffer = '';
_lastSeenLength = 0;
_streamComplete = false;
_onTranscript = null;
_onError = null;
state = const VoiceState();
}
/// Feed streaming assistant content for TTS synthesis.
/// Call from screens with the full [fullContent] string on each update.
/// Set [isComplete] to true when the stream has finished.
void feedContent(String fullContent, {required bool isComplete}) {
if (!state.voiceModeActive || !_enableTts) return;
final delta = fullContent.length > _lastSeenLength
? fullContent.substring(_lastSeenLength)
: '';
_lastSeenLength = fullContent.length;
_sentenceBuffer += delta;
_dispatchSentences(flush: isComplete);
if (isComplete) {
_lastAssistantContent = fullContent;
_streamComplete = true;
_checkRestartListening();
}
}
// ── Internal recording ──────────────────────────────────────────────────────
Future<void> _startListening() async {
if (!state.voiceModeActive) return;
_silenceMs = 0;
_recordingStartMs = DateTime.now().millisecondsSinceEpoch;
state = state.copyWith(mode: VoiceMode.recording);
final dir = _tempDir ?? await getTemporaryDirectory();
final path =
'${dir.path}/voice_rec_${DateTime.now().millisecondsSinceEpoch}.m4a';
try {
// Recreate the recorder each session — the record package can leave
// the native AudioRecord in a bad state after stop/error cycles,
// and a stale instance is the most common cause of "could not start".
_recorder?.dispose();
_recorder = AudioRecorder();
if (!await _recorder!.hasPermission()) {
_onError?.call('Microphone permission was revoked');
exitVoiceMode();
return;
}
await _recorder!.start(
const RecordConfig(encoder: AudioEncoder.aacLc, sampleRate: 16000),
path: path,
);
_amplitudeSubscription?.cancel();
_amplitudeSubscription = _recorder!
.onAmplitudeChanged(const Duration(milliseconds: 200))
.listen(_onAmplitude);
} catch (e) {
_onError?.call('Microphone error: $e');
exitVoiceMode();
}
}
void _onAmplitude(Amplitude event) {
if (!state.voiceModeActive) return;
final db = event.current;
final validDb = !(db.isNaN || db.isInfinite);
// Normalize dB to 0..1 for the pulse animation. -60dB is dead-quiet,
// 0dB is peak; we clamp and bias so the button visibly breathes even
// on soft speech without exploding on loud input.
if (validDb) {
final norm = ((db + 60.0) / 60.0).clamp(0.0, 1.0);
if ((norm - state.amplitude).abs() > 0.02) {
state = state.copyWith(amplitude: norm);
}
}
final elapsed =
DateTime.now().millisecondsSinceEpoch - _recordingStartMs;
if (elapsed < _minRecordingMs) return;
final isSilent = !validDb || db < _silenceThresholdDb;
if (isSilent) {
_silenceMs += 200;
if (_silenceMs >= _silenceDurationMs) {
_amplitudeSubscription?.cancel();
_amplitudeSubscription = null;
_handleSilence();
}
} else {
_silenceMs = 0;
}
}
Future<void> _handleSilence() async {
if (!state.voiceModeActive) return;
state = state.copyWith(mode: VoiceMode.transcribing);
final path = await _recorder!.stop();
if (path == null || !state.voiceModeActive) return;
try {
final bytes = await File(path).readAsBytes();
await File(path).delete().catchError((_) => File(path));
if (!state.voiceModeActive) return;
final transcript = await ref.read(voiceRepositoryProvider).transcribe(
bytes,
context: _lastAssistantContent.isNotEmpty ? _lastAssistantContent : null,
);
if (!state.voiceModeActive) return;
if (transcript.isEmpty) {
_emptyTranscriptCount++;
if (_emptyTranscriptCount >= 3) {
_onError?.call('No speech detected — tap the mic to try again');
exitVoiceMode();
return;
}
await _startListening();
return;
}
_emptyTranscriptCount = 0;
// Reset TTS state for this new turn
_sentenceBuffer = '';
_lastSeenLength = 0;
_streamComplete = false;
if (_enableTts) {
state = state.copyWith(mode: VoiceMode.playing);
}
await _onTranscript?.call(transcript);
// If TTS is not enabled, loop immediately
if (!_enableTts && state.voiceModeActive) {
await _startListening();
}
} catch (e) {
_onError?.call('Voice error: transcription failed');
exitVoiceMode();
}
}
// ── Internal TTS ────────────────────────────────────────────────────────────
void _dispatchSentences({required bool flush}) {
final result = extractSentences(_sentenceBuffer);
_sentenceBuffer = flush ? '' : result.remainder;
for (final sentence in result.sentences) {
_enqueueSentence(sentence);
}
if (flush && result.remainder.trim().length >= 3) {
_enqueueSentence(result.remainder.trim());
}
}
void _enqueueSentence(String sentence) {
final stripped = stripMarkdownForTts(sentence);
if (stripped.length < 3) return;
_synthesiseSentence(stripped);
}
Future<void> _synthesiseSentence(String text) async {
try {
final wavBytes =
await ref.read(voiceRepositoryProvider).synthesise(text);
if (!state.voiceModeActive) return;
_ttsQueue.add(wavBytes);
if (!_ttsPlaying) _drainTtsQueue();
} catch (_) {
// Skip failed sentence — TTS errors are non-fatal
}
}
Future<void> _drainTtsQueue() async {
if (_ttsPlaying) return;
_ttsPlaying = true;
final dir = _tempDir ?? await getTemporaryDirectory();
try {
while (_ttsQueue.isNotEmpty && state.voiceModeActive) {
final wavBytes = _ttsQueue.removeFirst();
final path = '${dir.path}/tts_${_ttsCounter++}.wav';
final file = File(path);
await file.writeAsBytes(wavBytes);
try {
await _player!.setFilePath(path);
await _player!.play();
await _player!.processingStateStream.firstWhere(
(s) =>
s == ProcessingState.completed || s == ProcessingState.idle,
);
} finally {
await file.delete().catchError((_) => file);
}
}
} finally {
_ttsPlaying = false;
}
_checkRestartListening();
}
void _checkRestartListening() {
if (_streamComplete &&
_ttsQueue.isEmpty &&
!_ttsPlaying &&
state.voiceModeActive) {
_streamComplete = false;
_lastSeenLength = 0;
_sentenceBuffer = '';
_startListening();
}
}
}