48c134ce6a
VoiceState now carries a normalized mic amplitude (0..1) updated from the existing onAmplitudeChanged subscription, with a 0.02 change threshold so we don't spam rebuilds. VoiceMicButton swaps the constant-rate AnimationController for an AnimatedScale + animated glow driven by the live amplitude. 0.1 floor keeps the button breathing on silence; 120ms ease-out smooths between 200ms samples. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
435 lines
14 KiB
Dart
435 lines
14 KiB
Dart
import 'dart:async';
|
||
import 'dart:collection';
|
||
import 'dart:io';
|
||
import 'dart:typed_data';
|
||
|
||
import 'package:flutter_riverpod/flutter_riverpod.dart';
|
||
import 'package:just_audio/just_audio.dart';
|
||
import 'package:path_provider/path_provider.dart';
|
||
import 'package:permission_handler/permission_handler.dart';
|
||
import 'package:record/record.dart';
|
||
|
||
import 'api_client_provider.dart';
|
||
|
||
// ── Public helpers (also used by tests) ──────────────────────────────────────
|
||
|
||
class SentenceResult {
|
||
final List<String> sentences;
|
||
final String remainder;
|
||
const SentenceResult({required this.sentences, required this.remainder});
|
||
}
|
||
|
||
/// Extract completed sentences from [text] at `.`, `!`, `?` boundaries.
|
||
/// Returns the completed sentences and the unconsumed remainder.
|
||
SentenceResult extractSentences(String text) {
|
||
final boundary = RegExp(r'[.!?]+(?=\s|$)');
|
||
final sentences = <String>[];
|
||
var remaining = text;
|
||
RegExpMatch? match;
|
||
while ((match = boundary.firstMatch(remaining)) != null) {
|
||
final end = match!.end;
|
||
final sentence = remaining.substring(0, end).trim();
|
||
if (sentence.isNotEmpty) sentences.add(sentence);
|
||
remaining = remaining.substring(end).trimLeft();
|
||
}
|
||
return SentenceResult(sentences: sentences, remainder: remaining);
|
||
}
|
||
|
||
/// Strip markdown formatting before sending text to TTS.
|
||
String stripMarkdownForTts(String text) {
|
||
return text
|
||
.replaceAll(RegExp(r'```[\s\S]*?```'), '') // fenced code blocks
|
||
.replaceAllMapped(RegExp(r'`([^`]+)`'), (m) => m[1]!) // inline code
|
||
.replaceAll(RegExp(r'#{1,6}\s+'), '') // headings
|
||
.replaceAllMapped(RegExp(r'\*\*([^*]+)\*\*'), (m) => m[1]!) // bold
|
||
.replaceAllMapped(RegExp(r'\*([^*]+)\*'), (m) => m[1]!) // italic
|
||
.replaceAllMapped(
|
||
RegExp(r'\[([^\]]+)\]\([^)]+\)'), (m) => m[1]!) // links → text
|
||
.replaceAll(RegExp(r'^\s*[-*+]\s+', multiLine: true), '') // list markers
|
||
.replaceAll(RegExp(r'\n{2,}'), ' ') // multiple newlines → space
|
||
.replaceAll('\n', ' ')
|
||
.trim();
|
||
}
|
||
|
||
// ── State ─────────────────────────────────────────────────────────────────────
|
||
|
||
enum VoiceMode { idle, recording, transcribing, playing }
|
||
|
||
class VoiceState {
|
||
final VoiceMode mode;
|
||
final bool voiceModeActive;
|
||
final bool available;
|
||
/// Normalized mic amplitude 0.0–1.0 while recording. Drives the live
|
||
/// pulse on VoiceMicButton so the user has obvious feedback that audio
|
||
/// is actually being picked up.
|
||
final double amplitude;
|
||
|
||
const VoiceState({
|
||
this.mode = VoiceMode.idle,
|
||
this.voiceModeActive = false,
|
||
this.available = true,
|
||
this.amplitude = 0.0,
|
||
});
|
||
|
||
VoiceState copyWith({
|
||
VoiceMode? mode,
|
||
bool? voiceModeActive,
|
||
bool? available,
|
||
double? amplitude,
|
||
}) =>
|
||
VoiceState(
|
||
mode: mode ?? this.mode,
|
||
voiceModeActive: voiceModeActive ?? this.voiceModeActive,
|
||
available: available ?? this.available,
|
||
amplitude: amplitude ?? this.amplitude,
|
||
);
|
||
}
|
||
|
||
// ── Provider ──────────────────────────────────────────────────────────────────
|
||
|
||
final voiceProvider =
|
||
NotifierProvider<VoiceNotifier, VoiceState>(VoiceNotifier.new);
|
||
|
||
// ── Notifier ──────────────────────────────────────────────────────────────────
|
||
|
||
class VoiceNotifier extends Notifier<VoiceState> {
|
||
// Audio I/O
|
||
AudioRecorder? _recorder;
|
||
AudioPlayer? _player;
|
||
StreamSubscription<Amplitude>? _amplitudeSubscription;
|
||
|
||
// Recording / silence detection
|
||
int _recordingStartMs = 0;
|
||
int _silenceMs = 0;
|
||
static const _silenceThresholdDb = -40.0;
|
||
static const _silenceDurationMs = 1500;
|
||
static const _minRecordingMs = 300;
|
||
|
||
// Voice mode callbacks
|
||
Future<void> Function(String transcript)? _onTranscript;
|
||
void Function(String message)? _onError;
|
||
bool _enableTts = false;
|
||
|
||
// Streaming TTS state
|
||
String _sentenceBuffer = '';
|
||
int _lastSeenLength = 0;
|
||
bool _streamComplete = false;
|
||
|
||
// Last complete assistant response — passed to Whisper as initial_prompt
|
||
// to reduce STT mishearings of domain-specific words.
|
||
String _lastAssistantContent = '';
|
||
|
||
// Empty transcript counter — show feedback after consecutive blanks
|
||
int _emptyTranscriptCount = 0;
|
||
|
||
// TTS playback queue
|
||
final _ttsQueue = Queue<Uint8List>();
|
||
bool _ttsPlaying = false;
|
||
int _ttsCounter = 0;
|
||
Directory? _tempDir;
|
||
|
||
@override
|
||
VoiceState build() {
|
||
_player = AudioPlayer();
|
||
ref.onDispose(() {
|
||
_amplitudeSubscription?.cancel();
|
||
_recorder?.dispose();
|
||
_player?.dispose();
|
||
});
|
||
return const VoiceState();
|
||
}
|
||
|
||
// ── Public API ──────────────────────────────────────────────────────────────
|
||
|
||
/// Enter voice mode. Checks server availability and mic permission first.
|
||
/// [onTranscript] is called with the transcript when a recording completes.
|
||
/// [enableTts] — if true, TTS plays when [feedContent] is called.
|
||
/// [onError] — called with a human-readable message on failure.
|
||
Future<void> enterVoiceMode({
|
||
required Future<void> Function(String transcript) onTranscript,
|
||
bool enableTts = false,
|
||
required void Function(String message) onError,
|
||
}) async {
|
||
if (state.voiceModeActive) {
|
||
exitVoiceMode();
|
||
return;
|
||
}
|
||
|
||
// Check server availability — STT is required, TTS is optional.
|
||
try {
|
||
final status = await ref.read(voiceRepositoryProvider).checkStatus();
|
||
if (!status.enabled || !status.stt) {
|
||
onError('Speech-to-text not available on this server');
|
||
return;
|
||
}
|
||
// Downgrade to STT-only when TTS is unavailable
|
||
if (!status.tts) enableTts = false;
|
||
} catch (_) {
|
||
onError('Could not reach voice service');
|
||
return;
|
||
}
|
||
|
||
// Check microphone permission
|
||
var permStatus = await Permission.microphone.request();
|
||
if (permStatus == PermissionStatus.permanentlyDenied) {
|
||
onError('Microphone blocked — opening settings');
|
||
final opened = await openAppSettings();
|
||
if (!opened) return;
|
||
// Re-check after user returns from settings
|
||
permStatus = await Permission.microphone.status;
|
||
}
|
||
if (!permStatus.isGranted) {
|
||
onError('Microphone permission required');
|
||
return;
|
||
}
|
||
|
||
_onTranscript = onTranscript;
|
||
_onError = onError;
|
||
_enableTts = enableTts;
|
||
_emptyTranscriptCount = 0;
|
||
_tempDir = await getTemporaryDirectory();
|
||
|
||
state = state.copyWith(voiceModeActive: true, available: true);
|
||
await _startListening();
|
||
}
|
||
|
||
/// Exit voice mode, stop all recording and TTS.
|
||
void exitVoiceMode() {
|
||
_amplitudeSubscription?.cancel();
|
||
_amplitudeSubscription = null;
|
||
_recorder?.stop();
|
||
_player?.stop();
|
||
_ttsQueue.clear();
|
||
_ttsPlaying = false;
|
||
_sentenceBuffer = '';
|
||
_lastSeenLength = 0;
|
||
_streamComplete = false;
|
||
_onTranscript = null;
|
||
_onError = null;
|
||
state = const VoiceState();
|
||
}
|
||
|
||
/// Feed streaming assistant content for TTS synthesis.
|
||
/// Call from screens with the full [fullContent] string on each update.
|
||
/// Set [isComplete] to true when the stream has finished.
|
||
void feedContent(String fullContent, {required bool isComplete}) {
|
||
if (!state.voiceModeActive || !_enableTts) return;
|
||
|
||
final delta = fullContent.length > _lastSeenLength
|
||
? fullContent.substring(_lastSeenLength)
|
||
: '';
|
||
_lastSeenLength = fullContent.length;
|
||
_sentenceBuffer += delta;
|
||
|
||
_dispatchSentences(flush: isComplete);
|
||
|
||
if (isComplete) {
|
||
_lastAssistantContent = fullContent;
|
||
_streamComplete = true;
|
||
_checkRestartListening();
|
||
}
|
||
}
|
||
|
||
// ── Internal recording ──────────────────────────────────────────────────────
|
||
|
||
Future<void> _startListening() async {
|
||
if (!state.voiceModeActive) return;
|
||
|
||
_silenceMs = 0;
|
||
_recordingStartMs = DateTime.now().millisecondsSinceEpoch;
|
||
state = state.copyWith(mode: VoiceMode.recording);
|
||
|
||
final dir = _tempDir ?? await getTemporaryDirectory();
|
||
final path =
|
||
'${dir.path}/voice_rec_${DateTime.now().millisecondsSinceEpoch}.m4a';
|
||
|
||
try {
|
||
// Recreate the recorder each session — the record package can leave
|
||
// the native AudioRecord in a bad state after stop/error cycles,
|
||
// and a stale instance is the most common cause of "could not start".
|
||
_recorder?.dispose();
|
||
_recorder = AudioRecorder();
|
||
|
||
if (!await _recorder!.hasPermission()) {
|
||
_onError?.call('Microphone permission was revoked');
|
||
exitVoiceMode();
|
||
return;
|
||
}
|
||
|
||
await _recorder!.start(
|
||
const RecordConfig(encoder: AudioEncoder.aacLc, sampleRate: 16000),
|
||
path: path,
|
||
);
|
||
|
||
_amplitudeSubscription?.cancel();
|
||
_amplitudeSubscription = _recorder!
|
||
.onAmplitudeChanged(const Duration(milliseconds: 200))
|
||
.listen(_onAmplitude);
|
||
} catch (e) {
|
||
_onError?.call('Microphone error: $e');
|
||
exitVoiceMode();
|
||
}
|
||
}
|
||
|
||
void _onAmplitude(Amplitude event) {
|
||
if (!state.voiceModeActive) return;
|
||
|
||
final db = event.current;
|
||
final validDb = !(db.isNaN || db.isInfinite);
|
||
|
||
// Normalize dB to 0..1 for the pulse animation. -60dB is dead-quiet,
|
||
// 0dB is peak; we clamp and bias so the button visibly breathes even
|
||
// on soft speech without exploding on loud input.
|
||
if (validDb) {
|
||
final norm = ((db + 60.0) / 60.0).clamp(0.0, 1.0);
|
||
if ((norm - state.amplitude).abs() > 0.02) {
|
||
state = state.copyWith(amplitude: norm);
|
||
}
|
||
}
|
||
|
||
final elapsed =
|
||
DateTime.now().millisecondsSinceEpoch - _recordingStartMs;
|
||
if (elapsed < _minRecordingMs) return;
|
||
|
||
final isSilent = !validDb || db < _silenceThresholdDb;
|
||
|
||
if (isSilent) {
|
||
_silenceMs += 200;
|
||
if (_silenceMs >= _silenceDurationMs) {
|
||
_amplitudeSubscription?.cancel();
|
||
_amplitudeSubscription = null;
|
||
_handleSilence();
|
||
}
|
||
} else {
|
||
_silenceMs = 0;
|
||
}
|
||
}
|
||
|
||
Future<void> _handleSilence() async {
|
||
if (!state.voiceModeActive) return;
|
||
state = state.copyWith(mode: VoiceMode.transcribing);
|
||
|
||
final path = await _recorder!.stop();
|
||
if (path == null || !state.voiceModeActive) return;
|
||
|
||
try {
|
||
final bytes = await File(path).readAsBytes();
|
||
await File(path).delete().catchError((_) => File(path));
|
||
|
||
if (!state.voiceModeActive) return;
|
||
|
||
final transcript = await ref.read(voiceRepositoryProvider).transcribe(
|
||
bytes,
|
||
context: _lastAssistantContent.isNotEmpty ? _lastAssistantContent : null,
|
||
);
|
||
|
||
if (!state.voiceModeActive) return;
|
||
|
||
if (transcript.isEmpty) {
|
||
_emptyTranscriptCount++;
|
||
if (_emptyTranscriptCount >= 3) {
|
||
_onError?.call('No speech detected — tap the mic to try again');
|
||
exitVoiceMode();
|
||
return;
|
||
}
|
||
await _startListening();
|
||
return;
|
||
}
|
||
_emptyTranscriptCount = 0;
|
||
|
||
// Reset TTS state for this new turn
|
||
_sentenceBuffer = '';
|
||
_lastSeenLength = 0;
|
||
_streamComplete = false;
|
||
|
||
if (_enableTts) {
|
||
state = state.copyWith(mode: VoiceMode.playing);
|
||
}
|
||
|
||
await _onTranscript?.call(transcript);
|
||
|
||
// If TTS is not enabled, loop immediately
|
||
if (!_enableTts && state.voiceModeActive) {
|
||
await _startListening();
|
||
}
|
||
} catch (e) {
|
||
_onError?.call('Voice error: transcription failed');
|
||
exitVoiceMode();
|
||
}
|
||
}
|
||
|
||
// ── Internal TTS ────────────────────────────────────────────────────────────
|
||
|
||
void _dispatchSentences({required bool flush}) {
|
||
final result = extractSentences(_sentenceBuffer);
|
||
_sentenceBuffer = flush ? '' : result.remainder;
|
||
|
||
for (final sentence in result.sentences) {
|
||
_enqueueSentence(sentence);
|
||
}
|
||
if (flush && result.remainder.trim().length >= 3) {
|
||
_enqueueSentence(result.remainder.trim());
|
||
}
|
||
}
|
||
|
||
void _enqueueSentence(String sentence) {
|
||
final stripped = stripMarkdownForTts(sentence);
|
||
if (stripped.length < 3) return;
|
||
_synthesiseSentence(stripped);
|
||
}
|
||
|
||
Future<void> _synthesiseSentence(String text) async {
|
||
try {
|
||
final wavBytes =
|
||
await ref.read(voiceRepositoryProvider).synthesise(text);
|
||
if (!state.voiceModeActive) return;
|
||
_ttsQueue.add(wavBytes);
|
||
if (!_ttsPlaying) _drainTtsQueue();
|
||
} catch (_) {
|
||
// Skip failed sentence — TTS errors are non-fatal
|
||
}
|
||
}
|
||
|
||
Future<void> _drainTtsQueue() async {
|
||
if (_ttsPlaying) return;
|
||
_ttsPlaying = true;
|
||
final dir = _tempDir ?? await getTemporaryDirectory();
|
||
|
||
try {
|
||
while (_ttsQueue.isNotEmpty && state.voiceModeActive) {
|
||
final wavBytes = _ttsQueue.removeFirst();
|
||
final path = '${dir.path}/tts_${_ttsCounter++}.wav';
|
||
final file = File(path);
|
||
await file.writeAsBytes(wavBytes);
|
||
|
||
try {
|
||
await _player!.setFilePath(path);
|
||
await _player!.play();
|
||
await _player!.processingStateStream.firstWhere(
|
||
(s) =>
|
||
s == ProcessingState.completed || s == ProcessingState.idle,
|
||
);
|
||
} finally {
|
||
await file.delete().catchError((_) => file);
|
||
}
|
||
}
|
||
} finally {
|
||
_ttsPlaying = false;
|
||
}
|
||
|
||
_checkRestartListening();
|
||
}
|
||
|
||
void _checkRestartListening() {
|
||
if (_streamComplete &&
|
||
_ttsQueue.isEmpty &&
|
||
!_ttsPlaying &&
|
||
state.voiceModeActive) {
|
||
_streamComplete = false;
|
||
_lastSeenLength = 0;
|
||
_sentenceBuffer = '';
|
||
_startListening();
|
||
}
|
||
}
|
||
}
|