4240c90d55
Voice: recreate AudioRecorder each session to avoid stale native state, improve permission handling (re-check after settings), show feedback after 3 consecutive empty transcripts, allow STT-only mode when TTS is unavailable. Knowledge: hydrate IDs immediately after loading more pages so scroll-based pagination doesn't stall at the bottom. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
418 lines
13 KiB
Dart
418 lines
13 KiB
Dart
import 'dart:async';
|
|
import 'dart:collection';
|
|
import 'dart:io';
|
|
import 'dart:typed_data';
|
|
|
|
import 'package:flutter_riverpod/flutter_riverpod.dart';
|
|
import 'package:just_audio/just_audio.dart';
|
|
import 'package:path_provider/path_provider.dart';
|
|
import 'package:permission_handler/permission_handler.dart';
|
|
import 'package:record/record.dart';
|
|
|
|
import 'api_client_provider.dart';
|
|
|
|
// ── Public helpers (also used by tests) ──────────────────────────────────────
|
|
|
|
class SentenceResult {
|
|
final List<String> sentences;
|
|
final String remainder;
|
|
const SentenceResult({required this.sentences, required this.remainder});
|
|
}
|
|
|
|
/// Extract completed sentences from [text] at `.`, `!`, `?` boundaries.
|
|
/// Returns the completed sentences and the unconsumed remainder.
|
|
SentenceResult extractSentences(String text) {
|
|
final boundary = RegExp(r'[.!?]+(?=\s|$)');
|
|
final sentences = <String>[];
|
|
var remaining = text;
|
|
RegExpMatch? match;
|
|
while ((match = boundary.firstMatch(remaining)) != null) {
|
|
final end = match!.end;
|
|
final sentence = remaining.substring(0, end).trim();
|
|
if (sentence.isNotEmpty) sentences.add(sentence);
|
|
remaining = remaining.substring(end).trimLeft();
|
|
}
|
|
return SentenceResult(sentences: sentences, remainder: remaining);
|
|
}
|
|
|
|
/// Strip markdown formatting before sending text to TTS.
|
|
String stripMarkdownForTts(String text) {
|
|
return text
|
|
.replaceAll(RegExp(r'```[\s\S]*?```'), '') // fenced code blocks
|
|
.replaceAllMapped(RegExp(r'`([^`]+)`'), (m) => m[1]!) // inline code
|
|
.replaceAll(RegExp(r'#{1,6}\s+'), '') // headings
|
|
.replaceAllMapped(RegExp(r'\*\*([^*]+)\*\*'), (m) => m[1]!) // bold
|
|
.replaceAllMapped(RegExp(r'\*([^*]+)\*'), (m) => m[1]!) // italic
|
|
.replaceAllMapped(
|
|
RegExp(r'\[([^\]]+)\]\([^)]+\)'), (m) => m[1]!) // links → text
|
|
.replaceAll(RegExp(r'^\s*[-*+]\s+', multiLine: true), '') // list markers
|
|
.replaceAll(RegExp(r'\n{2,}'), ' ') // multiple newlines → space
|
|
.replaceAll('\n', ' ')
|
|
.trim();
|
|
}
|
|
|
|
// ── State ─────────────────────────────────────────────────────────────────────
|
|
|
|
enum VoiceMode { idle, recording, transcribing, playing }
|
|
|
|
class VoiceState {
|
|
final VoiceMode mode;
|
|
final bool voiceModeActive;
|
|
final bool available;
|
|
|
|
const VoiceState({
|
|
this.mode = VoiceMode.idle,
|
|
this.voiceModeActive = false,
|
|
this.available = true,
|
|
});
|
|
|
|
VoiceState copyWith({
|
|
VoiceMode? mode,
|
|
bool? voiceModeActive,
|
|
bool? available,
|
|
}) =>
|
|
VoiceState(
|
|
mode: mode ?? this.mode,
|
|
voiceModeActive: voiceModeActive ?? this.voiceModeActive,
|
|
available: available ?? this.available,
|
|
);
|
|
}
|
|
|
|
// ── Provider ──────────────────────────────────────────────────────────────────
|
|
|
|
final voiceProvider =
|
|
NotifierProvider<VoiceNotifier, VoiceState>(VoiceNotifier.new);
|
|
|
|
// ── Notifier ──────────────────────────────────────────────────────────────────
|
|
|
|
class VoiceNotifier extends Notifier<VoiceState> {
|
|
// Audio I/O
|
|
AudioRecorder? _recorder;
|
|
AudioPlayer? _player;
|
|
StreamSubscription<Amplitude>? _amplitudeSubscription;
|
|
|
|
// Recording / silence detection
|
|
int _recordingStartMs = 0;
|
|
int _silenceMs = 0;
|
|
static const _silenceThresholdDb = -40.0;
|
|
static const _silenceDurationMs = 1500;
|
|
static const _minRecordingMs = 300;
|
|
|
|
// Voice mode callbacks
|
|
Future<void> Function(String transcript)? _onTranscript;
|
|
void Function(String message)? _onError;
|
|
bool _enableTts = false;
|
|
|
|
// Streaming TTS state
|
|
String _sentenceBuffer = '';
|
|
int _lastSeenLength = 0;
|
|
bool _streamComplete = false;
|
|
|
|
// Last complete assistant response — passed to Whisper as initial_prompt
|
|
// to reduce STT mishearings of domain-specific words.
|
|
String _lastAssistantContent = '';
|
|
|
|
// Empty transcript counter — show feedback after consecutive blanks
|
|
int _emptyTranscriptCount = 0;
|
|
|
|
// TTS playback queue
|
|
final _ttsQueue = Queue<Uint8List>();
|
|
bool _ttsPlaying = false;
|
|
int _ttsCounter = 0;
|
|
Directory? _tempDir;
|
|
|
|
@override
|
|
VoiceState build() {
|
|
_player = AudioPlayer();
|
|
ref.onDispose(() {
|
|
_amplitudeSubscription?.cancel();
|
|
_recorder?.dispose();
|
|
_player?.dispose();
|
|
});
|
|
return const VoiceState();
|
|
}
|
|
|
|
// ── Public API ──────────────────────────────────────────────────────────────
|
|
|
|
/// Enter voice mode. Checks server availability and mic permission first.
|
|
/// [onTranscript] is called with the transcript when a recording completes.
|
|
/// [enableTts] — if true, TTS plays when [feedContent] is called.
|
|
/// [onError] — called with a human-readable message on failure.
|
|
Future<void> enterVoiceMode({
|
|
required Future<void> Function(String transcript) onTranscript,
|
|
bool enableTts = false,
|
|
required void Function(String message) onError,
|
|
}) async {
|
|
if (state.voiceModeActive) {
|
|
exitVoiceMode();
|
|
return;
|
|
}
|
|
|
|
// Check server availability — STT is required, TTS is optional.
|
|
try {
|
|
final status = await ref.read(voiceRepositoryProvider).checkStatus();
|
|
if (!status.enabled || !status.stt) {
|
|
onError('Speech-to-text not available on this server');
|
|
return;
|
|
}
|
|
// Downgrade to STT-only when TTS is unavailable
|
|
if (!status.tts) enableTts = false;
|
|
} catch (_) {
|
|
onError('Could not reach voice service');
|
|
return;
|
|
}
|
|
|
|
// Check microphone permission
|
|
var permStatus = await Permission.microphone.request();
|
|
if (permStatus == PermissionStatus.permanentlyDenied) {
|
|
onError('Microphone blocked — opening settings');
|
|
final opened = await openAppSettings();
|
|
if (!opened) return;
|
|
// Re-check after user returns from settings
|
|
permStatus = await Permission.microphone.status;
|
|
}
|
|
if (!permStatus.isGranted) {
|
|
onError('Microphone permission required');
|
|
return;
|
|
}
|
|
|
|
_onTranscript = onTranscript;
|
|
_onError = onError;
|
|
_enableTts = enableTts;
|
|
_emptyTranscriptCount = 0;
|
|
_tempDir = await getTemporaryDirectory();
|
|
|
|
state = state.copyWith(voiceModeActive: true, available: true);
|
|
await _startListening();
|
|
}
|
|
|
|
/// Exit voice mode, stop all recording and TTS.
|
|
void exitVoiceMode() {
|
|
_amplitudeSubscription?.cancel();
|
|
_amplitudeSubscription = null;
|
|
_recorder?.stop();
|
|
_player?.stop();
|
|
_ttsQueue.clear();
|
|
_ttsPlaying = false;
|
|
_sentenceBuffer = '';
|
|
_lastSeenLength = 0;
|
|
_streamComplete = false;
|
|
_onTranscript = null;
|
|
_onError = null;
|
|
state = const VoiceState();
|
|
}
|
|
|
|
/// Feed streaming assistant content for TTS synthesis.
|
|
/// Call from screens with the full [fullContent] string on each update.
|
|
/// Set [isComplete] to true when the stream has finished.
|
|
void feedContent(String fullContent, {required bool isComplete}) {
|
|
if (!state.voiceModeActive || !_enableTts) return;
|
|
|
|
final delta = fullContent.length > _lastSeenLength
|
|
? fullContent.substring(_lastSeenLength)
|
|
: '';
|
|
_lastSeenLength = fullContent.length;
|
|
_sentenceBuffer += delta;
|
|
|
|
_dispatchSentences(flush: isComplete);
|
|
|
|
if (isComplete) {
|
|
_lastAssistantContent = fullContent;
|
|
_streamComplete = true;
|
|
_checkRestartListening();
|
|
}
|
|
}
|
|
|
|
// ── Internal recording ──────────────────────────────────────────────────────
|
|
|
|
Future<void> _startListening() async {
|
|
if (!state.voiceModeActive) return;
|
|
|
|
_silenceMs = 0;
|
|
_recordingStartMs = DateTime.now().millisecondsSinceEpoch;
|
|
state = state.copyWith(mode: VoiceMode.recording);
|
|
|
|
final dir = _tempDir ?? await getTemporaryDirectory();
|
|
final path =
|
|
'${dir.path}/voice_rec_${DateTime.now().millisecondsSinceEpoch}.m4a';
|
|
|
|
try {
|
|
// Recreate the recorder each session — the record package can leave
|
|
// the native AudioRecord in a bad state after stop/error cycles,
|
|
// and a stale instance is the most common cause of "could not start".
|
|
_recorder?.dispose();
|
|
_recorder = AudioRecorder();
|
|
|
|
if (!await _recorder!.hasPermission()) {
|
|
_onError?.call('Microphone permission was revoked');
|
|
exitVoiceMode();
|
|
return;
|
|
}
|
|
|
|
await _recorder!.start(
|
|
const RecordConfig(encoder: AudioEncoder.aacLc, sampleRate: 16000),
|
|
path: path,
|
|
);
|
|
|
|
_amplitudeSubscription?.cancel();
|
|
_amplitudeSubscription = _recorder!
|
|
.onAmplitudeChanged(const Duration(milliseconds: 200))
|
|
.listen(_onAmplitude);
|
|
} catch (e) {
|
|
_onError?.call('Microphone error: $e');
|
|
exitVoiceMode();
|
|
}
|
|
}
|
|
|
|
void _onAmplitude(Amplitude event) {
|
|
if (!state.voiceModeActive) return;
|
|
|
|
final elapsed =
|
|
DateTime.now().millisecondsSinceEpoch - _recordingStartMs;
|
|
if (elapsed < _minRecordingMs) return;
|
|
|
|
final db = event.current;
|
|
// Guard against NaN / ±Infinity which can arrive on some Android devices
|
|
// when the recorder is initialising. Treat invalid readings as silence.
|
|
final isSilent = db.isNaN || db.isInfinite || db < _silenceThresholdDb;
|
|
|
|
if (isSilent) {
|
|
_silenceMs += 200;
|
|
if (_silenceMs >= _silenceDurationMs) {
|
|
_amplitudeSubscription?.cancel();
|
|
_amplitudeSubscription = null;
|
|
_handleSilence();
|
|
}
|
|
} else {
|
|
_silenceMs = 0;
|
|
}
|
|
}
|
|
|
|
Future<void> _handleSilence() async {
|
|
if (!state.voiceModeActive) return;
|
|
state = state.copyWith(mode: VoiceMode.transcribing);
|
|
|
|
final path = await _recorder!.stop();
|
|
if (path == null || !state.voiceModeActive) return;
|
|
|
|
try {
|
|
final bytes = await File(path).readAsBytes();
|
|
await File(path).delete().catchError((_) => File(path));
|
|
|
|
if (!state.voiceModeActive) return;
|
|
|
|
final transcript = await ref.read(voiceRepositoryProvider).transcribe(
|
|
bytes,
|
|
context: _lastAssistantContent.isNotEmpty ? _lastAssistantContent : null,
|
|
);
|
|
|
|
if (!state.voiceModeActive) return;
|
|
|
|
if (transcript.isEmpty) {
|
|
_emptyTranscriptCount++;
|
|
if (_emptyTranscriptCount >= 3) {
|
|
_onError?.call('No speech detected — tap the mic to try again');
|
|
exitVoiceMode();
|
|
return;
|
|
}
|
|
await _startListening();
|
|
return;
|
|
}
|
|
_emptyTranscriptCount = 0;
|
|
|
|
// Reset TTS state for this new turn
|
|
_sentenceBuffer = '';
|
|
_lastSeenLength = 0;
|
|
_streamComplete = false;
|
|
|
|
if (_enableTts) {
|
|
state = state.copyWith(mode: VoiceMode.playing);
|
|
}
|
|
|
|
await _onTranscript?.call(transcript);
|
|
|
|
// If TTS is not enabled, loop immediately
|
|
if (!_enableTts && state.voiceModeActive) {
|
|
await _startListening();
|
|
}
|
|
} catch (e) {
|
|
_onError?.call('Voice error: transcription failed');
|
|
exitVoiceMode();
|
|
}
|
|
}
|
|
|
|
// ── Internal TTS ────────────────────────────────────────────────────────────
|
|
|
|
void _dispatchSentences({required bool flush}) {
|
|
final result = extractSentences(_sentenceBuffer);
|
|
_sentenceBuffer = flush ? '' : result.remainder;
|
|
|
|
for (final sentence in result.sentences) {
|
|
_enqueueSentence(sentence);
|
|
}
|
|
if (flush && result.remainder.trim().length >= 3) {
|
|
_enqueueSentence(result.remainder.trim());
|
|
}
|
|
}
|
|
|
|
void _enqueueSentence(String sentence) {
|
|
final stripped = stripMarkdownForTts(sentence);
|
|
if (stripped.length < 3) return;
|
|
_synthesiseSentence(stripped);
|
|
}
|
|
|
|
Future<void> _synthesiseSentence(String text) async {
|
|
try {
|
|
final wavBytes =
|
|
await ref.read(voiceRepositoryProvider).synthesise(text);
|
|
if (!state.voiceModeActive) return;
|
|
_ttsQueue.add(wavBytes);
|
|
if (!_ttsPlaying) _drainTtsQueue();
|
|
} catch (_) {
|
|
// Skip failed sentence — TTS errors are non-fatal
|
|
}
|
|
}
|
|
|
|
Future<void> _drainTtsQueue() async {
|
|
if (_ttsPlaying) return;
|
|
_ttsPlaying = true;
|
|
final dir = _tempDir ?? await getTemporaryDirectory();
|
|
|
|
try {
|
|
while (_ttsQueue.isNotEmpty && state.voiceModeActive) {
|
|
final wavBytes = _ttsQueue.removeFirst();
|
|
final path = '${dir.path}/tts_${_ttsCounter++}.wav';
|
|
final file = File(path);
|
|
await file.writeAsBytes(wavBytes);
|
|
|
|
try {
|
|
await _player!.setFilePath(path);
|
|
await _player!.play();
|
|
await _player!.processingStateStream.firstWhere(
|
|
(s) =>
|
|
s == ProcessingState.completed || s == ProcessingState.idle,
|
|
);
|
|
} finally {
|
|
await file.delete().catchError((_) => file);
|
|
}
|
|
}
|
|
} finally {
|
|
_ttsPlaying = false;
|
|
}
|
|
|
|
_checkRestartListening();
|
|
}
|
|
|
|
void _checkRestartListening() {
|
|
if (_streamComplete &&
|
|
_ttsQueue.isEmpty &&
|
|
!_ttsPlaying &&
|
|
state.voiceModeActive) {
|
|
_streamComplete = false;
|
|
_lastSeenLength = 0;
|
|
_sentenceBuffer = '';
|
|
_startListening();
|
|
}
|
|
}
|
|
}
|