From baeb0b14e5c650be2e80441b9f47a0b16cc6d24d Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 31 Mar 2026 16:52:11 -0400 Subject: [PATCH] feat: listen mode + volume knob in chat; briefing discuss auto-send; fix LLM proactive note search - ChatView: listen mode toggle (auto-reads new responses via TTS), volume popup with range slider persisted per-device in localStorage via GainNode - useVoiceAudio: shared module-level _volume ref with localStorage persistence, GainNode for volume control, exported setVoiceVolume() - tts.py: pre-warm all Kokoro voices at pipeline load to eliminate HuggingFace HEAD requests at synthesis time (reduces TTS latency) - BriefingView: discuss article button now auto-sends instead of just filling input; prompt capped to 15 sentences; send() accepts optional overrideText - llm.py: instruct LLM not to proactively search notes or comment on note absence Co-Authored-By: Claude Sonnet 4.6 --- frontend/src/composables/useVoiceAudio.ts | 25 +++-- frontend/src/views/BriefingView.vue | 15 +-- frontend/src/views/ChatView.vue | 131 +++++++++++++++++++++- src/fabledassistant/services/llm.py | 3 +- src/fabledassistant/services/tts.py | 17 ++- 5 files changed, 169 insertions(+), 22 deletions(-) diff --git a/frontend/src/composables/useVoiceAudio.ts b/frontend/src/composables/useVoiceAudio.ts index 6645f65..be490f7 100644 --- a/frontend/src/composables/useVoiceAudio.ts +++ b/frontend/src/composables/useVoiceAudio.ts @@ -1,12 +1,10 @@ import { ref, readonly } from 'vue' -/** - * Audio playback composable wrapping the Web Audio API. - * - * Usage: - * const { playing, isSupported, play, stop } = useVoiceAudio() - * await play(wavBlob) - */ +const VOLUME_KEY = 'fa_voice_volume' + +// Shared volume across all instances — persisted to localStorage per device +const _volume = ref(parseFloat(localStorage.getItem(VOLUME_KEY) ?? '1.0')) + export function useVoiceAudio() { const playing = ref(false) const isSupported = typeof AudioContext !== 'undefined' || typeof (window as unknown as Record).webkitAudioContext !== 'undefined' @@ -32,7 +30,12 @@ export function useVoiceAudio() { const source = ctx.createBufferSource() source.buffer = audioBuffer - source.connect(ctx.destination) + + const gain = ctx.createGain() + gain.gain.value = _volume.value + source.connect(gain) + gain.connect(ctx.destination) + currentSource = source playing.value = true @@ -53,8 +56,14 @@ export function useVoiceAudio() { return { playing: readonly(playing), + volume: _volume, isSupported, play, stop, } } + +export function setVoiceVolume(v: number) { + _volume.value = Math.max(0, Math.min(1, v)) + localStorage.setItem(VOLUME_KEY, String(_volume.value)) +} diff --git a/frontend/src/views/BriefingView.vue b/frontend/src/views/BriefingView.vue index 34dbb35..d842fe9 100644 --- a/frontend/src/views/BriefingView.vue +++ b/frontend/src/views/BriefingView.vue @@ -160,8 +160,8 @@ watch(() => chatStore.streaming, async (streaming) => { const input = ref('') const sending = ref(false) -async function send() { - const text = input.value.trim() +async function send(overrideText?: string) { + const text = (overrideText ?? input.value).trim() if (!text || !todayConvId.value || chatStore.streaming || sending.value) return if (chatStore.currentConversation?.id !== todayConvId.value) { await chatStore.fetchConversation(todayConvId.value) @@ -175,13 +175,14 @@ async function send() { } } -function discussArticle(item: NewsItem) { +async function discussArticle(item: NewsItem) { const snippet = item.snippet ? `\n\n"${item.snippet.slice(0, 400)}${item.snippet.length > 400 ? '…' : ''}"` : '' - input.value = `Can you summarize and discuss this article for me?\n\n**${item.title}**${snippet}\n\nSource: ${item.source}` - // Scroll the chat panel into view on narrow layouts - nextTick(() => { + const text = `Please give me a brief summary and key takeaways for this article (15 sentences or fewer):\n\n**${item.title}**${snippet}\n\nSource: ${item.source}` + // Scroll the chat panel into view on narrow layouts, then send + await nextTick(() => { document.querySelector('.briefing-chat')?.scrollIntoView({ behavior: 'smooth', block: 'nearest' }) }) + await send(text) } function onKeydown(e: KeyboardEvent) { @@ -490,7 +491,7 @@ onMounted(async () => { diff --git a/frontend/src/views/ChatView.vue b/frontend/src/views/ChatView.vue index b8e0ebf..e3c1c08 100644 --- a/frontend/src/views/ChatView.vue +++ b/frontend/src/views/ChatView.vue @@ -3,9 +3,10 @@ import { onMounted, onUnmounted, ref, computed, watch, nextTick } from "vue"; import { useRoute, useRouter } from "vue-router"; import { useChatStore } from "@/stores/chat"; import { useSettingsStore } from "@/stores/settings"; -import { apiGet, transcribeAudio } from "@/api/client"; +import { apiGet, transcribeAudio, synthesiseSpeech } from "@/api/client"; import { renderMarkdown } from "@/utils/markdown"; import { useVoiceRecorder } from "@/composables/useVoiceRecorder"; +import { useVoiceAudio, setVoiceVolume } from "@/composables/useVoiceAudio"; import ChatMessage from "@/components/ChatMessage.vue"; import ToolCallCard from "@/components/ToolCallCard.vue"; import ToolConfirmCard from "@/components/ToolConfirmCard.vue"; @@ -26,10 +27,46 @@ const sidebarOpen = ref(false); // Voice PTT — use store so status is globally reactive const transcribingVoice = ref(false); const recorder = useVoiceRecorder(); +const audio = useVoiceAudio(); const voiceEnabled = computed(() => settingsStore.voiceSttReady); +const voiceTtsEnabled = computed(() => settingsStore.voiceTtsReady); +const listenMode = ref(false); +const synthesising = ref(false); +const showVolumeSlider = ref(false); + +async function speakLastAssistantMessage() { + const messages = store.currentConversation?.messages ?? []; + const last = [...messages].reverse().find((m) => m.role === "assistant"); + if (!last?.content) return; + const plain = last.content + .replace(/```[\s\S]*?```/g, "") + .replace(/`[^`]+`/g, (m: string) => m.slice(1, -1)) + .replace(/#{1,6}\s+/g, "") + .replace(/\*\*([^*]+)\*\*/g, "$1") + .replace(/\*([^*]+)\*/g, "$1") + .replace(/\[([^\]]+)\]\([^)]+\)/g, "$1") + .replace(/^\s*[-*+]\s+/gm, "") + .replace(/\n{2,}/g, " ") + .trim(); + if (!plain) return; + synthesising.value = true; + try { + const blob = await synthesiseSpeech(plain); + await audio.play(blob); + } catch { /* TTS failure non-critical */ } + finally { synthesising.value = false; } +} + +watch(() => store.streaming, async (streaming) => { + if (!streaming && listenMode.value && voiceTtsEnabled.value) { + await new Promise((r) => setTimeout(r, 200)); + await speakLastAssistantMessage(); + } +}); async function startPtt() { if (!voiceEnabled.value || recorder.recording.value) return; + audio.stop(); await recorder.startRecording(); } @@ -873,6 +910,55 @@ onUnmounted(() => { + + +