feat: listen mode + volume knob in chat; briefing discuss auto-send; fix LLM proactive note search
- ChatView: listen mode toggle (auto-reads new responses via TTS), volume popup with range slider persisted per-device in localStorage via GainNode - useVoiceAudio: shared module-level _volume ref with localStorage persistence, GainNode for volume control, exported setVoiceVolume() - tts.py: pre-warm all Kokoro voices at pipeline load to eliminate HuggingFace HEAD requests at synthesis time (reduces TTS latency) - BriefingView: discuss article button now auto-sends instead of just filling input; prompt capped to 15 sentences; send() accepts optional overrideText - llm.py: instruct LLM not to proactively search notes or comment on note absence Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -1,12 +1,10 @@
|
||||
import { ref, readonly } from 'vue'
|
||||
|
||||
/**
|
||||
* Audio playback composable wrapping the Web Audio API.
|
||||
*
|
||||
* Usage:
|
||||
* const { playing, isSupported, play, stop } = useVoiceAudio()
|
||||
* await play(wavBlob)
|
||||
*/
|
||||
const VOLUME_KEY = 'fa_voice_volume'
|
||||
|
||||
// Shared volume across all instances — persisted to localStorage per device
|
||||
const _volume = ref<number>(parseFloat(localStorage.getItem(VOLUME_KEY) ?? '1.0'))
|
||||
|
||||
export function useVoiceAudio() {
|
||||
const playing = ref(false)
|
||||
const isSupported = typeof AudioContext !== 'undefined' || typeof (window as unknown as Record<string, unknown>).webkitAudioContext !== 'undefined'
|
||||
@@ -32,7 +30,12 @@ export function useVoiceAudio() {
|
||||
|
||||
const source = ctx.createBufferSource()
|
||||
source.buffer = audioBuffer
|
||||
source.connect(ctx.destination)
|
||||
|
||||
const gain = ctx.createGain()
|
||||
gain.gain.value = _volume.value
|
||||
source.connect(gain)
|
||||
gain.connect(ctx.destination)
|
||||
|
||||
currentSource = source
|
||||
playing.value = true
|
||||
|
||||
@@ -53,8 +56,14 @@ export function useVoiceAudio() {
|
||||
|
||||
return {
|
||||
playing: readonly(playing),
|
||||
volume: _volume,
|
||||
isSupported,
|
||||
play,
|
||||
stop,
|
||||
}
|
||||
}
|
||||
|
||||
export function setVoiceVolume(v: number) {
|
||||
_volume.value = Math.max(0, Math.min(1, v))
|
||||
localStorage.setItem(VOLUME_KEY, String(_volume.value))
|
||||
}
|
||||
|
||||
@@ -160,8 +160,8 @@ watch(() => chatStore.streaming, async (streaming) => {
|
||||
const input = ref('')
|
||||
const sending = ref(false)
|
||||
|
||||
async function send() {
|
||||
const text = input.value.trim()
|
||||
async function send(overrideText?: string) {
|
||||
const text = (overrideText ?? input.value).trim()
|
||||
if (!text || !todayConvId.value || chatStore.streaming || sending.value) return
|
||||
if (chatStore.currentConversation?.id !== todayConvId.value) {
|
||||
await chatStore.fetchConversation(todayConvId.value)
|
||||
@@ -175,13 +175,14 @@ async function send() {
|
||||
}
|
||||
}
|
||||
|
||||
function discussArticle(item: NewsItem) {
|
||||
async function discussArticle(item: NewsItem) {
|
||||
const snippet = item.snippet ? `\n\n"${item.snippet.slice(0, 400)}${item.snippet.length > 400 ? '…' : ''}"` : ''
|
||||
input.value = `Can you summarize and discuss this article for me?\n\n**${item.title}**${snippet}\n\nSource: ${item.source}`
|
||||
// Scroll the chat panel into view on narrow layouts
|
||||
nextTick(() => {
|
||||
const text = `Please give me a brief summary and key takeaways for this article (15 sentences or fewer):\n\n**${item.title}**${snippet}\n\nSource: ${item.source}`
|
||||
// Scroll the chat panel into view on narrow layouts, then send
|
||||
await nextTick(() => {
|
||||
document.querySelector('.briefing-chat')?.scrollIntoView({ behavior: 'smooth', block: 'nearest' })
|
||||
})
|
||||
await send(text)
|
||||
}
|
||||
|
||||
function onKeydown(e: KeyboardEvent) {
|
||||
@@ -490,7 +491,7 @@ onMounted(async () => {
|
||||
</button>
|
||||
<button
|
||||
class="btn-send"
|
||||
@click="send"
|
||||
@click="send()"
|
||||
:disabled="!input.trim() || chatStore.streaming || sending"
|
||||
aria-label="Send"
|
||||
>↑</button>
|
||||
|
||||
@@ -3,9 +3,10 @@ import { onMounted, onUnmounted, ref, computed, watch, nextTick } from "vue";
|
||||
import { useRoute, useRouter } from "vue-router";
|
||||
import { useChatStore } from "@/stores/chat";
|
||||
import { useSettingsStore } from "@/stores/settings";
|
||||
import { apiGet, transcribeAudio } from "@/api/client";
|
||||
import { apiGet, transcribeAudio, synthesiseSpeech } from "@/api/client";
|
||||
import { renderMarkdown } from "@/utils/markdown";
|
||||
import { useVoiceRecorder } from "@/composables/useVoiceRecorder";
|
||||
import { useVoiceAudio, setVoiceVolume } from "@/composables/useVoiceAudio";
|
||||
import ChatMessage from "@/components/ChatMessage.vue";
|
||||
import ToolCallCard from "@/components/ToolCallCard.vue";
|
||||
import ToolConfirmCard from "@/components/ToolConfirmCard.vue";
|
||||
@@ -26,10 +27,46 @@ const sidebarOpen = ref(false);
|
||||
// Voice PTT — use store so status is globally reactive
|
||||
const transcribingVoice = ref(false);
|
||||
const recorder = useVoiceRecorder();
|
||||
const audio = useVoiceAudio();
|
||||
const voiceEnabled = computed(() => settingsStore.voiceSttReady);
|
||||
const voiceTtsEnabled = computed(() => settingsStore.voiceTtsReady);
|
||||
const listenMode = ref(false);
|
||||
const synthesising = ref(false);
|
||||
const showVolumeSlider = ref(false);
|
||||
|
||||
async function speakLastAssistantMessage() {
|
||||
const messages = store.currentConversation?.messages ?? [];
|
||||
const last = [...messages].reverse().find((m) => m.role === "assistant");
|
||||
if (!last?.content) return;
|
||||
const plain = last.content
|
||||
.replace(/```[\s\S]*?```/g, "")
|
||||
.replace(/`[^`]+`/g, (m: string) => m.slice(1, -1))
|
||||
.replace(/#{1,6}\s+/g, "")
|
||||
.replace(/\*\*([^*]+)\*\*/g, "$1")
|
||||
.replace(/\*([^*]+)\*/g, "$1")
|
||||
.replace(/\[([^\]]+)\]\([^)]+\)/g, "$1")
|
||||
.replace(/^\s*[-*+]\s+/gm, "")
|
||||
.replace(/\n{2,}/g, " ")
|
||||
.trim();
|
||||
if (!plain) return;
|
||||
synthesising.value = true;
|
||||
try {
|
||||
const blob = await synthesiseSpeech(plain);
|
||||
await audio.play(blob);
|
||||
} catch { /* TTS failure non-critical */ }
|
||||
finally { synthesising.value = false; }
|
||||
}
|
||||
|
||||
watch(() => store.streaming, async (streaming) => {
|
||||
if (!streaming && listenMode.value && voiceTtsEnabled.value) {
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
await speakLastAssistantMessage();
|
||||
}
|
||||
});
|
||||
|
||||
async function startPtt() {
|
||||
if (!voiceEnabled.value || recorder.recording.value) return;
|
||||
audio.stop();
|
||||
await recorder.startRecording();
|
||||
}
|
||||
|
||||
@@ -873,6 +910,55 @@ onUnmounted(() => {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Listen mode + volume (TTS) -->
|
||||
<template v-if="voiceTtsEnabled">
|
||||
<div class="volume-wrapper">
|
||||
<button
|
||||
class="btn-attach btn-listen"
|
||||
:class="{ 'btn-listen--active': listenMode, 'btn-listen--busy': synthesising || audio.playing.value }"
|
||||
@click="listenMode = !listenMode; if (listenMode) speakLastAssistantMessage()"
|
||||
:title="listenMode ? 'Stop auto-read' : 'Read responses aloud'"
|
||||
aria-label="Toggle listen mode"
|
||||
>
|
||||
<svg v-if="!synthesising && !audio.playing.value" width="17" height="17" viewBox="0 0 24 24" fill="currentColor">
|
||||
<path d="M3 9v6h4l5 5V4L7 9H3zm13.5 3c0-1.77-1.02-3.29-2.5-4.03v8.05c1.48-.73 2.5-2.25 2.5-4.02zM14 3.23v2.06c2.89.86 5 3.54 5 6.71s-2.11 5.85-5 6.71v2.06c4.01-.91 7-4.49 7-8.77s-2.99-7.86-7-8.77z"/>
|
||||
</svg>
|
||||
<svg v-else width="17" height="17" viewBox="0 0 24 24" fill="currentColor">
|
||||
<path d="M18 12c0-1.77-1.02-3.29-2.5-4.03v2.21l2.45 2.45c.03-.2.05-.41.05-.63zm2.5 0c0 .94-.2 1.82-.54 2.64l1.51 1.51C21.8 14.82 22 13.43 22 12c0-4.28-2.99-7.86-7-8.77v2.06c2.89.86 5 3.54 5 6.71zM4.27 3L3 4.27 7.73 9H3v6h4l5 5v-6.73l4.25 4.25c-.67.52-1.42.93-2.25 1.18v2.06c1.38-.31 2.63-.95 3.69-1.81L19.73 21 21 19.73l-9-9L4.27 3zM12 4L9.91 6.09 12 8.18V4z"/>
|
||||
</svg>
|
||||
</button>
|
||||
<button
|
||||
class="btn-attach btn-volume"
|
||||
@click="showVolumeSlider = !showVolumeSlider"
|
||||
:title="`Volume: ${Math.round(audio.volume.value * 100)}%`"
|
||||
aria-label="Volume"
|
||||
>
|
||||
<svg width="15" height="15" viewBox="0 0 24 24" fill="currentColor">
|
||||
<path v-if="audio.volume.value === 0" d="M16.5 12c0-1.77-1.02-3.29-2.5-4.03v2.21l2.45 2.45c.03-.2.05-.41.05-.63zm2.5 0c0 .94-.2 1.82-.54 2.64l1.51 1.51C21.8 14.82 22 13.43 22 12c0-4.28-2.99-7.86-7-8.77v2.06c2.89.86 5 3.54 5 6.71zM4.27 3L3 4.27 7.73 9H3v6h4l5 5v-6.73l4.25 4.25c-.67.52-1.42.93-2.25 1.18v2.06c1.38-.31 2.63-.95 3.69-1.81L19.73 21 21 19.73l-9-9L4.27 3z"/>
|
||||
<path v-else d="M3 9v6h4l5 5V4L7 9H3zm13.5 3c0-1.77-1.02-3.29-2.5-4.03v8.05c1.48-.73 2.5-2.25 2.5-4.02z"/>
|
||||
</svg>
|
||||
</button>
|
||||
<div v-if="showVolumeSlider" class="volume-popup">
|
||||
<input
|
||||
type="range" min="0" max="1" step="0.05"
|
||||
:value="audio.volume.value"
|
||||
@input="setVoiceVolume(+($event.target as HTMLInputElement).value)"
|
||||
class="volume-range"
|
||||
aria-label="Volume"
|
||||
/>
|
||||
<span class="volume-pct">{{ Math.round(audio.volume.value * 100) }}%</span>
|
||||
</div>
|
||||
</div>
|
||||
<button
|
||||
v-if="synthesising || audio.playing.value"
|
||||
class="btn-attach"
|
||||
@click="audio.stop(); synthesising = false"
|
||||
title="Stop playback"
|
||||
>
|
||||
<svg width="14" height="14" viewBox="0 0 14 14" fill="currentColor"><rect width="14" height="14" rx="2"/></svg>
|
||||
</button>
|
||||
</template>
|
||||
|
||||
<!-- Mic PTT button -->
|
||||
<button
|
||||
v-if="voiceEnabled && recorder.isSupported"
|
||||
@@ -1692,6 +1778,49 @@ details[open] .thinking-summary::before {
|
||||
50% { opacity: 1; }
|
||||
}
|
||||
|
||||
/* Listen mode + volume controls */
|
||||
.volume-wrapper {
|
||||
position: relative;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 2px;
|
||||
}
|
||||
.btn-listen--active {
|
||||
color: var(--color-primary) !important;
|
||||
opacity: 1 !important;
|
||||
}
|
||||
.btn-listen--busy {
|
||||
color: #a78bfa !important;
|
||||
opacity: 1 !important;
|
||||
}
|
||||
.volume-popup {
|
||||
position: absolute;
|
||||
bottom: calc(100% + 8px);
|
||||
left: 50%;
|
||||
transform: translateX(-50%);
|
||||
background: var(--color-surface, #1e1e2e);
|
||||
border: 1px solid var(--color-border, rgba(255,255,255,0.08));
|
||||
border-radius: 10px;
|
||||
padding: 8px 12px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
box-shadow: 0 4px 16px rgba(0,0,0,0.4);
|
||||
white-space: nowrap;
|
||||
z-index: 50;
|
||||
}
|
||||
.volume-range {
|
||||
width: 96px;
|
||||
accent-color: var(--color-primary, #6366f1);
|
||||
cursor: pointer;
|
||||
}
|
||||
.volume-pct {
|
||||
font-size: 0.72rem;
|
||||
color: var(--color-muted, rgba(255,255,255,0.4));
|
||||
min-width: 28px;
|
||||
text-align: right;
|
||||
}
|
||||
|
||||
.btn-send {
|
||||
width: 34px;
|
||||
min-width: 34px;
|
||||
|
||||
@@ -507,7 +507,8 @@ async def build_context(
|
||||
"Use search_notes for conceptual/semantic queries — e.g. 'what notes do I have about X' or "
|
||||
"'find notes related to Y' — it uses semantic understanding to find thematically related content "
|
||||
"even when exact words don't match. Pass project= to scope the search to a specific project. "
|
||||
"Use delete_note / delete_task only when explicitly asked to delete — these require confirmation."
|
||||
"Use delete_note / delete_task only when explicitly asked to delete — these require confirmation. "
|
||||
"Never proactively search notes or comment on the absence of notes/context unless the user explicitly asks about their notes."
|
||||
)
|
||||
tool_guidance = "\n".join(tool_lines)
|
||||
|
||||
|
||||
@@ -46,11 +46,18 @@ async def load_tts_model() -> None:
|
||||
|
||||
logger.info("Loading Kokoro TTS pipeline...")
|
||||
loop = asyncio.get_running_loop()
|
||||
_pipeline = await loop.run_in_executor(
|
||||
None,
|
||||
lambda: KPipeline(lang_code="a"), # "a" = American English
|
||||
)
|
||||
logger.info("Kokoro TTS pipeline loaded")
|
||||
def _load_and_warm():
|
||||
p = KPipeline(lang_code="a") # "a" = American English
|
||||
# Pre-load all voice tensors so synthesis never hits HuggingFace at request time
|
||||
for v in _VOICES:
|
||||
try:
|
||||
p.load_voice(v["id"])
|
||||
except Exception:
|
||||
pass
|
||||
return p
|
||||
|
||||
_pipeline = await loop.run_in_executor(None, _load_and_warm)
|
||||
logger.info("Kokoro TTS pipeline loaded and voices pre-warmed")
|
||||
except Exception:
|
||||
_load_error = "Failed to load Kokoro TTS pipeline"
|
||||
logger.exception(_load_error)
|
||||
|
||||
Reference in New Issue
Block a user