From d290bebad27c24ae46ae7ed0f121b446039d9b03 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 7 Apr 2026 08:48:27 -0400 Subject: [PATCH 1/6] feat(stt): pass conversation context as Whisper initial_prompt to reduce mishearings --- frontend/src/api/client.ts | 3 ++- frontend/src/components/VoiceOverlay.vue | 3 ++- src/fabledassistant/routes/voice.py | 4 +++- src/fabledassistant/services/stt.py | 14 +++++++++++--- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index c97b290..c0585b2 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -649,9 +649,10 @@ export const getVoiceStatus = () => apiGet('/api/voice/status export const getVoiceList = () => apiGet<{ voices: VoiceEntry[] }>('/api/voice/voices').then(r => r.voices) -export async function transcribeAudio(blob: Blob): Promise<{ transcript: string; duration_ms: number }> { +export async function transcribeAudio(blob: Blob, context?: string): Promise<{ transcript: string; duration_ms: number }> { const form = new FormData() form.append('audio', blob, 'audio.webm') + if (context) form.append('context', context) const res = await fetch('/api/voice/transcribe', { method: 'POST', body: form }) if (!res.ok) { const err = await res.json().catch(() => ({ error: `HTTP ${res.status}` })) diff --git a/frontend/src/components/VoiceOverlay.vue b/frontend/src/components/VoiceOverlay.vue index 447aebb..63e0ef6 100644 --- a/frontend/src/components/VoiceOverlay.vue +++ b/frontend/src/components/VoiceOverlay.vue @@ -102,7 +102,8 @@ async function stopPtt() { let transcript: string try { - const result = await transcribeAudio(blob) + const lastAssistant = [...messages.value].reverse().find(m => m.role === 'assistant')?.content + const result = await transcribeAudio(blob, lastAssistant) transcript = result.transcript.trim() } catch { phase.value = 'error' diff --git a/src/fabledassistant/routes/voice.py b/src/fabledassistant/routes/voice.py index b0ed890..8841121 100644 --- a/src/fabledassistant/routes/voice.py +++ b/src/fabledassistant/routes/voice.py @@ -80,10 +80,12 @@ async def transcribe_audio(): return jsonify({"error": "Audio file too large (max 25 MB)"}), 413 mime_type = audio_file.content_type or "audio/webm" + form = await request.form + context = (form.get("context") or "").strip() or None t0 = time.monotonic() try: - transcript = await transcribe(audio_bytes, mime_type) + transcript = await transcribe(audio_bytes, mime_type, initial_prompt=context) except Exception: logger.exception("STT transcription failed") return jsonify({"error": "Transcription failed"}), 500 diff --git a/src/fabledassistant/services/stt.py b/src/fabledassistant/services/stt.py index 2de2f41..adc8c5a 100644 --- a/src/fabledassistant/services/stt.py +++ b/src/fabledassistant/services/stt.py @@ -55,8 +55,12 @@ def stt_available() -> bool: return _model is not None -async def transcribe(audio_bytes: bytes, mime_type: str = "audio/webm") -> str: - """Transcribe audio bytes to text. Runs the model in a thread executor.""" +async def transcribe(audio_bytes: bytes, mime_type: str = "audio/webm", initial_prompt: str | None = None) -> str: + """Transcribe audio bytes to text. Runs the model in a thread executor. + + initial_prompt: optional text to bias the model toward domain-specific vocabulary + (e.g. recent conversation context). Reduces mishearings like "gold" for "cold". + """ if _model is None: raise RuntimeError("STT model not loaded") @@ -73,7 +77,11 @@ async def transcribe(audio_bytes: bytes, mime_type: str = "audio/webm") -> str: f.write(audio_bytes) f.flush() t0 = time.monotonic() - segments, _ = _model.transcribe(f.name, beam_size=5) # type: ignore[union-attr] + segments, _ = _model.transcribe( # type: ignore[union-attr] + f.name, + beam_size=5, + initial_prompt=initial_prompt or None, + ) text = " ".join(seg.text.strip() for seg in segments).strip() logger.debug("STT transcription took %.2fs", time.monotonic() - t0) return text From b3cf42863ab773dd11240a2322bec192d095f771 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 7 Apr 2026 12:45:08 -0400 Subject: [PATCH 2/6] fix: reinforce no-project-inference in system prompt; filter tool messages from title generation --- src/fabledassistant/services/generation_task.py | 7 +++++-- src/fabledassistant/services/llm.py | 5 +++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/fabledassistant/services/generation_task.py b/src/fabledassistant/services/generation_task.py index 83fdad3..5a63b9a 100644 --- a/src/fabledassistant/services/generation_task.py +++ b/src/fabledassistant/services/generation_task.py @@ -122,10 +122,13 @@ async def _generate_title(messages: list[dict], user_id: int) -> str: # Build conversation text like summarize_conversation_as_note conv_lines = [] for m in messages: - if m["role"] == "system": + if m["role"] in ("system", "tool"): continue label = "User" if m["role"] == "user" else "Assistant" - conv_lines.append(f"{label}: {m['content']}") + content = m.get("content", "") + if not content: + continue + conv_lines.append(f"{label}: {content}") # Keep only last 6 pairs worth of text conv_lines = conv_lines[-12:] diff --git a/src/fabledassistant/services/llm.py b/src/fabledassistant/services/llm.py index 178e501..d1872e5 100644 --- a/src/fabledassistant/services/llm.py +++ b/src/fabledassistant/services/llm.py @@ -538,6 +538,11 @@ async def build_context( "Delete tools require an explicit user request. " "Never proactively search notes or comment on absent context." ) + tool_lines.append( + "IMPORTANT: When creating tasks or notes, NEVER infer or guess a project name. " + "Only set the project parameter if the user explicitly names a project. " + "If the user says 'create a task to buy milk', do NOT assign it to a project." + ) tool_guidance = "\n".join(tool_lines) static_block = ( From 39e554d9380cd685004354e913540aecdb1f9c32 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 7 Apr 2026 12:53:33 -0400 Subject: [PATCH 3/6] fix: rewrite title generator to only use user messages; bump background model to qwen2.5:3b --- src/fabledassistant/config.py | 2 +- .../services/generation_task.py | 47 ++++++++++++------- src/fabledassistant/services/projects.py | 2 +- 3 files changed, 31 insertions(+), 20 deletions(-) diff --git a/src/fabledassistant/config.py b/src/fabledassistant/config.py index 2675f3c..9a69db9 100644 --- a/src/fabledassistant/config.py +++ b/src/fabledassistant/config.py @@ -27,7 +27,7 @@ class Config: # Lightweight model for background tasks (title generation, tag suggestions, # project summaries, RSS classification). Using a separate model keeps the # main model's KV cache intact between user messages, enabling prefix cache hits. - OLLAMA_BACKGROUND_MODEL: str = os.environ.get("OLLAMA_BACKGROUND_MODEL", "qwen2.5:0.5b") + OLLAMA_BACKGROUND_MODEL: str = os.environ.get("OLLAMA_BACKGROUND_MODEL", "qwen2.5:3b") # KV cache context window for generation. Keep this as small as practical — # a larger context forces more KV cache into CPU RAM, drastically slowing prefill. # 16384 covers ~30+ message conversations with our system prompt comfortably. diff --git a/src/fabledassistant/services/generation_task.py b/src/fabledassistant/services/generation_task.py index 5a63b9a..329628c 100644 --- a/src/fabledassistant/services/generation_task.py +++ b/src/fabledassistant/services/generation_task.py @@ -118,34 +118,45 @@ _TOOL_LABELS: dict[str, str] = { async def _generate_title(messages: list[dict], user_id: int) -> str: - """Ask the LLM for a concise conversation title.""" - # Build conversation text like summarize_conversation_as_note - conv_lines = [] + """Ask the LLM for a concise conversation title. + + Only uses user messages to avoid feeding tool-call JSON, system prompt + fragments, or other noise into the title generator. Caps input length + to keep the task fast and focused. + """ + user_texts = [] for m in messages: - if m["role"] in ("system", "tool"): - continue - label = "User" if m["role"] == "user" else "Assistant" - content = m.get("content", "") - if not content: - continue - conv_lines.append(f"{label}: {content}") - # Keep only last 6 pairs worth of text - conv_lines = conv_lines[-12:] + if m["role"] == "user": + content = (m.get("content") or "").strip() + if content: + user_texts.append(content[:300]) + if not user_texts: + return "" + # First + last user messages capture intent best + if len(user_texts) > 2: + user_texts = [user_texts[0], user_texts[-1]] prompt_messages = [ { - "role": "system", + "role": "user", "content": ( - "Generate a concise 3-8 word title for this conversation. " - "Reply with ONLY the title, no quotes or punctuation." + "Generate a concise 3-8 word title for a conversation that started with:\n\n" + + "\n\n".join(user_texts) + + "\n\nReply with ONLY the title. No quotes, no punctuation, no explanation." ), }, - {"role": "user", "content": "\n\n".join(conv_lines)}, ] bg_model = await get_setting(user_id, "background_model", Config.OLLAMA_BACKGROUND_MODEL) - title = await generate_completion(prompt_messages, bg_model, max_tokens=30) + title = await generate_completion(prompt_messages, bg_model, max_tokens=30, num_ctx=1024) + # Strip common LLM noise: quotes, thinking tags, role labels title = title.strip().strip('"\'').strip() - return title[:100] if title else "" + for prefix in ("Title:", "title:", "Assistant:", "User:"): + if title.startswith(prefix): + title = title[len(prefix):].strip() + # Drop anything after a newline (model sometimes adds explanation) + if "\n" in title: + title = title.split("\n")[0].strip() + return title[:80] if title else "" async def _update_message( diff --git a/src/fabledassistant/services/projects.py b/src/fabledassistant/services/projects.py index a2ce5af..d3e1e38 100644 --- a/src/fabledassistant/services/projects.py +++ b/src/fabledassistant/services/projects.py @@ -124,7 +124,7 @@ async def generate_project_summary(user_id: int, project_id: int) -> None: from fabledassistant.services.settings import get_setting messages = [{"role": "user", "content": prompt}] bg_model = await get_setting(user_id, "background_model", Config.OLLAMA_BACKGROUND_MODEL) - summary = await generate_completion(messages, model=bg_model, max_tokens=400) + summary = await generate_completion(messages, model=bg_model, max_tokens=400, num_ctx=2048) if not summary: return From 7f37cee49f5d45fbf977a22836247d991421cea5 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 7 Apr 2026 13:17:25 -0400 Subject: [PATCH 4/6] refactor(voice): integrate click-to-toggle mic into ChatInputBar; remove VoiceOverlay --- frontend/src/App.vue | 10 - frontend/src/components/ChatInputBar.vue | 40 +- frontend/src/components/VoiceOverlay.vue | 569 ----------------------- 3 files changed, 28 insertions(+), 591 deletions(-) delete mode 100644 frontend/src/components/VoiceOverlay.vue diff --git a/frontend/src/App.vue b/frontend/src/App.vue index 0db3609..86ff228 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -3,7 +3,6 @@ import { onMounted, onUnmounted, ref, watch } from "vue"; import { useRouter } from "vue-router"; import AppHeader from "@/components/AppHeader.vue"; import ToastNotification from "@/components/ToastNotification.vue"; -import VoiceOverlay from "@/components/VoiceOverlay.vue"; import { useTheme } from "@/composables/useTheme"; import { useShortcuts } from "@/composables/useShortcuts"; import { useAuthStore } from "@/stores/auth"; @@ -121,10 +120,6 @@ function onGlobalKeydown(e: KeyboardEvent) { router.push("/chat"); } break; - case " ": - e.preventDefault(); - document.dispatchEvent(new CustomEvent("voice:ptt-toggle")); - break; } } @@ -274,10 +269,6 @@ onUnmounted(() => { Enter New line -
- Space - Tap to speak (voice, when enabled) -
@@ -287,7 +278,6 @@ onUnmounted(() => { - diff --git a/frontend/src/components/ChatInputBar.vue b/frontend/src/components/ChatInputBar.vue index 218961e..35862ad 100644 --- a/frontend/src/components/ChatInputBar.vue +++ b/frontend/src/components/ChatInputBar.vue @@ -2,6 +2,7 @@ import { ref, computed, nextTick } from 'vue' import { apiGet, transcribeAudio } from '@/api/client' import { useVoiceRecorder } from '@/composables/useVoiceRecorder' +import { useSilenceDetector } from '@/composables/useSilenceDetector' import { useChatStore } from '@/stores/chat' import { useSettingsStore } from '@/stores/settings' import type { Note } from '@/types/note' @@ -107,21 +108,39 @@ function removeAttachedNote() { attachedNote.value = null } -// ── PTT ─────────────────────────────────────────────────────────────────────── +// ── Voice (click-to-toggle + silence detection) ───────────────────────────── const transcribingVoice = ref(false) const recorder = useVoiceRecorder() +const silenceDetector = useSilenceDetector() -async function startPtt() { - if (!voiceEnabled.value || recorder.recording.value) return - await recorder.startRecording() +async function toggleVoice() { + if (transcribingVoice.value) return + if (recorder.recording.value) { + await stopRecording() + } else { + await startRecording() + } } -async function stopPtt() { +async function startRecording() { + if (!voiceEnabled.value || recorder.recording.value) return + await recorder.startRecording() + if (recorder.error.value) return + if (recorder.stream.value) { + silenceDetector.start(recorder.stream.value, () => stopRecording()) + } +} + +async function stopRecording() { + silenceDetector.stop() if (!recorder.recording.value) return transcribingVoice.value = true try { const blob = await recorder.stopRecording() - const { transcript } = await transcribeAudio(blob) + // Pass last assistant message as context to reduce STT mishearings + const msgs = store.currentConversation?.messages ?? [] + const lastAssistant = [...msgs].reverse().find(m => m.role === 'assistant')?.content + const { transcript } = await transcribeAudio(blob, lastAssistant) if (transcript.trim()) { messageInput.value = transcript.trim() await nextTick() @@ -208,13 +227,10 @@ defineExpose({ focus, prefill }) v-if="voiceEnabled && recorder.isSupported" class="btn-icon btn-mic" :class="{ 'mic-recording': recorder.recording.value, 'mic-transcribing': transcribingVoice }" - @mousedown.prevent="startPtt" - @mouseup.prevent="stopPtt" - @touchstart.prevent="startPtt" - @touchend.prevent="stopPtt" + @click.prevent="toggleVoice" :disabled="transcribingVoice || !store.chatReady" - :title="recorder.recording.value ? 'Release to send' : 'Hold to speak'" - aria-label="Push to talk" + :title="recorder.recording.value ? 'Click to stop (or wait for silence)' : 'Click to speak'" + :aria-label="recorder.recording.value ? 'Stop recording' : 'Start recording'" > diff --git a/frontend/src/components/VoiceOverlay.vue b/frontend/src/components/VoiceOverlay.vue deleted file mode 100644 index 63e0ef6..0000000 --- a/frontend/src/components/VoiceOverlay.vue +++ /dev/null @@ -1,569 +0,0 @@ - - -
- - - -
-
- Voice - -
-
-
{{ msg.content }}
- -
- {{ streamContent }}▌ -
-
-
- - - -
- -
- Recording… - Transcribing… - Thinking… - Speaking… - {{ errorMsg || 'Error' }} -
-
- Tap or press Space -
- - - - - - -
- -
- - - - From 6d57840dc7ccd90facd3810e80368bce66c11178 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 7 Apr 2026 20:01:47 -0400 Subject: [PATCH 5/6] fix(voice): re-check voice status on tab visibility change and after saving voice settings --- frontend/src/App.vue | 10 ++++++++++ frontend/src/views/SettingsView.vue | 2 ++ 2 files changed, 12 insertions(+) diff --git a/frontend/src/App.vue b/frontend/src/App.vue index 86ff228..6b4e139 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -25,6 +25,15 @@ function startAppServices() { settingsStore.checkVoiceStatus(); // Sync browser timezone to the server on every login/page load. apiPut("/api/settings", { user_timezone: Intl.DateTimeFormat().resolvedOptions().timeZone }).catch(() => {}); + // Re-check voice status when the tab becomes visible again (model may + // have finished loading while the user was away). + document.addEventListener("visibilitychange", onVisibilityChange); +} + +function onVisibilityChange() { + if (document.visibilityState === "visible" && authStore.isAuthenticated) { + settingsStore.checkVoiceStatus(); + } } function stopAppServices() { @@ -150,6 +159,7 @@ watch( onUnmounted(() => { document.removeEventListener("keydown", onGlobalKeydown); + document.removeEventListener("visibilitychange", onVisibilityChange); stopAppServices(); }); diff --git a/frontend/src/views/SettingsView.vue b/frontend/src/views/SettingsView.vue index 0c50c89..ba1fffc 100644 --- a/frontend/src/views/SettingsView.vue +++ b/frontend/src/views/SettingsView.vue @@ -500,6 +500,8 @@ async function saveAdminVoice() { adminVoiceSaved.value = true; setTimeout(() => { adminVoiceSaved.value = false; }, 2000); voiceTabLoaded.value = false; + // Always update global voice state so mic buttons appear/disappear immediately + await store.checkVoiceStatus(); if (adminVoiceEnabled.value) { await reloadVoiceModels(); } From 56b687c9f4fe56485800e36a13faa056117376bf Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 7 Apr 2026 20:06:57 -0400 Subject: [PATCH 6/6] fix(voice): show mic button on HTTP; graceful error for non-secure contexts --- frontend/src/components/ChatInputBar.vue | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/frontend/src/components/ChatInputBar.vue b/frontend/src/components/ChatInputBar.vue index 35862ad..da56ec4 100644 --- a/frontend/src/components/ChatInputBar.vue +++ b/frontend/src/components/ChatInputBar.vue @@ -5,6 +5,7 @@ import { useVoiceRecorder } from '@/composables/useVoiceRecorder' import { useSilenceDetector } from '@/composables/useSilenceDetector' import { useChatStore } from '@/stores/chat' import { useSettingsStore } from '@/stores/settings' +import { useToastStore } from '@/stores/toast' import type { Note } from '@/types/note' const props = withDefaults(defineProps<{ @@ -124,8 +125,15 @@ async function toggleVoice() { async function startRecording() { if (!voiceEnabled.value || recorder.recording.value) return + if (!recorder.isSupported) { + useToastStore().show('Microphone requires HTTPS or localhost', 'error') + return + } await recorder.startRecording() - if (recorder.error.value) return + if (recorder.error.value) { + useToastStore().show(recorder.error.value, 'error') + return + } if (recorder.stream.value) { silenceDetector.start(recorder.stream.value, () => stopRecording()) } @@ -224,7 +232,7 @@ defineExpose({ focus, prefill })