From d290bebad27c24ae46ae7ed0f121b446039d9b03 Mon Sep 17 00:00:00 2001
From: Bryan Van Deusen <bryan.vandeusen@fabledsword.com>
Date: Tue, 7 Apr 2026 08:48:27 -0400
Subject: [PATCH] feat(stt): pass conversation context as Whisper
 initial_prompt to reduce mishearings

---
 frontend/src/api/client.ts               |  3 ++-
 frontend/src/components/VoiceOverlay.vue |  3 ++-
 src/fabledassistant/routes/voice.py      |  4 +++-
 src/fabledassistant/services/stt.py      | 14 +++++++++++---
 4 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts
index c97b290..c0585b2 100644
--- a/frontend/src/api/client.ts
+++ b/frontend/src/api/client.ts
@@ -649,9 +649,10 @@ export const getVoiceStatus = () => apiGet<VoiceStatusResult>('/api/voice/status
 export const getVoiceList = () =>
   apiGet<{ voices: VoiceEntry[] }>('/api/voice/voices').then(r => r.voices)
 
-export async function transcribeAudio(blob: Blob): Promise<{ transcript: string; duration_ms: number }> {
+export async function transcribeAudio(blob: Blob, context?: string): Promise<{ transcript: string; duration_ms: number }> {
   const form = new FormData()
   form.append('audio', blob, 'audio.webm')
+  if (context) form.append('context', context)
   const res = await fetch('/api/voice/transcribe', { method: 'POST', body: form })
   if (!res.ok) {
     const err = await res.json().catch(() => ({ error: `HTTP ${res.status}` }))
diff --git a/frontend/src/components/VoiceOverlay.vue b/frontend/src/components/VoiceOverlay.vue
index 447aebb..63e0ef6 100644
--- a/frontend/src/components/VoiceOverlay.vue
+++ b/frontend/src/components/VoiceOverlay.vue
@@ -102,7 +102,8 @@ async function stopPtt() {
 
   let transcript: string
   try {
-    const result = await transcribeAudio(blob)
+    const lastAssistant = [...messages.value].reverse().find(m => m.role === 'assistant')?.content
+    const result = await transcribeAudio(blob, lastAssistant)
     transcript = result.transcript.trim()
   } catch {
     phase.value = 'error'
diff --git a/src/fabledassistant/routes/voice.py b/src/fabledassistant/routes/voice.py
index b0ed890..8841121 100644
--- a/src/fabledassistant/routes/voice.py
+++ b/src/fabledassistant/routes/voice.py
@@ -80,10 +80,12 @@ async def transcribe_audio():
         return jsonify({"error": "Audio file too large (max 25 MB)"}), 413
 
     mime_type = audio_file.content_type or "audio/webm"
+    form = await request.form
+    context = (form.get("context") or "").strip() or None
 
     t0 = time.monotonic()
     try:
-        transcript = await transcribe(audio_bytes, mime_type)
+        transcript = await transcribe(audio_bytes, mime_type, initial_prompt=context)
     except Exception:
         logger.exception("STT transcription failed")
         return jsonify({"error": "Transcription failed"}), 500
diff --git a/src/fabledassistant/services/stt.py b/src/fabledassistant/services/stt.py
index 2de2f41..adc8c5a 100644
--- a/src/fabledassistant/services/stt.py
+++ b/src/fabledassistant/services/stt.py
@@ -55,8 +55,12 @@ def stt_available() -> bool:
     return _model is not None
 
 
-async def transcribe(audio_bytes: bytes, mime_type: str = "audio/webm") -> str:
-    """Transcribe audio bytes to text. Runs the model in a thread executor."""
+async def transcribe(audio_bytes: bytes, mime_type: str = "audio/webm", initial_prompt: str | None = None) -> str:
+    """Transcribe audio bytes to text. Runs the model in a thread executor.
+
+    initial_prompt: optional text to bias the model toward domain-specific vocabulary
+    (e.g. recent conversation context). Reduces mishearings like "gold" for "cold".
+    """
     if _model is None:
         raise RuntimeError("STT model not loaded")
 
@@ -73,7 +77,11 @@ async def transcribe(audio_bytes: bytes, mime_type: str = "audio/webm") -> str:
             f.write(audio_bytes)
             f.flush()
             t0 = time.monotonic()
-            segments, _ = _model.transcribe(f.name, beam_size=5)  # type: ignore[union-attr]
+            segments, _ = _model.transcribe(  # type: ignore[union-attr]
+                f.name,
+                beam_size=5,
+                initial_prompt=initial_prompt or None,
+            )
             text = " ".join(seg.text.strip() for seg in segments).strip()
             logger.debug("STT transcription took %.2fs", time.monotonic() - t0)
             return text