From 3bd0dc6879deda47e82f4804311a11ec8d29f62d Mon Sep 17 00:00:00 2001
From: Bryan Van Deusen <bryan.vandeusen@fabledsword.com>
Date: Fri, 3 Apr 2026 10:44:08 -0400
Subject: [PATCH] feat(settings): add background model picker with KV cache
 performance warning

Exposes OLLAMA_BACKGROUND_MODEL as a per-user setting in General settings,
alongside the Chat Model selector. Includes an inline warning when the same
model is selected for both, explaining the KV cache performance impact.

All background task callers (title generation, tag suggestions, project
summaries, RSS classification) now read background_model from user settings,
falling back to OLLAMA_BACKGROUND_MODEL env var.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 frontend/src/views/SettingsView.vue           | 22 +++++++++++++++++++
 src/fabledassistant/services/chat.py          |  4 +++-
 .../services/generation_task.py               |  8 ++++---
 src/fabledassistant/services/projects.py      |  4 +++-
 .../services/rss_classifier.py                |  2 +-
 .../services/tag_suggestions.py               |  3 ++-
 6 files changed, 36 insertions(+), 7 deletions(-)

diff --git a/frontend/src/views/SettingsView.vue b/frontend/src/views/SettingsView.vue
index dd1b408..32a2722 100644
--- a/frontend/src/views/SettingsView.vue
+++ b/frontend/src/views/SettingsView.vue
@@ -15,6 +15,7 @@ const toastStore = useToastStore();
 const pushStore = usePushStore();
 const assistantName = ref("");
 const defaultModel = ref("");
+const backgroundModel = ref("");
 const installedModels = ref<string[]>([]);
 const defaultChatModel = ref("");
 
@@ -708,6 +709,7 @@ onMounted(async () => {
   // Load notification preferences from user settings
   const allSettings = await apiGet<Record<string, string>>("/api/settings");
   defaultModel.value = allSettings.default_model ?? "";
+  backgroundModel.value = allSettings.background_model ?? "";
   chatRetentionDays.value = allSettings.chat_retention_days !== undefined
     ? Number(allSettings.chat_retention_days)
     : 90;
@@ -929,6 +931,7 @@ async function saveAssistant() {
     await store.updateSettings({
       assistant_name: assistantName.value.trim() || "Fable",
       default_model: defaultModel.value,
+      background_model: backgroundModel.value,
     });
     saved.value = true;
     setTimeout(() => (saved.value = false), 2000);
@@ -1437,6 +1440,20 @@ function formatUserDate(iso: string): string {
             </select>
             <p class="field-hint">Model used for new conversations.</p>
           </div>
+          <div class="field">
+            <label for="background-model">Background Model</label>
+            <select id="background-model" v-model="backgroundModel" class="input">
+              <option value="">Default (qwen2.5:0.5b)</option>
+              <option v-for="m in installedModels" :key="m" :value="m">{{ m }}</option>
+            </select>
+            <p class="field-hint">
+              Model used for background tasks: title generation, tag suggestions, project summaries, and RSS classification.
+              Using a small dedicated model (e.g. qwen2.5:0.5b) keeps the chat model's KV cache warm between messages, significantly reducing response time.
+              <span v-if="backgroundModel && backgroundModel === (defaultModel || defaultChatModel)" class="field-hint-warn">
+                ⚠ Setting this to the same model as Chat Model will wipe the KV cache after every background task, increasing response latency.
+              </span>
+            </p>
+          </div>
         </div>
         <div class="actions">
           <button class="btn-save" @click="saveAssistant" :disabled="saving">
@@ -3078,6 +3095,11 @@ FABLE_API_KEY=&lt;your-api-key&gt;</pre>
   font-size: 0.78rem;
   color: var(--color-text-muted);
 }
+.field-hint-warn {
+  display: block;
+  margin-top: 0.3rem;
+  color: #f59e0b;
+}
 .actions {
   display: flex;
   align-items: center;
diff --git a/src/fabledassistant/services/chat.py b/src/fabledassistant/services/chat.py
index 698bc0e..b8175bc 100644
--- a/src/fabledassistant/services/chat.py
+++ b/src/fabledassistant/services/chat.py
@@ -9,6 +9,7 @@ from fabledassistant.models.conversation import Conversation, Message
 from fabledassistant.config import Config
 from fabledassistant.services.llm import generate_completion
 from fabledassistant.services.notes import create_note
+from fabledassistant.services.settings import get_setting
 
 logger = logging.getLogger(__name__)
 
@@ -234,7 +235,8 @@ async def save_response_as_note(user_id: int, message_id: int) -> dict:
                 },
                 {"role": "user", "content": msg.content[:2000]},
             ]
-            title = await generate_completion(prompt_messages, Config.OLLAMA_BACKGROUND_MODEL)
+            bg_model = await get_setting(user_id, "background_model", Config.OLLAMA_BACKGROUND_MODEL)
+            title = await generate_completion(prompt_messages, bg_model)
             title = title.strip().strip('"\'').strip()[:100]
         except Exception:
             logger.warning("Failed to generate note title, using fallback", exc_info=True)
diff --git a/src/fabledassistant/services/generation_task.py b/src/fabledassistant/services/generation_task.py
index fbb19e9..f757b97 100644
--- a/src/fabledassistant/services/generation_task.py
+++ b/src/fabledassistant/services/generation_task.py
@@ -24,6 +24,7 @@ from fabledassistant.services.generation_buffer import (
 )
 from fabledassistant.services.llm import ChatChunk, build_context, generate_completion, stream_chat, stream_chat_with_tools, summarize_history_for_context
 from fabledassistant.services.chat import update_conversation_title
+from fabledassistant.services.settings import get_setting
 from fabledassistant.services.logging import log_generation
 from fabledassistant.services.tools import get_tools_for_user, execute_tool
 from fabledassistant.services.research import run_research_pipeline
@@ -116,7 +117,7 @@ _TOOL_LABELS: dict[str, str] = {
 }
 
 
-async def _generate_title(messages: list[dict], model: str) -> str:
+async def _generate_title(messages: list[dict], user_id: int) -> str:
     """Ask the LLM for a concise conversation title."""
     # Build conversation text like summarize_conversation_as_note
     conv_lines = []
@@ -138,7 +139,8 @@ async def _generate_title(messages: list[dict], model: str) -> str:
         },
         {"role": "user", "content": "\n\n".join(conv_lines)},
     ]
-    title = await generate_completion(prompt_messages, Config.OLLAMA_BACKGROUND_MODEL, max_tokens=30)
+    bg_model = await get_setting(user_id, "background_model", Config.OLLAMA_BACKGROUND_MODEL)
+    title = await generate_completion(prompt_messages, bg_model, max_tokens=30)
     title = title.strip().strip('"\'').strip()
     return title[:100] if title else ""
 
@@ -483,7 +485,7 @@ async def run_generation(
 
             async def _bg_title() -> None:
                 try:
-                    title = await _generate_title(title_messages, model)
+                    title = await _generate_title(title_messages, user_id)
                     if title:
                         await update_conversation_title(user_id, conv_id, title)
                 except Exception:
diff --git a/src/fabledassistant/services/projects.py b/src/fabledassistant/services/projects.py
index 799ad50..a2ce5af 100644
--- a/src/fabledassistant/services/projects.py
+++ b/src/fabledassistant/services/projects.py
@@ -121,8 +121,10 @@ async def generate_project_summary(user_id: int, project_id: int) -> None:
 
         from fabledassistant.services.llm import generate_completion
         from fabledassistant.config import Config
+        from fabledassistant.services.settings import get_setting
         messages = [{"role": "user", "content": prompt}]
-        summary = await generate_completion(messages, model=Config.OLLAMA_BACKGROUND_MODEL, max_tokens=400)
+        bg_model = await get_setting(user_id, "background_model", Config.OLLAMA_BACKGROUND_MODEL)
+        summary = await generate_completion(messages, model=bg_model, max_tokens=400)
         if not summary:
             return
 
diff --git a/src/fabledassistant/services/rss_classifier.py b/src/fabledassistant/services/rss_classifier.py
index a9402ca..00934cc 100644
--- a/src/fabledassistant/services/rss_classifier.py
+++ b/src/fabledassistant/services/rss_classifier.py
@@ -122,7 +122,7 @@ async def classify_and_store(
     except Exception:
         include_topics = []
 
-    model = await get_setting(user_id, "default_model", Config.OLLAMA_MODEL)
+    model = await get_setting(user_id, "background_model", Config.OLLAMA_BACKGROUND_MODEL)
 
     # Classify in batches of 10
     batch_size = 10
diff --git a/src/fabledassistant/services/tag_suggestions.py b/src/fabledassistant/services/tag_suggestions.py
index 6ae0e5e..b39a622 100644
--- a/src/fabledassistant/services/tag_suggestions.py
+++ b/src/fabledassistant/services/tag_suggestions.py
@@ -7,6 +7,7 @@ import re
 from fabledassistant.config import Config
 from fabledassistant.services.llm import generate_completion
 from fabledassistant.services.notes import get_all_tags
+from fabledassistant.services.settings import get_setting
 
 logger = logging.getLogger(__name__)
 
@@ -21,7 +22,7 @@ async def suggest_tags(user_id: int, title: str, body: str, current_tags: list[s
         return []
 
     existing_tags = await get_all_tags(user_id)
-    model = Config.OLLAMA_BACKGROUND_MODEL
+    model = await get_setting(user_id, "background_model", Config.OLLAMA_BACKGROUND_MODEL)
 
     existing_list = ", ".join(f"#{t}" for t in existing_tags) if existing_tags else "(none yet)"