perf(llm): route background tasks to dedicated model to preserve KV cache

Background tasks (title generation, tag suggestions, project summaries,
RSS classification) were using qwen3:8b and wiping its KV cache after
every response, preventing prefix cache hits on subsequent user messages.

Adds OLLAMA_BACKGROUND_MODEL (default: qwen2.5:0.5b) config var and
routes all background LLM calls to it, keeping qwen3:8b's KV cache
warm between user messages for consistent sub-second TTFT.

Also adds infinite scroll to KnowledgeView (replaces load-more button)
and bakes spaCy en_core_web_sm into the Docker image to eliminate the
pip install on every startup.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Dockerfile

@@ -19,7 +19,8 @@ RUN --mount=type=cache,target=/root/.cache/pip \
 # Install CPU-only torch first so pip doesn't pull full CUDA wheels (~2 GB) for kokoro/transformers.
 RUN --mount=type=cache,target=/root/.cache/pip \
     pip install torch --index-url https://download.pytorch.org/whl/cpu \
-    && pip install faster-whisper kokoro soundfile
+    && pip install faster-whisper kokoro soundfile \
+    && python -m spacy download en_core_web_sm
 
 # Build the fable-mcp wheel so it can be served for download
 COPY fable-mcp/ fable-mcp/

frontend/src/views/KnowledgeView.vue

@@ -1,5 +1,5 @@
 <script setup lang="ts">
-import { ref, computed, watch, onMounted, onUnmounted, nextTick } from "vue";
+import { ref, computed, watch, onMounted, onUnmounted, nextTick, watchEffect } from "vue";
 import { useRouter } from "vue-router";
 import { apiGet, apiPatch, transcribeAudio } from "@/api/client";
 import { fmtCompact } from "@/utils/dateFormat";
@@ -320,17 +320,45 @@ function formatDate(iso: string): string {
   return d.toLocaleDateString(undefined, { month: "short", day: "numeric", year: "numeric" });
 }
 
+// ─── Infinite scroll ──────────────────────────────────────────────────────────
+
+const sentinelEl = ref<HTMLElement | null>(null);
+let scrollObserver: IntersectionObserver | null = null;
+
+function setupScrollObserver() {
+  if (scrollObserver) scrollObserver.disconnect();
+  if (!sentinelEl.value) return;
+  scrollObserver = new IntersectionObserver(
+    (entries) => {
+      if (entries[0].isIntersecting && !loading.value && items.value.length < total.value) {
+        page.value++;
+        fetchItems();
+      }
+    },
+    { rootMargin: "200px" }
+  );
+  scrollObserver.observe(sentinelEl.value);
+}
+
 // ─── Lifecycle ────────────────────────────────────────────────────────────────
 
 onMounted(() => {
   fetchItems(true);
   fetchTags();
   fetchTodayBar();
-  nextTick(() => chatInputEl.value?.focus());
+  nextTick(() => {
+    chatInputEl.value?.focus();
+    setupScrollObserver();
+  });
 });
 
 onUnmounted(() => {
   if (searchDebounce) clearTimeout(searchDebounce);
+  scrollObserver?.disconnect();
+});
+
+watchEffect(() => {
+  if (sentinelEl.value) setupScrollObserver();
 });
 </script>
 
@@ -499,11 +527,9 @@ onUnmounted(() => {
           </div>
         </div>
 
-        <!-- Load more -->
+        <!-- Infinite scroll sentinel -->
-        <div v-if="items.length < total && !loading" class="load-more">
+        <div ref="sentinelEl" class="scroll-sentinel">
-          <button class="btn-load-more" @click="() => { page++; fetchItems() }">
+          <span v-if="loading && items.length > 0" class="sentinel-loading">Loading…</span>
-            Load more ({{ total - items.length }} remaining)
-          </button>
         </div>
       </div>
 
@@ -975,23 +1001,18 @@ onUnmounted(() => {
 }
 .empty-hint { font-size: 0.85rem; opacity: 0.7; }
 
-/* ── Load more ───────────────────────────────────────────── */
+/* ── Infinite scroll sentinel ────────────────────────────── */
-.load-more {
+.scroll-sentinel {
-  padding: 16px 20px;
+  height: 40px;
-  text-align: center;
+  display: flex;
+  align-items: center;
+  justify-content: center;
   flex-shrink: 0;
 }
-.btn-load-more {
+.sentinel-loading {
-  padding: 8px 20px;
+  font-size: 0.8rem;
-  border-radius: 8px;
-  border: 1px solid var(--color-border, rgba(255,255,255,0.1));
-  background: transparent;
   color: var(--color-muted);
-  cursor: pointer;
-  font-size: 0.85rem;
-  transition: all 0.15s;
 }
-.btn-load-more:hover { color: var(--color-text); border-color: rgba(255,255,255,0.2); }
 
 /* ── Graph panel ─────────────────────────────────────────── */
 .graph-panel {

src/fabledassistant/app.py

@@ -249,6 +249,7 @@ def create_app() -> Quart:
         # Also ensure the embedding model is pulled (no warm needed).
         asyncio.create_task(_warm_user_models())
         asyncio.create_task(_pull_model(Config.EMBEDDING_MODEL, warm=False))
+        asyncio.create_task(_pull_model(Config.OLLAMA_BACKGROUND_MODEL, warm=False))
 
         # After models are pulled, backfill embeddings for existing notes.
         # Runs in the background so it never blocks the server from accepting requests.

src/fabledassistant/config.py

@@ -24,6 +24,10 @@ class Config:
     )
     OLLAMA_URL: str = os.environ.get("OLLAMA_URL", "http://localhost:11434")
     OLLAMA_MODEL: str = os.environ.get("OLLAMA_MODEL", "qwen3:latest")
+    # Lightweight model for background tasks (title generation, tag suggestions,
+    # project summaries, RSS classification). Using a separate model keeps the
+    # main model's KV cache intact between user messages, enabling prefix cache hits.
+    OLLAMA_BACKGROUND_MODEL: str = os.environ.get("OLLAMA_BACKGROUND_MODEL", "qwen2.5:0.5b")
     # KV cache context window for generation. Keep this as small as practical —
     # a larger context forces more KV cache into CPU RAM, drastically slowing prefill.
     # 16384 covers ~30+ message conversations with our system prompt comfortably.

src/fabledassistant/services/chat.py

@@ -238,7 +238,7 @@ async def save_response_as_note(user_id: int, message_id: int) -> dict:
                 },
                 {"role": "user", "content": msg.content[:2000]},
             ]
-            title = await generate_completion(prompt_messages, model)
+            title = await generate_completion(prompt_messages, Config.OLLAMA_BACKGROUND_MODEL)
             title = title.strip().strip('"\'').strip()[:100]
         except Exception:
             logger.warning("Failed to generate note title, using fallback", exc_info=True)

src/fabledassistant/services/generation_task.py

@@ -138,7 +138,7 @@ async def _generate_title(messages: list[dict], model: str) -> str:
         },
         {"role": "user", "content": "\n\n".join(conv_lines)},
     ]
-    title = await generate_completion(prompt_messages, model, max_tokens=30)
+    title = await generate_completion(prompt_messages, Config.OLLAMA_BACKGROUND_MODEL, max_tokens=30)
     title = title.strip().strip('"\'').strip()
     return title[:100] if title else ""
 

src/fabledassistant/services/projects.py

@@ -122,7 +122,7 @@ async def generate_project_summary(user_id: int, project_id: int) -> None:
         from fabledassistant.services.llm import generate_completion
         from fabledassistant.config import Config
         messages = [{"role": "user", "content": prompt}]
-        summary = await generate_completion(messages, model=Config.OLLAMA_MODEL, max_tokens=400)
+        summary = await generate_completion(messages, model=Config.OLLAMA_BACKGROUND_MODEL, max_tokens=400)
         if not summary:
             return
 

src/fabledassistant/services/rss_classifier.py

@@ -64,7 +64,7 @@ async def classify_items_batch(
         return {}
 
     if model is None:
-        model = Config.OLLAMA_MODEL
+        model = Config.OLLAMA_BACKGROUND_MODEL
 
     vocab = STANDARD_TOPICS + [t for t in user_include_topics if t not in STANDARD_TOPICS]
     items_block = "\n".join(

src/fabledassistant/services/tag_suggestions.py

@@ -7,7 +7,6 @@ import re
 from fabledassistant.config import Config
 from fabledassistant.services.llm import generate_completion
 from fabledassistant.services.notes import get_all_tags
-from fabledassistant.services.settings import get_setting
 
 logger = logging.getLogger(__name__)
 
@@ -22,7 +21,7 @@ async def suggest_tags(user_id: int, title: str, body: str, current_tags: list[s
         return []
 
     existing_tags = await get_all_tags(user_id)
-    model = await get_setting(user_id, "default_model", Config.OLLAMA_MODEL)
+    model = Config.OLLAMA_BACKGROUND_MODEL
 
     existing_list = ", ".join(f"#{t}" for t in existing_tags) if existing_tags else "(none yet)"