From 750a91898a38b8dd5c3ef01a2ae669313fda967d Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 3 Apr 2026 01:33:54 -0400 Subject: [PATCH] perf(llm): route background tasks to dedicated model to preserve KV cache Background tasks (title generation, tag suggestions, project summaries, RSS classification) were using qwen3:8b and wiping its KV cache after every response, preventing prefix cache hits on subsequent user messages. Adds OLLAMA_BACKGROUND_MODEL (default: qwen2.5:0.5b) config var and routes all background LLM calls to it, keeping qwen3:8b's KV cache warm between user messages for consistent sub-second TTFT. Also adds infinite scroll to KnowledgeView (replaces load-more button) and bakes spaCy en_core_web_sm into the Docker image to eliminate the pip install on every startup. Co-Authored-By: Claude Sonnet 4.6 --- Dockerfile | 3 +- frontend/src/views/KnowledgeView.vue | 61 +++++++++++++------ src/fabledassistant/app.py | 1 + src/fabledassistant/config.py | 4 ++ src/fabledassistant/services/chat.py | 2 +- .../services/generation_task.py | 2 +- src/fabledassistant/services/projects.py | 2 +- .../services/rss_classifier.py | 2 +- .../services/tag_suggestions.py | 3 +- 9 files changed, 53 insertions(+), 27 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9892895..931d8cd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,8 @@ RUN --mount=type=cache,target=/root/.cache/pip \ # Install CPU-only torch first so pip doesn't pull full CUDA wheels (~2 GB) for kokoro/transformers. RUN --mount=type=cache,target=/root/.cache/pip \ pip install torch --index-url https://download.pytorch.org/whl/cpu \ - && pip install faster-whisper kokoro soundfile + && pip install faster-whisper kokoro soundfile \ + && python -m spacy download en_core_web_sm # Build the fable-mcp wheel so it can be served for download COPY fable-mcp/ fable-mcp/ diff --git a/frontend/src/views/KnowledgeView.vue b/frontend/src/views/KnowledgeView.vue index 88245af..fc6737a 100644 --- a/frontend/src/views/KnowledgeView.vue +++ b/frontend/src/views/KnowledgeView.vue @@ -1,5 +1,5 @@ @@ -499,11 +527,9 @@ onUnmounted(() => { - -
- + +
+ Loading…
@@ -975,23 +1001,18 @@ onUnmounted(() => { } .empty-hint { font-size: 0.85rem; opacity: 0.7; } -/* ── Load more ───────────────────────────────────────────── */ -.load-more { - padding: 16px 20px; - text-align: center; +/* ── Infinite scroll sentinel ────────────────────────────── */ +.scroll-sentinel { + height: 40px; + display: flex; + align-items: center; + justify-content: center; flex-shrink: 0; } -.btn-load-more { - padding: 8px 20px; - border-radius: 8px; - border: 1px solid var(--color-border, rgba(255,255,255,0.1)); - background: transparent; +.sentinel-loading { + font-size: 0.8rem; color: var(--color-muted); - cursor: pointer; - font-size: 0.85rem; - transition: all 0.15s; } -.btn-load-more:hover { color: var(--color-text); border-color: rgba(255,255,255,0.2); } /* ── Graph panel ─────────────────────────────────────────── */ .graph-panel { diff --git a/src/fabledassistant/app.py b/src/fabledassistant/app.py index b5d45a0..592805b 100644 --- a/src/fabledassistant/app.py +++ b/src/fabledassistant/app.py @@ -249,6 +249,7 @@ def create_app() -> Quart: # Also ensure the embedding model is pulled (no warm needed). asyncio.create_task(_warm_user_models()) asyncio.create_task(_pull_model(Config.EMBEDDING_MODEL, warm=False)) + asyncio.create_task(_pull_model(Config.OLLAMA_BACKGROUND_MODEL, warm=False)) # After models are pulled, backfill embeddings for existing notes. # Runs in the background so it never blocks the server from accepting requests. diff --git a/src/fabledassistant/config.py b/src/fabledassistant/config.py index 8897fda..2675f3c 100644 --- a/src/fabledassistant/config.py +++ b/src/fabledassistant/config.py @@ -24,6 +24,10 @@ class Config: ) OLLAMA_URL: str = os.environ.get("OLLAMA_URL", "http://localhost:11434") OLLAMA_MODEL: str = os.environ.get("OLLAMA_MODEL", "qwen3:latest") + # Lightweight model for background tasks (title generation, tag suggestions, + # project summaries, RSS classification). Using a separate model keeps the + # main model's KV cache intact between user messages, enabling prefix cache hits. + OLLAMA_BACKGROUND_MODEL: str = os.environ.get("OLLAMA_BACKGROUND_MODEL", "qwen2.5:0.5b") # KV cache context window for generation. Keep this as small as practical — # a larger context forces more KV cache into CPU RAM, drastically slowing prefill. # 16384 covers ~30+ message conversations with our system prompt comfortably. diff --git a/src/fabledassistant/services/chat.py b/src/fabledassistant/services/chat.py index de00cd9..9e1f9e2 100644 --- a/src/fabledassistant/services/chat.py +++ b/src/fabledassistant/services/chat.py @@ -238,7 +238,7 @@ async def save_response_as_note(user_id: int, message_id: int) -> dict: }, {"role": "user", "content": msg.content[:2000]}, ] - title = await generate_completion(prompt_messages, model) + title = await generate_completion(prompt_messages, Config.OLLAMA_BACKGROUND_MODEL) title = title.strip().strip('"\'').strip()[:100] except Exception: logger.warning("Failed to generate note title, using fallback", exc_info=True) diff --git a/src/fabledassistant/services/generation_task.py b/src/fabledassistant/services/generation_task.py index e7fa185..fbb19e9 100644 --- a/src/fabledassistant/services/generation_task.py +++ b/src/fabledassistant/services/generation_task.py @@ -138,7 +138,7 @@ async def _generate_title(messages: list[dict], model: str) -> str: }, {"role": "user", "content": "\n\n".join(conv_lines)}, ] - title = await generate_completion(prompt_messages, model, max_tokens=30) + title = await generate_completion(prompt_messages, Config.OLLAMA_BACKGROUND_MODEL, max_tokens=30) title = title.strip().strip('"\'').strip() return title[:100] if title else "" diff --git a/src/fabledassistant/services/projects.py b/src/fabledassistant/services/projects.py index 4e50655..799ad50 100644 --- a/src/fabledassistant/services/projects.py +++ b/src/fabledassistant/services/projects.py @@ -122,7 +122,7 @@ async def generate_project_summary(user_id: int, project_id: int) -> None: from fabledassistant.services.llm import generate_completion from fabledassistant.config import Config messages = [{"role": "user", "content": prompt}] - summary = await generate_completion(messages, model=Config.OLLAMA_MODEL, max_tokens=400) + summary = await generate_completion(messages, model=Config.OLLAMA_BACKGROUND_MODEL, max_tokens=400) if not summary: return diff --git a/src/fabledassistant/services/rss_classifier.py b/src/fabledassistant/services/rss_classifier.py index 6b368e3..a9402ca 100644 --- a/src/fabledassistant/services/rss_classifier.py +++ b/src/fabledassistant/services/rss_classifier.py @@ -64,7 +64,7 @@ async def classify_items_batch( return {} if model is None: - model = Config.OLLAMA_MODEL + model = Config.OLLAMA_BACKGROUND_MODEL vocab = STANDARD_TOPICS + [t for t in user_include_topics if t not in STANDARD_TOPICS] items_block = "\n".join( diff --git a/src/fabledassistant/services/tag_suggestions.py b/src/fabledassistant/services/tag_suggestions.py index 4cd9772..6ae0e5e 100644 --- a/src/fabledassistant/services/tag_suggestions.py +++ b/src/fabledassistant/services/tag_suggestions.py @@ -7,7 +7,6 @@ import re from fabledassistant.config import Config from fabledassistant.services.llm import generate_completion from fabledassistant.services.notes import get_all_tags -from fabledassistant.services.settings import get_setting logger = logging.getLogger(__name__) @@ -22,7 +21,7 @@ async def suggest_tags(user_id: int, title: str, body: str, current_tags: list[s return [] existing_tags = await get_all_tags(user_id) - model = await get_setting(user_id, "default_model", Config.OLLAMA_MODEL) + model = Config.OLLAMA_BACKGROUND_MODEL existing_list = ", ".join(f"#{t}" for t in existing_tags) if existing_tags else "(none yet)"