From b23e78b0ac066f743b98d2d97f485d4ec69cce64 Mon Sep 17 00:00:00 2001
From: Bryan Van Deusen <bryan.vandeusen@fabledsword.com>
Date: Wed, 25 Feb 2026 23:18:07 -0500
Subject: [PATCH] Warm both chat and intent models into VRAM on startup

ensure_model only downloads a model if missing; it does not load it
into VRAM. The frontend warm call only covers the chat model (and only
after a user opens the dashboard). This left qwen2.5:1.5b (intent) cold,
causing simultaneous cold-load 500s when the first chat arrived.

Now both Config.OLLAMA_MODEL and Config.OLLAMA_INTENT_MODEL are warmed
at startup (after ensuring they're installed) via a fire-and-forget
/api/generate call with keep_alive=30m. The embedding model is still
pulled but not warmed (it's loaded on demand during backfill).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 src/fabledassistant/app.py | 34 +++++++++++++++++++++++++---------
 1 file changed, 25 insertions(+), 9 deletions(-)

diff --git a/src/fabledassistant/app.py b/src/fabledassistant/app.py
index 6073f1f..5824dba 100644
--- a/src/fabledassistant/app.py
+++ b/src/fabledassistant/app.py
@@ -3,6 +3,8 @@ import time
 import traceback as tb_module
 from pathlib import Path
 
+import httpx
+
 from quart import Quart, g, jsonify, make_response, request, send_from_directory
 
 from fabledassistant.config import Config
@@ -104,7 +106,7 @@ def create_app() -> Quart:
         start_log_retention_loop()
         start_notification_loop()
 
-        async def _pull_model(model: str) -> None:
+        async def _pull_model(model: str, warm: bool = False) -> None:
             try:
                 await ensure_model(model)
             except Exception:
@@ -113,16 +115,30 @@ def create_app() -> Quart:
                     model,
                     exc_info=True,
                 )
+                return
+            if warm:
+                try:
+                    async with httpx.AsyncClient(timeout=300.0) as client:
+                        await client.post(
+                            f"{Config.OLLAMA_URL}/api/generate",
+                            json={"model": model, "prompt": "", "keep_alive": "30m"},
+                        )
+                    logger.info("Warmed model '%s' into VRAM", model)
+                except Exception:
+                    logger.warning("Failed to warm model '%s'", model, exc_info=True)
 
-        # Pull main model and (if configured) a separate intent model concurrently.
-        # Fire-and-forget so pulls don't block startup.
-        models_to_pull = {Config.OLLAMA_MODEL}
+        # Pull main model and (if configured) a separate intent model concurrently,
+        # then warm them into VRAM so the first user request doesn't cold-load both
+        # simultaneously (which causes Ollama 500 races).
+        # Fire-and-forget so pulls/warming don't block startup.
+        models_to_warm = {Config.OLLAMA_MODEL}
         if Config.OLLAMA_INTENT_MODEL and Config.OLLAMA_INTENT_MODEL != Config.OLLAMA_MODEL:
-            models_to_pull.add(Config.OLLAMA_INTENT_MODEL)
-        # Also pull the embedding model (nomic-embed-text by default).
-        models_to_pull.add(Config.EMBEDDING_MODEL)
-        for _model in models_to_pull:
-            asyncio.create_task(_pull_model(_model))
+            models_to_warm.add(Config.OLLAMA_INTENT_MODEL)
+        for _model in models_to_warm:
+            asyncio.create_task(_pull_model(_model, warm=True))
+        # Also pull the embedding model (nomic-embed-text by default), but no need to warm it.
+        if Config.EMBEDDING_MODEL not in models_to_warm:
+            asyncio.create_task(_pull_model(Config.EMBEDDING_MODEL, warm=False))
 
         # After models are pulled, backfill embeddings for existing notes.
         # Runs in the background so it never blocks the server from accepting requests.