feat(ollama): configurable per-model keep_alive durations

Replace the hardcoded "2h" keep_alive everywhere with a helper that
returns OLLAMA_KEEP_ALIVE_MAIN (default 30m) for the interactive model
and OLLAMA_KEEP_ALIVE_BACKGROUND (default 10m) for the background
model. Lets the main model release VRAM during long idle periods
while keeping it warm enough for bursty chat use, and stops the
sporadic background model from camping VRAM it rarely needs.

Seven call sites updated to route through llm.keep_alive_for(model):
the streaming helpers, generate_completion, the two startup warmers,
the settings KV-cache primer, and the chat warmer endpoint.

Override via env vars: OLLAMA_KEEP_ALIVE_MAIN, OLLAMA_KEEP_ALIVE_BACKGROUND.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

src/fabledassistant/app.py

@@ -169,11 +169,12 @@ def create_app() -> Quart:
 
         async def _warm_model(model: str) -> None:
             """Warm an already-installed model into VRAM (no pull)."""
+            from fabledassistant.services.llm import keep_alive_for
             try:
                 async with httpx.AsyncClient(timeout=300.0) as client:
                     await client.post(
                         f"{Config.OLLAMA_URL}/api/generate",
-                        json={"model": model, "prompt": "", "keep_alive": "2h"},
+                        json={"model": model, "prompt": "", "keep_alive": keep_alive_for(model)},
                     )
                 logger.info("Warmed model '%s' into VRAM", model)
             except Exception:
@@ -187,7 +188,7 @@ def create_app() -> Quart:
             The num_ctx must match what real requests will use so Ollama doesn't reload.
             """
             try:
-                from fabledassistant.services.llm import build_context, pick_num_ctx
+                from fabledassistant.services.llm import build_context, keep_alive_for, pick_num_ctx
                 messages, _ = await build_context(
                     user_id=user_id,
                     history=[],
@@ -203,7 +204,7 @@ def create_app() -> Quart:
                             "messages": messages,
                             "stream": False,
                             "options": {"num_predict": 1, "num_ctx": num_ctx},
-                            "keep_alive": "2h",
+                            "keep_alive": keep_alive_for(model),
                         },
                     )
                 logger.info("Primed KV cache for user %d with model '%s' (num_ctx=%d)", user_id, model, num_ctx)

src/fabledassistant/config.py

@@ -28,6 +28,12 @@ class Config:
     # project summaries, RSS classification). Using a separate model keeps the
     # main model's KV cache intact between user messages, enabling prefix cache hits.
     OLLAMA_BACKGROUND_MODEL: str = os.environ.get("OLLAMA_BACKGROUND_MODEL", "qwen2.5:3b")
+    # Ollama keep_alive — how long a model stays resident in VRAM after its last
+    # request. Main model gets a longer window since it's used interactively;
+    # the background model is called sporadically and doesn't need to camp VRAM.
+    # Format matches Ollama's duration strings: "30m", "10m", "1h", "0s", "-1" (forever).
+    OLLAMA_KEEP_ALIVE_MAIN: str = os.environ.get("OLLAMA_KEEP_ALIVE_MAIN", "30m")
+    OLLAMA_KEEP_ALIVE_BACKGROUND: str = os.environ.get("OLLAMA_KEEP_ALIVE_BACKGROUND", "10m")
     # KV cache context window for generation. Keep this as small as practical —
     # a larger context forces more KV cache into CPU RAM, drastically slowing prefill.
     # 16384 covers ~30+ message conversations with our system prompt comfortably.

src/fabledassistant/routes/chat.py

@@ -350,11 +350,12 @@ async def warm_model_route():
         return jsonify({"error": "model is required"}), 400
 
     async def _warm():
+        from fabledassistant.services.llm import keep_alive_for
         try:
             async with httpx.AsyncClient(timeout=300.0) as client:
                 await client.post(
                     f"{Config.OLLAMA_URL}/api/generate",
-                    json={"model": model, "prompt": "", "keep_alive": "30m"},
+                    json={"model": model, "prompt": "", "keep_alive": keep_alive_for(model)},
                 )
             logger.info("Warmed model %s", model)
         except Exception as e:

src/fabledassistant/routes/settings.py

@@ -24,6 +24,7 @@ async def _prime_kv_cache_bg(user_id: int, model: str) -> None:
             user_message=" ",
         )
         num_ctx = pick_num_ctx(messages)
+        from fabledassistant.services.llm import keep_alive_for
         async with httpx.AsyncClient(timeout=120.0) as client:
             await client.post(
                 f"{Config.OLLAMA_URL}/api/chat",
@@ -32,7 +33,7 @@ async def _prime_kv_cache_bg(user_id: int, model: str) -> None:
                     "messages": messages,
                     "stream": False,
                     "options": {"num_predict": 1, "num_ctx": num_ctx},
-                    "keep_alive": "2h",
+                    "keep_alive": keep_alive_for(model),
                 },
             )
         logger.info("Primed KV cache for user %d with model '%s' (num_ctx=%d)", user_id, model, num_ctx)

src/fabledassistant/services/llm.py

@@ -26,6 +26,18 @@ logger = logging.getLogger(__name__)
 _CTX_TIERS = (8192, 16384, 32768)
 
 
+def keep_alive_for(model: str) -> str:
+    """Return the Ollama keep_alive duration for *model*.
+
+    Background models get a shorter window because they're called
+    sporadically; the main interactive model gets a longer one so it
+    stays warm between user messages.
+    """
+    if model == Config.OLLAMA_BACKGROUND_MODEL:
+        return Config.OLLAMA_KEEP_ALIVE_BACKGROUND
+    return Config.OLLAMA_KEEP_ALIVE_MAIN
+
+
 def pick_num_ctx(messages: list[dict]) -> int:
     """Return the smallest context tier that fits *messages* with 25% headroom.
 
@@ -145,7 +157,7 @@ async def stream_chat(
     merged_options = {"num_ctx": num_ctx or Config.OLLAMA_NUM_CTX}
     if options:
         merged_options.update(options)
-    payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options, "think": think, "keep_alive": "2h"}
+    payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options, "think": think, "keep_alive": keep_alive_for(model)}
     # read=None: no per-chunk timeout — Ollama may pause for any duration while
     # processing a large input context before the first token arrives.
     async with httpx.AsyncClient(timeout=httpx.Timeout(connect=30.0, read=None, write=None, pool=30.0)) as client:
@@ -204,7 +216,7 @@ async def stream_chat_with_tools(
         "stream": True,
         "options": options,
         "think": think,
-        "keep_alive": "2h",
+        "keep_alive": keep_alive_for(model),
     }
     if tools:
         payload["tools"] = tools
@@ -289,7 +301,7 @@ async def generate_completion(
                         "stream": False,
                         "think": False,
                         "options": options,
-                        "keep_alive": "2h",
+                        "keep_alive": keep_alive_for(model),
                     },
                 )
                 resp.raise_for_status()