feat(ollama): configurable per-model keep_alive durations

Replace the hardcoded "2h" keep_alive everywhere with a helper that
returns OLLAMA_KEEP_ALIVE_MAIN (default 30m) for the interactive model
and OLLAMA_KEEP_ALIVE_BACKGROUND (default 10m) for the background
model. Lets the main model release VRAM during long idle periods
while keeping it warm enough for bursty chat use, and stops the
sporadic background model from camping VRAM it rarely needs.

Seven call sites updated to route through llm.keep_alive_for(model):
the streaming helpers, generate_completion, the two startup warmers,
the settings KV-cache primer, and the chat warmer endpoint.

Override via env vars: OLLAMA_KEEP_ALIVE_MAIN, OLLAMA_KEEP_ALIVE_BACKGROUND.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Bryan Van Deusen
2026-04-10 14:13:32 -04:00
parent 102c0b74a0
commit 3f3156db07
5 changed files with 29 additions and 8 deletions
+2 -1
View File
@@ -350,11 +350,12 @@ async def warm_model_route():
return jsonify({"error": "model is required"}), 400
async def _warm():
from fabledassistant.services.llm import keep_alive_for
try:
async with httpx.AsyncClient(timeout=300.0) as client:
await client.post(
f"{Config.OLLAMA_URL}/api/generate",
json={"model": model, "prompt": "", "keep_alive": "30m"},
json={"model": model, "prompt": "", "keep_alive": keep_alive_for(model)},
)
logger.info("Warmed model %s", model)
except Exception as e:
+2 -1
View File
@@ -24,6 +24,7 @@ async def _prime_kv_cache_bg(user_id: int, model: str) -> None:
user_message=" ",
)
num_ctx = pick_num_ctx(messages)
from fabledassistant.services.llm import keep_alive_for
async with httpx.AsyncClient(timeout=120.0) as client:
await client.post(
f"{Config.OLLAMA_URL}/api/chat",
@@ -32,7 +33,7 @@ async def _prime_kv_cache_bg(user_id: int, model: str) -> None:
"messages": messages,
"stream": False,
"options": {"num_predict": 1, "num_ctx": num_ctx},
"keep_alive": "2h",
"keep_alive": keep_alive_for(model),
},
)
logger.info("Primed KV cache for user %d with model '%s' (num_ctx=%d)", user_id, model, num_ctx)