feat(ollama): configurable per-model keep_alive durations
Replace the hardcoded "2h" keep_alive everywhere with a helper that returns OLLAMA_KEEP_ALIVE_MAIN (default 30m) for the interactive model and OLLAMA_KEEP_ALIVE_BACKGROUND (default 10m) for the background model. Lets the main model release VRAM during long idle periods while keeping it warm enough for bursty chat use, and stops the sporadic background model from camping VRAM it rarely needs. Seven call sites updated to route through llm.keep_alive_for(model): the streaming helpers, generate_completion, the two startup warmers, the settings KV-cache primer, and the chat warmer endpoint. Override via env vars: OLLAMA_KEEP_ALIVE_MAIN, OLLAMA_KEEP_ALIVE_BACKGROUND. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -169,11 +169,12 @@ def create_app() -> Quart:
|
||||
|
||||
async def _warm_model(model: str) -> None:
|
||||
"""Warm an already-installed model into VRAM (no pull)."""
|
||||
from fabledassistant.services.llm import keep_alive_for
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=300.0) as client:
|
||||
await client.post(
|
||||
f"{Config.OLLAMA_URL}/api/generate",
|
||||
json={"model": model, "prompt": "", "keep_alive": "2h"},
|
||||
json={"model": model, "prompt": "", "keep_alive": keep_alive_for(model)},
|
||||
)
|
||||
logger.info("Warmed model '%s' into VRAM", model)
|
||||
except Exception:
|
||||
@@ -187,7 +188,7 @@ def create_app() -> Quart:
|
||||
The num_ctx must match what real requests will use so Ollama doesn't reload.
|
||||
"""
|
||||
try:
|
||||
from fabledassistant.services.llm import build_context, pick_num_ctx
|
||||
from fabledassistant.services.llm import build_context, keep_alive_for, pick_num_ctx
|
||||
messages, _ = await build_context(
|
||||
user_id=user_id,
|
||||
history=[],
|
||||
@@ -203,7 +204,7 @@ def create_app() -> Quart:
|
||||
"messages": messages,
|
||||
"stream": False,
|
||||
"options": {"num_predict": 1, "num_ctx": num_ctx},
|
||||
"keep_alive": "2h",
|
||||
"keep_alive": keep_alive_for(model),
|
||||
},
|
||||
)
|
||||
logger.info("Primed KV cache for user %d with model '%s' (num_ctx=%d)", user_id, model, num_ctx)
|
||||
|
||||
Reference in New Issue
Block a user