diff --git a/src/fabledassistant/app.py b/src/fabledassistant/app.py index 3798017..9a76231 100644 --- a/src/fabledassistant/app.py +++ b/src/fabledassistant/app.py @@ -169,11 +169,12 @@ def create_app() -> Quart: async def _warm_model(model: str) -> None: """Warm an already-installed model into VRAM (no pull).""" + from fabledassistant.services.llm import keep_alive_for try: async with httpx.AsyncClient(timeout=300.0) as client: await client.post( f"{Config.OLLAMA_URL}/api/generate", - json={"model": model, "prompt": "", "keep_alive": "2h"}, + json={"model": model, "prompt": "", "keep_alive": keep_alive_for(model)}, ) logger.info("Warmed model '%s' into VRAM", model) except Exception: @@ -187,7 +188,7 @@ def create_app() -> Quart: The num_ctx must match what real requests will use so Ollama doesn't reload. """ try: - from fabledassistant.services.llm import build_context, pick_num_ctx + from fabledassistant.services.llm import build_context, keep_alive_for, pick_num_ctx messages, _ = await build_context( user_id=user_id, history=[], @@ -203,7 +204,7 @@ def create_app() -> Quart: "messages": messages, "stream": False, "options": {"num_predict": 1, "num_ctx": num_ctx}, - "keep_alive": "2h", + "keep_alive": keep_alive_for(model), }, ) logger.info("Primed KV cache for user %d with model '%s' (num_ctx=%d)", user_id, model, num_ctx) diff --git a/src/fabledassistant/config.py b/src/fabledassistant/config.py index 9a69db9..ebb9802 100644 --- a/src/fabledassistant/config.py +++ b/src/fabledassistant/config.py @@ -28,6 +28,12 @@ class Config: # project summaries, RSS classification). Using a separate model keeps the # main model's KV cache intact between user messages, enabling prefix cache hits. OLLAMA_BACKGROUND_MODEL: str = os.environ.get("OLLAMA_BACKGROUND_MODEL", "qwen2.5:3b") + # Ollama keep_alive — how long a model stays resident in VRAM after its last + # request. Main model gets a longer window since it's used interactively; + # the background model is called sporadically and doesn't need to camp VRAM. + # Format matches Ollama's duration strings: "30m", "10m", "1h", "0s", "-1" (forever). + OLLAMA_KEEP_ALIVE_MAIN: str = os.environ.get("OLLAMA_KEEP_ALIVE_MAIN", "30m") + OLLAMA_KEEP_ALIVE_BACKGROUND: str = os.environ.get("OLLAMA_KEEP_ALIVE_BACKGROUND", "10m") # KV cache context window for generation. Keep this as small as practical — # a larger context forces more KV cache into CPU RAM, drastically slowing prefill. # 16384 covers ~30+ message conversations with our system prompt comfortably. diff --git a/src/fabledassistant/routes/chat.py b/src/fabledassistant/routes/chat.py index 8bfa2b7..cf9ed73 100644 --- a/src/fabledassistant/routes/chat.py +++ b/src/fabledassistant/routes/chat.py @@ -350,11 +350,12 @@ async def warm_model_route(): return jsonify({"error": "model is required"}), 400 async def _warm(): + from fabledassistant.services.llm import keep_alive_for try: async with httpx.AsyncClient(timeout=300.0) as client: await client.post( f"{Config.OLLAMA_URL}/api/generate", - json={"model": model, "prompt": "", "keep_alive": "30m"}, + json={"model": model, "prompt": "", "keep_alive": keep_alive_for(model)}, ) logger.info("Warmed model %s", model) except Exception as e: diff --git a/src/fabledassistant/routes/settings.py b/src/fabledassistant/routes/settings.py index ad75d23..422d6e0 100644 --- a/src/fabledassistant/routes/settings.py +++ b/src/fabledassistant/routes/settings.py @@ -24,6 +24,7 @@ async def _prime_kv_cache_bg(user_id: int, model: str) -> None: user_message=" ", ) num_ctx = pick_num_ctx(messages) + from fabledassistant.services.llm import keep_alive_for async with httpx.AsyncClient(timeout=120.0) as client: await client.post( f"{Config.OLLAMA_URL}/api/chat", @@ -32,7 +33,7 @@ async def _prime_kv_cache_bg(user_id: int, model: str) -> None: "messages": messages, "stream": False, "options": {"num_predict": 1, "num_ctx": num_ctx}, - "keep_alive": "2h", + "keep_alive": keep_alive_for(model), }, ) logger.info("Primed KV cache for user %d with model '%s' (num_ctx=%d)", user_id, model, num_ctx) diff --git a/src/fabledassistant/services/llm.py b/src/fabledassistant/services/llm.py index d1872e5..1dbc315 100644 --- a/src/fabledassistant/services/llm.py +++ b/src/fabledassistant/services/llm.py @@ -26,6 +26,18 @@ logger = logging.getLogger(__name__) _CTX_TIERS = (8192, 16384, 32768) +def keep_alive_for(model: str) -> str: + """Return the Ollama keep_alive duration for *model*. + + Background models get a shorter window because they're called + sporadically; the main interactive model gets a longer one so it + stays warm between user messages. + """ + if model == Config.OLLAMA_BACKGROUND_MODEL: + return Config.OLLAMA_KEEP_ALIVE_BACKGROUND + return Config.OLLAMA_KEEP_ALIVE_MAIN + + def pick_num_ctx(messages: list[dict]) -> int: """Return the smallest context tier that fits *messages* with 25% headroom. @@ -145,7 +157,7 @@ async def stream_chat( merged_options = {"num_ctx": num_ctx or Config.OLLAMA_NUM_CTX} if options: merged_options.update(options) - payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options, "think": think, "keep_alive": "2h"} + payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options, "think": think, "keep_alive": keep_alive_for(model)} # read=None: no per-chunk timeout — Ollama may pause for any duration while # processing a large input context before the first token arrives. async with httpx.AsyncClient(timeout=httpx.Timeout(connect=30.0, read=None, write=None, pool=30.0)) as client: @@ -204,7 +216,7 @@ async def stream_chat_with_tools( "stream": True, "options": options, "think": think, - "keep_alive": "2h", + "keep_alive": keep_alive_for(model), } if tools: payload["tools"] = tools @@ -289,7 +301,7 @@ async def generate_completion( "stream": False, "think": False, "options": options, - "keep_alive": "2h", + "keep_alive": keep_alive_for(model), }, ) resp.raise_for_status()