From 2a8c0cfa5686b249082e1f122acff11d4f5a32e0 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 30 Mar 2026 10:21:59 -0400 Subject: [PATCH] fix: set keep_alive to 2h on all Ollama requests Prevents models from sitting in VRAM indefinitely. Applies to both streaming chat calls and the non-streaming generate_completion path, as well as the startup warm-up request. Co-Authored-By: Claude Sonnet 4.6 --- src/fabledassistant/app.py | 2 +- src/fabledassistant/services/llm.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/fabledassistant/app.py b/src/fabledassistant/app.py index 8ab854f..5ee4acf 100644 --- a/src/fabledassistant/app.py +++ b/src/fabledassistant/app.py @@ -169,7 +169,7 @@ def create_app() -> Quart: async with httpx.AsyncClient(timeout=300.0) as client: await client.post( f"{Config.OLLAMA_URL}/api/generate", - json={"model": model, "prompt": "", "keep_alive": "30m"}, + json={"model": model, "prompt": "", "keep_alive": "2h"}, ) logger.info("Warmed model '%s' into VRAM", model) except Exception: diff --git a/src/fabledassistant/services/llm.py b/src/fabledassistant/services/llm.py index 935cf68..f24fc55 100644 --- a/src/fabledassistant/services/llm.py +++ b/src/fabledassistant/services/llm.py @@ -122,7 +122,7 @@ async def stream_chat( merged_options = {"num_ctx": Config.OLLAMA_NUM_CTX} if options: merged_options.update(options) - payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options, "think": think} + payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options, "think": think, "keep_alive": "2h"} # read=None: no per-chunk timeout — Ollama may pause for any duration while # processing a large input context before the first token arrives. async with httpx.AsyncClient(timeout=httpx.Timeout(connect=30.0, read=None, write=None, pool=30.0)) as client: @@ -176,6 +176,7 @@ async def stream_chat_with_tools( "stream": True, "options": options, "think": think, + "keep_alive": "2h", } if tools: payload["tools"] = tools @@ -256,6 +257,7 @@ async def generate_completion( "stream": False, "think": False, "options": options, + "keep_alive": "2h", }, ) resp.raise_for_status()