diff --git a/src/fabledassistant/app.py b/src/fabledassistant/app.py index 8ab854f..5ee4acf 100644 --- a/src/fabledassistant/app.py +++ b/src/fabledassistant/app.py @@ -169,7 +169,7 @@ def create_app() -> Quart: async with httpx.AsyncClient(timeout=300.0) as client: await client.post( f"{Config.OLLAMA_URL}/api/generate", - json={"model": model, "prompt": "", "keep_alive": "30m"}, + json={"model": model, "prompt": "", "keep_alive": "2h"}, ) logger.info("Warmed model '%s' into VRAM", model) except Exception: diff --git a/src/fabledassistant/services/llm.py b/src/fabledassistant/services/llm.py index 935cf68..f24fc55 100644 --- a/src/fabledassistant/services/llm.py +++ b/src/fabledassistant/services/llm.py @@ -122,7 +122,7 @@ async def stream_chat( merged_options = {"num_ctx": Config.OLLAMA_NUM_CTX} if options: merged_options.update(options) - payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options, "think": think} + payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options, "think": think, "keep_alive": "2h"} # read=None: no per-chunk timeout — Ollama may pause for any duration while # processing a large input context before the first token arrives. async with httpx.AsyncClient(timeout=httpx.Timeout(connect=30.0, read=None, write=None, pool=30.0)) as client: @@ -176,6 +176,7 @@ async def stream_chat_with_tools( "stream": True, "options": options, "think": think, + "keep_alive": "2h", } if tools: payload["tools"] = tools @@ -256,6 +257,7 @@ async def generate_completion( "stream": False, "think": False, "options": options, + "keep_alive": "2h", }, ) resp.raise_for_status()