fix: set keep_alive to 2h on all Ollama requests

Prevents models from sitting in VRAM indefinitely. Applies to both
streaming chat calls and the non-streaming generate_completion path,
as well as the startup warm-up request.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-30 10:21:59 -04:00
parent dd304bb556
commit 2a8c0cfa56
2 changed files with 4 additions and 2 deletions
+1 -1
View File
@@ -169,7 +169,7 @@ def create_app() -> Quart:
async with httpx.AsyncClient(timeout=300.0) as client:
await client.post(
f"{Config.OLLAMA_URL}/api/generate",
json={"model": model, "prompt": "", "keep_alive": "30m"},
json={"model": model, "prompt": "", "keep_alive": "2h"},
)
logger.info("Warmed model '%s' into VRAM", model)
except Exception:
+3 -1
View File
@@ -122,7 +122,7 @@ async def stream_chat(
merged_options = {"num_ctx": Config.OLLAMA_NUM_CTX}
if options:
merged_options.update(options)
payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options, "think": think}
payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options, "think": think, "keep_alive": "2h"}
# read=None: no per-chunk timeout — Ollama may pause for any duration while
# processing a large input context before the first token arrives.
async with httpx.AsyncClient(timeout=httpx.Timeout(connect=30.0, read=None, write=None, pool=30.0)) as client:
@@ -176,6 +176,7 @@ async def stream_chat_with_tools(
"stream": True,
"options": options,
"think": think,
"keep_alive": "2h",
}
if tools:
payload["tools"] = tools
@@ -256,6 +257,7 @@ async def generate_completion(
"stream": False,
"think": False,
"options": options,
"keep_alive": "2h",
},
)
resp.raise_for_status()