fix: set keep_alive to 2h on all Ollama requests
Prevents models from sitting in VRAM indefinitely. Applies to both streaming chat calls and the non-streaming generate_completion path, as well as the startup warm-up request. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -169,7 +169,7 @@ def create_app() -> Quart:
|
||||
async with httpx.AsyncClient(timeout=300.0) as client:
|
||||
await client.post(
|
||||
f"{Config.OLLAMA_URL}/api/generate",
|
||||
json={"model": model, "prompt": "", "keep_alive": "30m"},
|
||||
json={"model": model, "prompt": "", "keep_alive": "2h"},
|
||||
)
|
||||
logger.info("Warmed model '%s' into VRAM", model)
|
||||
except Exception:
|
||||
|
||||
@@ -122,7 +122,7 @@ async def stream_chat(
|
||||
merged_options = {"num_ctx": Config.OLLAMA_NUM_CTX}
|
||||
if options:
|
||||
merged_options.update(options)
|
||||
payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options, "think": think}
|
||||
payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options, "think": think, "keep_alive": "2h"}
|
||||
# read=None: no per-chunk timeout — Ollama may pause for any duration while
|
||||
# processing a large input context before the first token arrives.
|
||||
async with httpx.AsyncClient(timeout=httpx.Timeout(connect=30.0, read=None, write=None, pool=30.0)) as client:
|
||||
@@ -176,6 +176,7 @@ async def stream_chat_with_tools(
|
||||
"stream": True,
|
||||
"options": options,
|
||||
"think": think,
|
||||
"keep_alive": "2h",
|
||||
}
|
||||
if tools:
|
||||
payload["tools"] = tools
|
||||
@@ -256,6 +257,7 @@ async def generate_completion(
|
||||
"stream": False,
|
||||
"think": False,
|
||||
"options": options,
|
||||
"keep_alive": "2h",
|
||||
},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
|
||||
Reference in New Issue
Block a user