bvandeusen/FabledScribe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 50 Wiki Activity

fix: set keep_alive to 2h on all Ollama requests

Browse Source 
Prevents models from sitting in VRAM indefinitely. Applies to both
streaming chat calls and the non-streaming generate_completion path,
as well as the startup warm-up request.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Bryan Van Deusen
2026-03-30 10:21:59 -04:00
parent dd304bb556
commit 2a8c0cfa56
2 changed files with 4 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/fabledassistant/app.py
+1 -1
View File
@@ -169,7 +169,7 @@ def create_app() -> Quart:
async with httpx.AsyncClient(timeout=300.0) as client: async with httpx.AsyncClient(timeout=300.0) as client:
await client.post( await client.post(
f"{Config.OLLAMA_URL}/api/generate", f"{Config.OLLAMA_URL}/api/generate",
json={"model": model, "prompt": "", "keep_alive": "30m"}, json={"model": model, "prompt": "", "keep_alive": "2h"},
) )
logger.info("Warmed model '%s' into VRAM", model) logger.info("Warmed model '%s' into VRAM", model)
except Exception: except Exception:
src/fabledassistant/services/llm.py
+3 -1
View File
@@ -122,7 +122,7 @@ async def stream_chat(
merged_options = {"num_ctx": Config.OLLAMA_NUM_CTX} merged_options = {"num_ctx": Config.OLLAMA_NUM_CTX}
if options: if options:
merged_options.update(options) merged_options.update(options)
payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options, "think": think} payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options, "think": think, "keep_alive": "2h"}
# read=None: no per-chunk timeout — Ollama may pause for any duration while # read=None: no per-chunk timeout — Ollama may pause for any duration while
# processing a large input context before the first token arrives. # processing a large input context before the first token arrives.
async with httpx.AsyncClient(timeout=httpx.Timeout(connect=30.0, read=None, write=None, pool=30.0)) as client: async with httpx.AsyncClient(timeout=httpx.Timeout(connect=30.0, read=None, write=None, pool=30.0)) as client:
@@ -176,6 +176,7 @@ async def stream_chat_with_tools(
"stream": True, "stream": True,
"options": options, "options": options,
"think": think, "think": think,
"keep_alive": "2h",
} }
if tools: if tools:
payload["tools"] = tools payload["tools"] = tools
@@ -256,6 +257,7 @@ async def generate_completion(
"stream": False, "stream": False,
"think": False, "think": False,
"options": options, "options": options,
"keep_alive": "2h",
}, },
) )
resp.raise_for_status() resp.raise_for_status()