feat(ollama): configurable per-model keep_alive durations

Replace the hardcoded "2h" keep_alive everywhere with a helper that
returns OLLAMA_KEEP_ALIVE_MAIN (default 30m) for the interactive model
and OLLAMA_KEEP_ALIVE_BACKGROUND (default 10m) for the background
model. Lets the main model release VRAM during long idle periods
while keeping it warm enough for bursty chat use, and stops the
sporadic background model from camping VRAM it rarely needs.

Seven call sites updated to route through llm.keep_alive_for(model):
the streaming helpers, generate_completion, the two startup warmers,
the settings KV-cache primer, and the chat warmer endpoint.

Override via env vars: OLLAMA_KEEP_ALIVE_MAIN, OLLAMA_KEEP_ALIVE_BACKGROUND.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Bryan Van Deusen
2026-04-10 14:13:32 -04:00
parent 102c0b74a0
commit 3f3156db07
5 changed files with 29 additions and 8 deletions
+15 -3
View File
@@ -26,6 +26,18 @@ logger = logging.getLogger(__name__)
_CTX_TIERS = (8192, 16384, 32768)
def keep_alive_for(model: str) -> str:
"""Return the Ollama keep_alive duration for *model*.
Background models get a shorter window because they're called
sporadically; the main interactive model gets a longer one so it
stays warm between user messages.
"""
if model == Config.OLLAMA_BACKGROUND_MODEL:
return Config.OLLAMA_KEEP_ALIVE_BACKGROUND
return Config.OLLAMA_KEEP_ALIVE_MAIN
def pick_num_ctx(messages: list[dict]) -> int:
"""Return the smallest context tier that fits *messages* with 25% headroom.
@@ -145,7 +157,7 @@ async def stream_chat(
merged_options = {"num_ctx": num_ctx or Config.OLLAMA_NUM_CTX}
if options:
merged_options.update(options)
payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options, "think": think, "keep_alive": "2h"}
payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options, "think": think, "keep_alive": keep_alive_for(model)}
# read=None: no per-chunk timeout — Ollama may pause for any duration while
# processing a large input context before the first token arrives.
async with httpx.AsyncClient(timeout=httpx.Timeout(connect=30.0, read=None, write=None, pool=30.0)) as client:
@@ -204,7 +216,7 @@ async def stream_chat_with_tools(
"stream": True,
"options": options,
"think": think,
"keep_alive": "2h",
"keep_alive": keep_alive_for(model),
}
if tools:
payload["tools"] = tools
@@ -289,7 +301,7 @@ async def generate_completion(
"stream": False,
"think": False,
"options": options,
"keep_alive": "2h",
"keep_alive": keep_alive_for(model),
},
)
resp.raise_for_status()