feat(ollama): configurable per-model keep_alive durations
Replace the hardcoded "2h" keep_alive everywhere with a helper that returns OLLAMA_KEEP_ALIVE_MAIN (default 30m) for the interactive model and OLLAMA_KEEP_ALIVE_BACKGROUND (default 10m) for the background model. Lets the main model release VRAM during long idle periods while keeping it warm enough for bursty chat use, and stops the sporadic background model from camping VRAM it rarely needs. Seven call sites updated to route through llm.keep_alive_for(model): the streaming helpers, generate_completion, the two startup warmers, the settings KV-cache primer, and the chat warmer endpoint. Override via env vars: OLLAMA_KEEP_ALIVE_MAIN, OLLAMA_KEEP_ALIVE_BACKGROUND. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -26,6 +26,18 @@ logger = logging.getLogger(__name__)
|
||||
_CTX_TIERS = (8192, 16384, 32768)
|
||||
|
||||
|
||||
def keep_alive_for(model: str) -> str:
|
||||
"""Return the Ollama keep_alive duration for *model*.
|
||||
|
||||
Background models get a shorter window because they're called
|
||||
sporadically; the main interactive model gets a longer one so it
|
||||
stays warm between user messages.
|
||||
"""
|
||||
if model == Config.OLLAMA_BACKGROUND_MODEL:
|
||||
return Config.OLLAMA_KEEP_ALIVE_BACKGROUND
|
||||
return Config.OLLAMA_KEEP_ALIVE_MAIN
|
||||
|
||||
|
||||
def pick_num_ctx(messages: list[dict]) -> int:
|
||||
"""Return the smallest context tier that fits *messages* with 25% headroom.
|
||||
|
||||
@@ -145,7 +157,7 @@ async def stream_chat(
|
||||
merged_options = {"num_ctx": num_ctx or Config.OLLAMA_NUM_CTX}
|
||||
if options:
|
||||
merged_options.update(options)
|
||||
payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options, "think": think, "keep_alive": "2h"}
|
||||
payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options, "think": think, "keep_alive": keep_alive_for(model)}
|
||||
# read=None: no per-chunk timeout — Ollama may pause for any duration while
|
||||
# processing a large input context before the first token arrives.
|
||||
async with httpx.AsyncClient(timeout=httpx.Timeout(connect=30.0, read=None, write=None, pool=30.0)) as client:
|
||||
@@ -204,7 +216,7 @@ async def stream_chat_with_tools(
|
||||
"stream": True,
|
||||
"options": options,
|
||||
"think": think,
|
||||
"keep_alive": "2h",
|
||||
"keep_alive": keep_alive_for(model),
|
||||
}
|
||||
if tools:
|
||||
payload["tools"] = tools
|
||||
@@ -289,7 +301,7 @@ async def generate_completion(
|
||||
"stream": False,
|
||||
"think": False,
|
||||
"options": options,
|
||||
"keep_alive": "2h",
|
||||
"keep_alive": keep_alive_for(model),
|
||||
},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
|
||||
Reference in New Issue
Block a user