Add OLLAMA_NUM_CTX config to reduce VRAM usage
Replaces the hardcoded num_ctx=32768 KV cache allocation with a configurable env var defaulting to 8192. This significantly reduces VRAM pressure when multiple services share the GPU. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -80,7 +80,7 @@ async def stream_chat(
|
||||
options: dict | None = None,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""Stream chat completion from Ollama, yielding content chunks."""
|
||||
merged_options = {"num_ctx": 32768}
|
||||
merged_options = {"num_ctx": Config.OLLAMA_NUM_CTX}
|
||||
if options:
|
||||
merged_options.update(options)
|
||||
payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options}
|
||||
@@ -126,7 +126,7 @@ async def stream_chat_with_tools(
|
||||
Thinking tokens are consumed by Ollama and not forwarded to the caller;
|
||||
only the final response content is yielded. Expect higher TTFT when enabled.
|
||||
"""
|
||||
options: dict = {"num_ctx": 32768}
|
||||
options: dict = {"num_ctx": Config.OLLAMA_NUM_CTX}
|
||||
if tools:
|
||||
options["num_predict"] = 8192
|
||||
payload: dict = {
|
||||
|
||||
Reference in New Issue
Block a user