Add OLLAMA_NUM_CTX config to reduce VRAM usage

Replaces the hardcoded num_ctx=32768 KV cache allocation with a
configurable env var defaulting to 8192. This significantly reduces
VRAM pressure when multiple services share the GPU.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-02-25 22:02:06 -05:00
parent d0525ab3bf
commit 7b6248c8a7
2 changed files with 5 additions and 2 deletions
+2 -2
View File
@@ -80,7 +80,7 @@ async def stream_chat(
options: dict | None = None,
) -> AsyncGenerator[str, None]:
"""Stream chat completion from Ollama, yielding content chunks."""
merged_options = {"num_ctx": 32768}
merged_options = {"num_ctx": Config.OLLAMA_NUM_CTX}
if options:
merged_options.update(options)
payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options}
@@ -126,7 +126,7 @@ async def stream_chat_with_tools(
Thinking tokens are consumed by Ollama and not forwarded to the caller;
only the final response content is yielded. Expect higher TTFT when enabled.
"""
options: dict = {"num_ctx": 32768}
options: dict = {"num_ctx": Config.OLLAMA_NUM_CTX}
if tools:
options["num_predict"] = 8192
payload: dict = {