From 7b6248c8a7e24b7227f098c6b9bcccca6246abfa Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 25 Feb 2026 22:02:06 -0500 Subject: [PATCH] Add OLLAMA_NUM_CTX config to reduce VRAM usage Replaces the hardcoded num_ctx=32768 KV cache allocation with a configurable env var defaulting to 8192. This significantly reduces VRAM pressure when multiple services share the GPU. Co-Authored-By: Claude Sonnet 4.6 --- src/fabledassistant/config.py | 3 +++ src/fabledassistant/services/llm.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/fabledassistant/config.py b/src/fabledassistant/config.py index f4ae9b4..d592043 100644 --- a/src/fabledassistant/config.py +++ b/src/fabledassistant/config.py @@ -27,6 +27,9 @@ class Config: # Optional dedicated model for intent classification (should be small/fast). # Falls back to OLLAMA_MODEL if not set. Can also be overridden per-user via settings. OLLAMA_INTENT_MODEL: str = os.environ.get("OLLAMA_INTENT_MODEL", "") + # KV cache context window for generation. Lower = less VRAM, less throughput impact. + # 8192 is sufficient for most conversations; raise if you paste large documents. + OLLAMA_NUM_CTX: int = int(os.environ.get("OLLAMA_NUM_CTX", "8192")) SECRET_KEY: str = _read_secret("SECRET_KEY", "SECRET_KEY_FILE", "dev-secret-change-me") SECURE_COOKIES: bool = os.environ.get("SECURE_COOKIES", "").lower() in ("1", "true", "yes") LOG_LEVEL: str = os.environ.get("LOG_LEVEL", "INFO") diff --git a/src/fabledassistant/services/llm.py b/src/fabledassistant/services/llm.py index b248db7..3a204e8 100644 --- a/src/fabledassistant/services/llm.py +++ b/src/fabledassistant/services/llm.py @@ -80,7 +80,7 @@ async def stream_chat( options: dict | None = None, ) -> AsyncGenerator[str, None]: """Stream chat completion from Ollama, yielding content chunks.""" - merged_options = {"num_ctx": 32768} + merged_options = {"num_ctx": Config.OLLAMA_NUM_CTX} if options: merged_options.update(options) payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options} @@ -126,7 +126,7 @@ async def stream_chat_with_tools( Thinking tokens are consumed by Ollama and not forwarded to the caller; only the final response content is yielded. Expect higher TTFT when enabled. """ - options: dict = {"num_ctx": 32768} + options: dict = {"num_ctx": Config.OLLAMA_NUM_CTX} if tools: options["num_predict"] = 8192 payload: dict = {