From b416fec2921f5485bb91f1a62633a75e3056a228 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 2 Apr 2026 19:49:55 -0400 Subject: [PATCH] perf: reduce OLLAMA_NUM_CTX default from 65536 to 16384 65536 was causing Ollama to allocate a ~50GB KV cache, spilling 77% of the model to CPU RAM and making prefill extremely slow (35-125s TTFT). 16384 covers 30+ message conversations comfortably while keeping the KV cache small enough to stay on GPU. Co-Authored-By: Claude Sonnet 4.6 --- src/fabledassistant/config.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/fabledassistant/config.py b/src/fabledassistant/config.py index e15a364..8897fda 100644 --- a/src/fabledassistant/config.py +++ b/src/fabledassistant/config.py @@ -24,9 +24,10 @@ class Config: ) OLLAMA_URL: str = os.environ.get("OLLAMA_URL", "http://localhost:11434") OLLAMA_MODEL: str = os.environ.get("OLLAMA_MODEL", "qwen3:latest") - # KV cache context window for generation. Higher = more RAM usage but longer inputs/outputs. - # 131072 is the practical maximum for most models. Lower this on RAM-constrained hosts. - OLLAMA_NUM_CTX: int = int(os.environ.get("OLLAMA_NUM_CTX", "65536")) + # KV cache context window for generation. Keep this as small as practical — + # a larger context forces more KV cache into CPU RAM, drastically slowing prefill. + # 16384 covers ~30+ message conversations with our system prompt comfortably. + OLLAMA_NUM_CTX: int = int(os.environ.get("OLLAMA_NUM_CTX", "16384")) SECRET_KEY: str = _read_secret("SECRET_KEY", "SECRET_KEY_FILE", "dev-secret-change-me") SECURE_COOKIES: bool = os.environ.get("SECURE_COOKIES", "").lower() in ("1", "true", "yes") LOG_LEVEL: str = os.environ.get("LOG_LEVEL", "INFO")