diff --git a/src/fabledassistant/config.py b/src/fabledassistant/config.py index f4ae9b4..d592043 100644 --- a/src/fabledassistant/config.py +++ b/src/fabledassistant/config.py @@ -27,6 +27,9 @@ class Config: # Optional dedicated model for intent classification (should be small/fast). # Falls back to OLLAMA_MODEL if not set. Can also be overridden per-user via settings. OLLAMA_INTENT_MODEL: str = os.environ.get("OLLAMA_INTENT_MODEL", "") + # KV cache context window for generation. Lower = less VRAM, less throughput impact. + # 8192 is sufficient for most conversations; raise if you paste large documents. + OLLAMA_NUM_CTX: int = int(os.environ.get("OLLAMA_NUM_CTX", "8192")) SECRET_KEY: str = _read_secret("SECRET_KEY", "SECRET_KEY_FILE", "dev-secret-change-me") SECURE_COOKIES: bool = os.environ.get("SECURE_COOKIES", "").lower() in ("1", "true", "yes") LOG_LEVEL: str = os.environ.get("LOG_LEVEL", "INFO") diff --git a/src/fabledassistant/services/llm.py b/src/fabledassistant/services/llm.py index b248db7..3a204e8 100644 --- a/src/fabledassistant/services/llm.py +++ b/src/fabledassistant/services/llm.py @@ -80,7 +80,7 @@ async def stream_chat( options: dict | None = None, ) -> AsyncGenerator[str, None]: """Stream chat completion from Ollama, yielding content chunks.""" - merged_options = {"num_ctx": 32768} + merged_options = {"num_ctx": Config.OLLAMA_NUM_CTX} if options: merged_options.update(options) payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options} @@ -126,7 +126,7 @@ async def stream_chat_with_tools( Thinking tokens are consumed by Ollama and not forwarded to the caller; only the final response content is yielded. Expect higher TTFT when enabled. """ - options: dict = {"num_ctx": 32768} + options: dict = {"num_ctx": Config.OLLAMA_NUM_CTX} if tools: options["num_predict"] = 8192 payload: dict = {