feat(llm): adaptive num_ctx tiers + fix KV cache priming num_ctx mismatch
Adds pick_num_ctx() which selects the smallest context window tier (8192, 16384, 32768) that fits the current messages with 25% headroom, capped at OLLAMA_NUM_CTX. Threads num_ctx through generation_task.py so every chat request uses the computed tier rather than a fixed 16384. Fixes a critical cache miss bug: KV cache priming in app.py and settings.py was sending requests without num_ctx, so Ollama sized the cache at its model default (different from the 16384 real requests used), forcing a full model reload on the first real user message. Both priming sites now call pick_num_ctx() and pass the matching value. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -15,7 +15,7 @@ logger = logging.getLogger(__name__)
|
||||
async def _prime_kv_cache_bg(user_id: int, model: str) -> None:
|
||||
"""Fire-and-forget: prime Ollama's KV cache with the user's system prompt."""
|
||||
import httpx
|
||||
from fabledassistant.services.llm import build_context
|
||||
from fabledassistant.services.llm import build_context, pick_num_ctx
|
||||
try:
|
||||
messages, _ = await build_context(
|
||||
user_id=user_id,
|
||||
@@ -23,6 +23,7 @@ async def _prime_kv_cache_bg(user_id: int, model: str) -> None:
|
||||
current_note_id=None,
|
||||
user_message=" ",
|
||||
)
|
||||
num_ctx = pick_num_ctx(messages)
|
||||
async with httpx.AsyncClient(timeout=120.0) as client:
|
||||
await client.post(
|
||||
f"{Config.OLLAMA_URL}/api/chat",
|
||||
@@ -30,11 +31,11 @@ async def _prime_kv_cache_bg(user_id: int, model: str) -> None:
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"stream": False,
|
||||
"options": {"num_predict": 1},
|
||||
"options": {"num_predict": 1, "num_ctx": num_ctx},
|
||||
"keep_alive": "2h",
|
||||
},
|
||||
)
|
||||
logger.info("Primed KV cache for user %d with model '%s'", user_id, model)
|
||||
logger.info("Primed KV cache for user %d with model '%s' (num_ctx=%d)", user_id, model, num_ctx)
|
||||
except Exception:
|
||||
logger.warning("Failed to prime KV cache for user %d", user_id, exc_info=True)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user