feat(llm): adaptive num_ctx tiers + fix KV cache priming num_ctx mismatch

Adds pick_num_ctx() which selects the smallest context window tier
(8192, 16384, 32768) that fits the current messages with 25% headroom,
capped at OLLAMA_NUM_CTX. Threads num_ctx through generation_task.py so
every chat request uses the computed tier rather than a fixed 16384.

Fixes a critical cache miss bug: KV cache priming in app.py and
settings.py was sending requests without num_ctx, so Ollama sized the
cache at its model default (different from the 16384 real requests used),
forcing a full model reload on the first real user message. Both priming
sites now call pick_num_ctx() and pass the matching value.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-04-03 11:47:39 -04:00
parent a6888953dc
commit ef55bcb560
4 changed files with 48 additions and 13 deletions
+4 -3
View File
@@ -15,7 +15,7 @@ logger = logging.getLogger(__name__)
async def _prime_kv_cache_bg(user_id: int, model: str) -> None:
"""Fire-and-forget: prime Ollama's KV cache with the user's system prompt."""
import httpx
from fabledassistant.services.llm import build_context
from fabledassistant.services.llm import build_context, pick_num_ctx
try:
messages, _ = await build_context(
user_id=user_id,
@@ -23,6 +23,7 @@ async def _prime_kv_cache_bg(user_id: int, model: str) -> None:
current_note_id=None,
user_message=" ",
)
num_ctx = pick_num_ctx(messages)
async with httpx.AsyncClient(timeout=120.0) as client:
await client.post(
f"{Config.OLLAMA_URL}/api/chat",
@@ -30,11 +31,11 @@ async def _prime_kv_cache_bg(user_id: int, model: str) -> None:
"model": model,
"messages": messages,
"stream": False,
"options": {"num_predict": 1},
"options": {"num_predict": 1, "num_ctx": num_ctx},
"keep_alive": "2h",
},
)
logger.info("Primed KV cache for user %d with model '%s'", user_id, model)
logger.info("Primed KV cache for user %d with model '%s' (num_ctx=%d)", user_id, model, num_ctx)
except Exception:
logger.warning("Failed to prime KV cache for user %d", user_id, exc_info=True)