From c9065c4481dce88755307507efdc9ab14ecb1956 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 3 Apr 2026 10:51:17 -0400 Subject: [PATCH] perf(settings): prime KV cache when user changes chat model When a user saves a new default_model in Settings, fire a background cache-prime request so the first message with the new model is fast rather than paying the full cold-start prefill cost. Co-Authored-By: Claude Sonnet 4.6 --- src/fabledassistant/routes/settings.py | 36 ++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/fabledassistant/routes/settings.py b/src/fabledassistant/routes/settings.py index 1d3e429..a180fb2 100644 --- a/src/fabledassistant/routes/settings.py +++ b/src/fabledassistant/routes/settings.py @@ -1,3 +1,6 @@ +import asyncio +import logging + from quart import Blueprint, jsonify, request from fabledassistant.auth import login_required, get_current_user_id @@ -6,6 +9,35 @@ from fabledassistant.services.caldav import CALDAV_SETTING_KEYS, get_caldav_conf from fabledassistant.services.llm import get_installed_models, _is_private_url from fabledassistant.services.settings import delete_setting, get_all_settings, set_settings_batch +logger = logging.getLogger(__name__) + + +async def _prime_kv_cache_bg(user_id: int, model: str) -> None: + """Fire-and-forget: prime Ollama's KV cache with the user's system prompt.""" + import httpx + from fabledassistant.services.llm import build_context + try: + messages, _ = await build_context( + user_id=user_id, + history=[], + current_note_id=None, + user_message=" ", + ) + async with httpx.AsyncClient(timeout=120.0) as client: + await client.post( + f"{Config.OLLAMA_URL}/api/chat", + json={ + "model": model, + "messages": messages, + "stream": False, + "options": {"num_predict": 1}, + "keep_alive": "2h", + }, + ) + logger.info("Primed KV cache for user %d with model '%s'", user_id, model) + except Exception: + logger.warning("Failed to prime KV cache for user %d", user_id, exc_info=True) + settings_bp = Blueprint("settings", __name__, url_prefix="/api/settings") @@ -46,6 +78,10 @@ async def update_settings_route(): if to_save: await set_settings_batch(uid, to_save) + + if "default_model" in to_save and to_save["default_model"]: + asyncio.create_task(_prime_kv_cache_bg(uid, to_save["default_model"])) + settings = await get_all_settings(uid) return jsonify(settings)