perf(settings): prime KV cache when user changes chat model

When a user saves a new default_model in Settings, fire a background
cache-prime request so the first message with the new model is fast
rather than paying the full cold-start prefill cost.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-04-03 10:51:17 -04:00
parent 4792e6459b
commit c9065c4481
+36
View File
@@ -1,3 +1,6 @@
import asyncio
import logging
from quart import Blueprint, jsonify, request
from fabledassistant.auth import login_required, get_current_user_id
@@ -6,6 +9,35 @@ from fabledassistant.services.caldav import CALDAV_SETTING_KEYS, get_caldav_conf
from fabledassistant.services.llm import get_installed_models, _is_private_url
from fabledassistant.services.settings import delete_setting, get_all_settings, set_settings_batch
logger = logging.getLogger(__name__)
async def _prime_kv_cache_bg(user_id: int, model: str) -> None:
"""Fire-and-forget: prime Ollama's KV cache with the user's system prompt."""
import httpx
from fabledassistant.services.llm import build_context
try:
messages, _ = await build_context(
user_id=user_id,
history=[],
current_note_id=None,
user_message=" ",
)
async with httpx.AsyncClient(timeout=120.0) as client:
await client.post(
f"{Config.OLLAMA_URL}/api/chat",
json={
"model": model,
"messages": messages,
"stream": False,
"options": {"num_predict": 1},
"keep_alive": "2h",
},
)
logger.info("Primed KV cache for user %d with model '%s'", user_id, model)
except Exception:
logger.warning("Failed to prime KV cache for user %d", user_id, exc_info=True)
settings_bp = Blueprint("settings", __name__, url_prefix="/api/settings")
@@ -46,6 +78,10 @@ async def update_settings_route():
if to_save:
await set_settings_batch(uid, to_save)
if "default_model" in to_save and to_save["default_model"]:
asyncio.create_task(_prime_kv_cache_bg(uid, to_save["default_model"]))
settings = await get_all_settings(uid)
return jsonify(settings)