perf(settings): prime KV cache when user changes chat model
When a user saves a new default_model in Settings, fire a background cache-prime request so the first message with the new model is fast rather than paying the full cold-start prefill cost. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -1,3 +1,6 @@
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
|
||||
from fabledassistant.auth import login_required, get_current_user_id
|
||||
@@ -6,6 +9,35 @@ from fabledassistant.services.caldav import CALDAV_SETTING_KEYS, get_caldav_conf
|
||||
from fabledassistant.services.llm import get_installed_models, _is_private_url
|
||||
from fabledassistant.services.settings import delete_setting, get_all_settings, set_settings_batch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def _prime_kv_cache_bg(user_id: int, model: str) -> None:
|
||||
"""Fire-and-forget: prime Ollama's KV cache with the user's system prompt."""
|
||||
import httpx
|
||||
from fabledassistant.services.llm import build_context
|
||||
try:
|
||||
messages, _ = await build_context(
|
||||
user_id=user_id,
|
||||
history=[],
|
||||
current_note_id=None,
|
||||
user_message=" ",
|
||||
)
|
||||
async with httpx.AsyncClient(timeout=120.0) as client:
|
||||
await client.post(
|
||||
f"{Config.OLLAMA_URL}/api/chat",
|
||||
json={
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"stream": False,
|
||||
"options": {"num_predict": 1},
|
||||
"keep_alive": "2h",
|
||||
},
|
||||
)
|
||||
logger.info("Primed KV cache for user %d with model '%s'", user_id, model)
|
||||
except Exception:
|
||||
logger.warning("Failed to prime KV cache for user %d", user_id, exc_info=True)
|
||||
|
||||
settings_bp = Blueprint("settings", __name__, url_prefix="/api/settings")
|
||||
|
||||
|
||||
@@ -46,6 +78,10 @@ async def update_settings_route():
|
||||
|
||||
if to_save:
|
||||
await set_settings_batch(uid, to_save)
|
||||
|
||||
if "default_model" in to_save and to_save["default_model"]:
|
||||
asyncio.create_task(_prime_kv_cache_bg(uid, to_save["default_model"]))
|
||||
|
||||
settings = await get_all_settings(uid)
|
||||
return jsonify(settings)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user