diff --git a/src/fabledassistant/app.py b/src/fabledassistant/app.py index 6073f1f..5824dba 100644 --- a/src/fabledassistant/app.py +++ b/src/fabledassistant/app.py @@ -3,6 +3,8 @@ import time import traceback as tb_module from pathlib import Path +import httpx + from quart import Quart, g, jsonify, make_response, request, send_from_directory from fabledassistant.config import Config @@ -104,7 +106,7 @@ def create_app() -> Quart: start_log_retention_loop() start_notification_loop() - async def _pull_model(model: str) -> None: + async def _pull_model(model: str, warm: bool = False) -> None: try: await ensure_model(model) except Exception: @@ -113,16 +115,30 @@ def create_app() -> Quart: model, exc_info=True, ) + return + if warm: + try: + async with httpx.AsyncClient(timeout=300.0) as client: + await client.post( + f"{Config.OLLAMA_URL}/api/generate", + json={"model": model, "prompt": "", "keep_alive": "30m"}, + ) + logger.info("Warmed model '%s' into VRAM", model) + except Exception: + logger.warning("Failed to warm model '%s'", model, exc_info=True) - # Pull main model and (if configured) a separate intent model concurrently. - # Fire-and-forget so pulls don't block startup. - models_to_pull = {Config.OLLAMA_MODEL} + # Pull main model and (if configured) a separate intent model concurrently, + # then warm them into VRAM so the first user request doesn't cold-load both + # simultaneously (which causes Ollama 500 races). + # Fire-and-forget so pulls/warming don't block startup. + models_to_warm = {Config.OLLAMA_MODEL} if Config.OLLAMA_INTENT_MODEL and Config.OLLAMA_INTENT_MODEL != Config.OLLAMA_MODEL: - models_to_pull.add(Config.OLLAMA_INTENT_MODEL) - # Also pull the embedding model (nomic-embed-text by default). - models_to_pull.add(Config.EMBEDDING_MODEL) - for _model in models_to_pull: - asyncio.create_task(_pull_model(_model)) + models_to_warm.add(Config.OLLAMA_INTENT_MODEL) + for _model in models_to_warm: + asyncio.create_task(_pull_model(_model, warm=True)) + # Also pull the embedding model (nomic-embed-text by default), but no need to warm it. + if Config.EMBEDDING_MODEL not in models_to_warm: + asyncio.create_task(_pull_model(Config.EMBEDDING_MODEL, warm=False)) # After models are pulled, backfill embeddings for existing notes. # Runs in the background so it never blocks the server from accepting requests.