Warm both chat and intent models into VRAM on startup
ensure_model only downloads a model if missing; it does not load it into VRAM. The frontend warm call only covers the chat model (and only after a user opens the dashboard). This left qwen2.5:1.5b (intent) cold, causing simultaneous cold-load 500s when the first chat arrived. Now both Config.OLLAMA_MODEL and Config.OLLAMA_INTENT_MODEL are warmed at startup (after ensuring they're installed) via a fire-and-forget /api/generate call with keep_alive=30m. The embedding model is still pulled but not warmed (it's loaded on demand during backfill). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -3,6 +3,8 @@ import time
|
|||||||
import traceback as tb_module
|
import traceback as tb_module
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
from quart import Quart, g, jsonify, make_response, request, send_from_directory
|
from quart import Quart, g, jsonify, make_response, request, send_from_directory
|
||||||
|
|
||||||
from fabledassistant.config import Config
|
from fabledassistant.config import Config
|
||||||
@@ -104,7 +106,7 @@ def create_app() -> Quart:
|
|||||||
start_log_retention_loop()
|
start_log_retention_loop()
|
||||||
start_notification_loop()
|
start_notification_loop()
|
||||||
|
|
||||||
async def _pull_model(model: str) -> None:
|
async def _pull_model(model: str, warm: bool = False) -> None:
|
||||||
try:
|
try:
|
||||||
await ensure_model(model)
|
await ensure_model(model)
|
||||||
except Exception:
|
except Exception:
|
||||||
@@ -113,16 +115,30 @@ def create_app() -> Quart:
|
|||||||
model,
|
model,
|
||||||
exc_info=True,
|
exc_info=True,
|
||||||
)
|
)
|
||||||
|
return
|
||||||
|
if warm:
|
||||||
|
try:
|
||||||
|
async with httpx.AsyncClient(timeout=300.0) as client:
|
||||||
|
await client.post(
|
||||||
|
f"{Config.OLLAMA_URL}/api/generate",
|
||||||
|
json={"model": model, "prompt": "", "keep_alive": "30m"},
|
||||||
|
)
|
||||||
|
logger.info("Warmed model '%s' into VRAM", model)
|
||||||
|
except Exception:
|
||||||
|
logger.warning("Failed to warm model '%s'", model, exc_info=True)
|
||||||
|
|
||||||
# Pull main model and (if configured) a separate intent model concurrently.
|
# Pull main model and (if configured) a separate intent model concurrently,
|
||||||
# Fire-and-forget so pulls don't block startup.
|
# then warm them into VRAM so the first user request doesn't cold-load both
|
||||||
models_to_pull = {Config.OLLAMA_MODEL}
|
# simultaneously (which causes Ollama 500 races).
|
||||||
|
# Fire-and-forget so pulls/warming don't block startup.
|
||||||
|
models_to_warm = {Config.OLLAMA_MODEL}
|
||||||
if Config.OLLAMA_INTENT_MODEL and Config.OLLAMA_INTENT_MODEL != Config.OLLAMA_MODEL:
|
if Config.OLLAMA_INTENT_MODEL and Config.OLLAMA_INTENT_MODEL != Config.OLLAMA_MODEL:
|
||||||
models_to_pull.add(Config.OLLAMA_INTENT_MODEL)
|
models_to_warm.add(Config.OLLAMA_INTENT_MODEL)
|
||||||
# Also pull the embedding model (nomic-embed-text by default).
|
for _model in models_to_warm:
|
||||||
models_to_pull.add(Config.EMBEDDING_MODEL)
|
asyncio.create_task(_pull_model(_model, warm=True))
|
||||||
for _model in models_to_pull:
|
# Also pull the embedding model (nomic-embed-text by default), but no need to warm it.
|
||||||
asyncio.create_task(_pull_model(_model))
|
if Config.EMBEDDING_MODEL not in models_to_warm:
|
||||||
|
asyncio.create_task(_pull_model(Config.EMBEDDING_MODEL, warm=False))
|
||||||
|
|
||||||
# After models are pulled, backfill embeddings for existing notes.
|
# After models are pulled, backfill embeddings for existing notes.
|
||||||
# Runs in the background so it never blocks the server from accepting requests.
|
# Runs in the background so it never blocks the server from accepting requests.
|
||||||
|
|||||||
Reference in New Issue
Block a user