Fix startup warm-up to use user model settings instead of env var
Startup no longer auto-warms Config.OLLAMA_MODEL. Instead it queries all distinct default_model values from user settings, cross-references with Ollama's installed models, and warms only the intersection. Models that users have selected but not yet installed are skipped with an info log — they are never auto-pulled. The embedding model pull behaviour is unchanged. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
+71
-16
@@ -117,33 +117,88 @@ def create_app() -> Quart:
|
|||||||
start_log_retention_loop()
|
start_log_retention_loop()
|
||||||
start_notification_loop()
|
start_notification_loop()
|
||||||
|
|
||||||
|
async def _warm_model(model: str) -> None:
|
||||||
|
"""Warm an already-installed model into VRAM (no pull)."""
|
||||||
|
try:
|
||||||
|
async with httpx.AsyncClient(timeout=300.0) as client:
|
||||||
|
await client.post(
|
||||||
|
f"{Config.OLLAMA_URL}/api/generate",
|
||||||
|
json={"model": model, "prompt": "", "keep_alive": "30m"},
|
||||||
|
)
|
||||||
|
logger.info("Warmed model '%s' into VRAM", model)
|
||||||
|
except Exception:
|
||||||
|
logger.warning("Failed to warm model '%s'", model, exc_info=True)
|
||||||
|
|
||||||
|
async def _warm_user_models() -> None:
|
||||||
|
"""
|
||||||
|
Warm whichever chat model(s) users have selected in Settings.
|
||||||
|
|
||||||
|
Only warms models that are already installed in Ollama — never auto-pulls.
|
||||||
|
Falls back silently if no user preferences exist or Ollama is unreachable.
|
||||||
|
"""
|
||||||
|
from sqlalchemy import select as sa_select, distinct
|
||||||
|
|
||||||
|
from fabledassistant.models import async_session
|
||||||
|
from fabledassistant.models.setting import Setting
|
||||||
|
|
||||||
|
# 1. Collect all distinct default_model values users have saved.
|
||||||
|
try:
|
||||||
|
async with async_session() as session:
|
||||||
|
rows = await session.execute(
|
||||||
|
sa_select(distinct(Setting.value)).where(
|
||||||
|
Setting.key == "default_model",
|
||||||
|
Setting.value.isnot(None),
|
||||||
|
Setting.value != "",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
user_models: set[str] = {r for (r,) in rows}
|
||||||
|
except Exception:
|
||||||
|
logger.debug("Could not read user model preferences from DB", exc_info=True)
|
||||||
|
return
|
||||||
|
|
||||||
|
if not user_models:
|
||||||
|
logger.debug("No user model preferences found; skipping warm-up")
|
||||||
|
return
|
||||||
|
|
||||||
|
# 2. Ask Ollama which models are currently installed.
|
||||||
|
try:
|
||||||
|
async with httpx.AsyncClient(timeout=10.0) as client:
|
||||||
|
resp = await client.get(f"{Config.OLLAMA_URL}/api/tags")
|
||||||
|
resp.raise_for_status()
|
||||||
|
installed: set[str] = {m["name"] for m in resp.json().get("models", [])}
|
||||||
|
except Exception:
|
||||||
|
logger.debug("Could not reach Ollama to check installed models", exc_info=True)
|
||||||
|
return
|
||||||
|
|
||||||
|
# 3. Warm only the intersection (installed AND user-preferred).
|
||||||
|
for model in user_models:
|
||||||
|
base = model.removesuffix(":latest")
|
||||||
|
if model in installed or f"{base}:latest" in installed or base in installed:
|
||||||
|
await _warm_model(model)
|
||||||
|
else:
|
||||||
|
logger.info(
|
||||||
|
"User-preferred model '%s' is not installed; skipping warm-up "
|
||||||
|
"(install it via Settings → Models to enable auto-warm)",
|
||||||
|
model,
|
||||||
|
)
|
||||||
|
|
||||||
async def _pull_model(model: str, warm: bool = False) -> None:
|
async def _pull_model(model: str, warm: bool = False) -> None:
|
||||||
try:
|
try:
|
||||||
await ensure_model(model)
|
await ensure_model(model)
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Failed to ensure model '%s' — chat may not work until model is available",
|
"Failed to ensure model '%s'",
|
||||||
model,
|
model,
|
||||||
exc_info=True,
|
exc_info=True,
|
||||||
)
|
)
|
||||||
return
|
return
|
||||||
if warm:
|
if warm:
|
||||||
try:
|
await _warm_model(model)
|
||||||
async with httpx.AsyncClient(timeout=300.0) as client:
|
|
||||||
await client.post(
|
|
||||||
f"{Config.OLLAMA_URL}/api/generate",
|
|
||||||
json={"model": model, "prompt": "", "keep_alive": "30m"},
|
|
||||||
)
|
|
||||||
logger.info("Warmed model '%s' into VRAM", model)
|
|
||||||
except Exception:
|
|
||||||
logger.warning("Failed to warm model '%s'", model, exc_info=True)
|
|
||||||
|
|
||||||
# Pull and warm the main model into VRAM at startup so the first request is fast.
|
# Warm user-preferred chat models that are already installed.
|
||||||
asyncio.create_task(_pull_model(Config.OLLAMA_MODEL, warm=True))
|
# Also ensure the embedding model is pulled (no warm needed).
|
||||||
models_to_warm = {Config.OLLAMA_MODEL}
|
asyncio.create_task(_warm_user_models())
|
||||||
# Also pull the embedding model (nomic-embed-text by default), but no need to warm it.
|
asyncio.create_task(_pull_model(Config.EMBEDDING_MODEL, warm=False))
|
||||||
if Config.EMBEDDING_MODEL not in models_to_warm:
|
|
||||||
asyncio.create_task(_pull_model(Config.EMBEDDING_MODEL, warm=False))
|
|
||||||
|
|
||||||
# After models are pulled, backfill embeddings for existing notes.
|
# After models are pulled, backfill embeddings for existing notes.
|
||||||
# Runs in the background so it never blocks the server from accepting requests.
|
# Runs in the background so it never blocks the server from accepting requests.
|
||||||
|
|||||||
Reference in New Issue
Block a user