perf: remove model-load polling before generation

wait_for_model_loaded() polled /api/ps for up to 180s waiting for the
model to appear as loaded. But Ollama lazy-loads models on the first
/api/chat request, so the poll will never succeed — it just blocks for
the full 180s after every Ollama restart before proceeding.

Removed the wait entirely. Ollama handles on-demand loading correctly.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-04-02 22:36:31 -04:00
parent b416fec292
commit 2422946b4f
@@ -22,7 +22,7 @@ from fabledassistant.services.generation_buffer import (
GenerationBuffer,
GenerationState,
)
from fabledassistant.services.llm import ChatChunk, build_context, generate_completion, stream_chat, stream_chat_with_tools, summarize_history_for_context, wait_for_model_loaded
from fabledassistant.services.llm import ChatChunk, build_context, generate_completion, stream_chat, stream_chat_with_tools, summarize_history_for_context
from fabledassistant.services.chat import update_conversation_title
from fabledassistant.services.logging import log_generation
from fabledassistant.services.tools import get_tools_for_user, execute_tool
@@ -175,8 +175,10 @@ async def run_generation(
buf.append_event("status", {"status": "Summarizing conversation history..."})
history_to_use, history_summary = await summarize_history_for_context(history, model)
# Phase 3: Build context and wait for model in parallel.
model_load_task = asyncio.create_task(wait_for_model_loaded(model, timeout=180.0))
# Phase 3: Build context.
# Note: Ollama lazy-loads models on the first /api/chat request, so polling
# /api/ps for model readiness only causes delay. We proceed immediately and
# let Ollama handle loading on demand.
# Fetch voice_speech_style from user settings when voice_mode is active.
voice_speech_style = "conversational"
@@ -184,7 +186,7 @@ async def run_generation(
from fabledassistant.services.settings import get_setting
voice_speech_style = await get_setting(user_id, "voice_speech_style", "conversational")
context_task = asyncio.create_task(build_context(
messages, context_meta = await build_context(
user_id, history_to_use, context_note_id, user_content,
history_summary=history_summary,
include_note_ids=include_note_ids,
@@ -195,21 +197,11 @@ async def run_generation(
conv_id=conv_id,
voice_mode=voice_mode,
voice_speech_style=voice_speech_style,
))
messages, context_meta = await context_task
)
# Emit context event
buf.append_event("context", {"context": context_meta})
# Wait for main model to be loaded before starting any generation.
# If it's already loaded (common case), this returns immediately.
if not model_load_task.done():
buf.append_event("status", {"status": "Loading model..."})
loaded = await model_load_task
if not loaded:
logger.warning("Model %s did not load within 180s — proceeding anyway", model)
t_start = time.monotonic()
timing: dict = {
"think": think,