perf: remove model-load polling before generation
wait_for_model_loaded() polled /api/ps for up to 180s waiting for the model to appear as loaded. But Ollama lazy-loads models on the first /api/chat request, so the poll will never succeed — it just blocks for the full 180s after every Ollama restart before proceeding. Removed the wait entirely. Ollama handles on-demand loading correctly. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -22,7 +22,7 @@ from fabledassistant.services.generation_buffer import (
|
||||
GenerationBuffer,
|
||||
GenerationState,
|
||||
)
|
||||
from fabledassistant.services.llm import ChatChunk, build_context, generate_completion, stream_chat, stream_chat_with_tools, summarize_history_for_context, wait_for_model_loaded
|
||||
from fabledassistant.services.llm import ChatChunk, build_context, generate_completion, stream_chat, stream_chat_with_tools, summarize_history_for_context
|
||||
from fabledassistant.services.chat import update_conversation_title
|
||||
from fabledassistant.services.logging import log_generation
|
||||
from fabledassistant.services.tools import get_tools_for_user, execute_tool
|
||||
@@ -175,8 +175,10 @@ async def run_generation(
|
||||
buf.append_event("status", {"status": "Summarizing conversation history..."})
|
||||
history_to_use, history_summary = await summarize_history_for_context(history, model)
|
||||
|
||||
# Phase 3: Build context and wait for model in parallel.
|
||||
model_load_task = asyncio.create_task(wait_for_model_loaded(model, timeout=180.0))
|
||||
# Phase 3: Build context.
|
||||
# Note: Ollama lazy-loads models on the first /api/chat request, so polling
|
||||
# /api/ps for model readiness only causes delay. We proceed immediately and
|
||||
# let Ollama handle loading on demand.
|
||||
|
||||
# Fetch voice_speech_style from user settings when voice_mode is active.
|
||||
voice_speech_style = "conversational"
|
||||
@@ -184,7 +186,7 @@ async def run_generation(
|
||||
from fabledassistant.services.settings import get_setting
|
||||
voice_speech_style = await get_setting(user_id, "voice_speech_style", "conversational")
|
||||
|
||||
context_task = asyncio.create_task(build_context(
|
||||
messages, context_meta = await build_context(
|
||||
user_id, history_to_use, context_note_id, user_content,
|
||||
history_summary=history_summary,
|
||||
include_note_ids=include_note_ids,
|
||||
@@ -195,21 +197,11 @@ async def run_generation(
|
||||
conv_id=conv_id,
|
||||
voice_mode=voice_mode,
|
||||
voice_speech_style=voice_speech_style,
|
||||
))
|
||||
|
||||
messages, context_meta = await context_task
|
||||
)
|
||||
|
||||
# Emit context event
|
||||
buf.append_event("context", {"context": context_meta})
|
||||
|
||||
# Wait for main model to be loaded before starting any generation.
|
||||
# If it's already loaded (common case), this returns immediately.
|
||||
if not model_load_task.done():
|
||||
buf.append_event("status", {"status": "Loading model..."})
|
||||
loaded = await model_load_task
|
||||
if not loaded:
|
||||
logger.warning("Model %s did not load within 180s — proceeding anyway", model)
|
||||
|
||||
t_start = time.monotonic()
|
||||
timing: dict = {
|
||||
"think": think,
|
||||
|
||||
Reference in New Issue
Block a user