From 2422946b4fb9e50d2e334023361ef582bf56a267 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 2 Apr 2026 22:36:31 -0400 Subject: [PATCH] perf: remove model-load polling before generation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit wait_for_model_loaded() polled /api/ps for up to 180s waiting for the model to appear as loaded. But Ollama lazy-loads models on the first /api/chat request, so the poll will never succeed — it just blocks for the full 180s after every Ollama restart before proceeding. Removed the wait entirely. Ollama handles on-demand loading correctly. Co-Authored-By: Claude Sonnet 4.6 --- .../services/generation_task.py | 22 ++++++------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/src/fabledassistant/services/generation_task.py b/src/fabledassistant/services/generation_task.py index 2fe4699..527143a 100644 --- a/src/fabledassistant/services/generation_task.py +++ b/src/fabledassistant/services/generation_task.py @@ -22,7 +22,7 @@ from fabledassistant.services.generation_buffer import ( GenerationBuffer, GenerationState, ) -from fabledassistant.services.llm import ChatChunk, build_context, generate_completion, stream_chat, stream_chat_with_tools, summarize_history_for_context, wait_for_model_loaded +from fabledassistant.services.llm import ChatChunk, build_context, generate_completion, stream_chat, stream_chat_with_tools, summarize_history_for_context from fabledassistant.services.chat import update_conversation_title from fabledassistant.services.logging import log_generation from fabledassistant.services.tools import get_tools_for_user, execute_tool @@ -175,8 +175,10 @@ async def run_generation( buf.append_event("status", {"status": "Summarizing conversation history..."}) history_to_use, history_summary = await summarize_history_for_context(history, model) - # Phase 3: Build context and wait for model in parallel. - model_load_task = asyncio.create_task(wait_for_model_loaded(model, timeout=180.0)) + # Phase 3: Build context. + # Note: Ollama lazy-loads models on the first /api/chat request, so polling + # /api/ps for model readiness only causes delay. We proceed immediately and + # let Ollama handle loading on demand. # Fetch voice_speech_style from user settings when voice_mode is active. voice_speech_style = "conversational" @@ -184,7 +186,7 @@ async def run_generation( from fabledassistant.services.settings import get_setting voice_speech_style = await get_setting(user_id, "voice_speech_style", "conversational") - context_task = asyncio.create_task(build_context( + messages, context_meta = await build_context( user_id, history_to_use, context_note_id, user_content, history_summary=history_summary, include_note_ids=include_note_ids, @@ -195,21 +197,11 @@ async def run_generation( conv_id=conv_id, voice_mode=voice_mode, voice_speech_style=voice_speech_style, - )) - - messages, context_meta = await context_task + ) # Emit context event buf.append_event("context", {"context": context_meta}) - # Wait for main model to be loaded before starting any generation. - # If it's already loaded (common case), this returns immediately. - if not model_load_task.done(): - buf.append_event("status", {"status": "Loading model..."}) - loaded = await model_load_task - if not loaded: - logger.warning("Model %s did not load within 180s — proceeding anyway", model) - t_start = time.monotonic() timing: dict = { "think": think,