diff --git a/src/fabledassistant/services/generation_task.py b/src/fabledassistant/services/generation_task.py index ee2bb6a..72d7a3b 100644 --- a/src/fabledassistant/services/generation_task.py +++ b/src/fabledassistant/services/generation_task.py @@ -11,6 +11,8 @@ import re import time from collections.abc import AsyncGenerator +import httpx + from sqlalchemy import update from fabledassistant.config import Config @@ -291,13 +293,30 @@ async def run_generation( stream_queue: asyncio.Queue[ChatChunk | BaseException | None] = asyncio.Queue(maxsize=256) async def _fill_queue() -> None: - try: - async for c in stream_chat_with_tools(messages, model, tools=tools, think=think): - await stream_queue.put(c) - except BaseException as exc: - await stream_queue.put(exc) - finally: - await stream_queue.put(None) + # Retry on Ollama 500 (model cold-loading race) up to 2 times. + last_exc: BaseException | None = None + for attempt in range(3): + if attempt > 0: + wait = 3.0 * attempt + logger.warning( + "Ollama stream 500 (attempt %d/3), retrying in %.0fs", attempt, wait + ) + await asyncio.sleep(wait) + try: + async for c in stream_chat_with_tools(messages, model, tools=tools, think=think): + await stream_queue.put(c) + last_exc = None + break + except httpx.HTTPStatusError as exc: + last_exc = exc + if exc.response.status_code != 500: + break # non-500 errors are not retryable + except BaseException as exc: + last_exc = exc + break + if last_exc is not None: + await stream_queue.put(last_exc) + await stream_queue.put(None) stream_fill_task = asyncio.create_task(_fill_queue()) buf.append_event("status", {"status": "Generating response..."})