From fc7b2e7305a57916dc1e211975b0a608d6476e12 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 25 Feb 2026 23:15:05 -0500 Subject: [PATCH] Retry Ollama 500 errors in optimistic stream with backoff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With optimistic streaming, intent (qwen2.5:1.5b) and the main stream (qwen3:latest) start concurrently. When both models are cold-loading, Ollama returns 500 for both simultaneously. The intent 500 was already handled silently in classify_intent; the stream 500 now retries up to 2 times (3s then 6s delay) before propagating as an error. 500s only occur on the first cold-load pair — subsequent requests hit warm models. Co-Authored-By: Claude Sonnet 4.6 --- .../services/generation_task.py | 33 +++++++++++++++---- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/src/fabledassistant/services/generation_task.py b/src/fabledassistant/services/generation_task.py index ee2bb6a..72d7a3b 100644 --- a/src/fabledassistant/services/generation_task.py +++ b/src/fabledassistant/services/generation_task.py @@ -11,6 +11,8 @@ import re import time from collections.abc import AsyncGenerator +import httpx + from sqlalchemy import update from fabledassistant.config import Config @@ -291,13 +293,30 @@ async def run_generation( stream_queue: asyncio.Queue[ChatChunk | BaseException | None] = asyncio.Queue(maxsize=256) async def _fill_queue() -> None: - try: - async for c in stream_chat_with_tools(messages, model, tools=tools, think=think): - await stream_queue.put(c) - except BaseException as exc: - await stream_queue.put(exc) - finally: - await stream_queue.put(None) + # Retry on Ollama 500 (model cold-loading race) up to 2 times. + last_exc: BaseException | None = None + for attempt in range(3): + if attempt > 0: + wait = 3.0 * attempt + logger.warning( + "Ollama stream 500 (attempt %d/3), retrying in %.0fs", attempt, wait + ) + await asyncio.sleep(wait) + try: + async for c in stream_chat_with_tools(messages, model, tools=tools, think=think): + await stream_queue.put(c) + last_exc = None + break + except httpx.HTTPStatusError as exc: + last_exc = exc + if exc.response.status_code != 500: + break # non-500 errors are not retryable + except BaseException as exc: + last_exc = exc + break + if last_exc is not None: + await stream_queue.put(last_exc) + await stream_queue.put(None) stream_fill_task = asyncio.create_task(_fill_queue()) buf.append_event("status", {"status": "Generating response..."})