From 2c2874a1cc49155418847b2868edb970d7314a86 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 2 Mar 2026 17:50:42 -0500 Subject: [PATCH] Fix streaming timeout and false-offline status indicator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - stream_chat and stream_chat_with_tools: remove read=300s per-chunk timeout, replace with read=None. In httpx streaming mode, the read timeout applies per-chunk — if Ollama pauses >300s while processing a large input context before the first token, it raises ReadTimeout, killing generation and leaving the assistant message as an empty stub. With read=None the stream is unbounded; connect=30s still guards the initial connection. - chat_status_route: increase Ollama status check timeout 5s → 10s. When Ollama is busy processing a large prompt it can be slow to respond to /api/tags, causing the status indicator to briefly flip to "offline" even though generation is running normally. Co-Authored-By: Claude Sonnet 4.6 --- src/fabledassistant/routes/chat.py | 2 +- src/fabledassistant/services/llm.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/fabledassistant/routes/chat.py b/src/fabledassistant/routes/chat.py index dfe8660..0e554b0 100644 --- a/src/fabledassistant/routes/chat.py +++ b/src/fabledassistant/routes/chat.py @@ -328,7 +328,7 @@ async def chat_status_route(): "default_model": default_model, } try: - async with httpx.AsyncClient(timeout=5.0) as client: + async with httpx.AsyncClient(timeout=10.0) as client: # Query installed models and loaded models in parallel tags_task = asyncio.create_task(client.get(f"{Config.OLLAMA_URL}/api/tags")) ps_task = asyncio.create_task(client.get(f"{Config.OLLAMA_URL}/api/ps")) diff --git a/src/fabledassistant/services/llm.py b/src/fabledassistant/services/llm.py index 5365700..8b41672 100644 --- a/src/fabledassistant/services/llm.py +++ b/src/fabledassistant/services/llm.py @@ -110,7 +110,9 @@ async def stream_chat( if options: merged_options.update(options) payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options} - async with httpx.AsyncClient(timeout=httpx.Timeout(1800.0, connect=30.0, read=300.0)) as client: + # read=None: no per-chunk timeout — Ollama may pause for any duration while + # processing a large input context before the first token arrives. + async with httpx.AsyncClient(timeout=httpx.Timeout(connect=30.0, read=None, write=None, pool=30.0)) as client: async with client.stream( "POST", f"{Config.OLLAMA_URL}/api/chat", @@ -164,7 +166,8 @@ async def stream_chat_with_tools( } if tools: payload["tools"] = tools - async with httpx.AsyncClient(timeout=httpx.Timeout(1800.0, connect=30.0, read=300.0)) as client: + # read=None: no per-chunk timeout for the same reason as stream_chat. + async with httpx.AsyncClient(timeout=httpx.Timeout(connect=30.0, read=None, write=None, pool=30.0)) as client: async with client.stream( "POST", f"{Config.OLLAMA_URL}/api/chat",