Fix streaming timeout and false-offline status indicator

- stream_chat and stream_chat_with_tools: remove read=300s per-chunk
  timeout, replace with read=None. In httpx streaming mode, the read
  timeout applies per-chunk — if Ollama pauses >300s while processing
  a large input context before the first token, it raises ReadTimeout,
  killing generation and leaving the assistant message as an empty stub.
  With read=None the stream is unbounded; connect=30s still guards the
  initial connection.

- chat_status_route: increase Ollama status check timeout 5s → 10s.
  When Ollama is busy processing a large prompt it can be slow to
  respond to /api/tags, causing the status indicator to briefly flip to
  "offline" even though generation is running normally.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-02 17:50:42 -05:00
parent 6f5170854b
commit 2c2874a1cc
2 changed files with 6 additions and 3 deletions
+1 -1
View File
@@ -328,7 +328,7 @@ async def chat_status_route():
"default_model": default_model,
}
try:
async with httpx.AsyncClient(timeout=5.0) as client:
async with httpx.AsyncClient(timeout=10.0) as client:
# Query installed models and loaded models in parallel
tags_task = asyncio.create_task(client.get(f"{Config.OLLAMA_URL}/api/tags"))
ps_task = asyncio.create_task(client.get(f"{Config.OLLAMA_URL}/api/ps"))
+5 -2
View File
@@ -110,7 +110,9 @@ async def stream_chat(
if options:
merged_options.update(options)
payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options}
async with httpx.AsyncClient(timeout=httpx.Timeout(1800.0, connect=30.0, read=300.0)) as client:
# read=None: no per-chunk timeout — Ollama may pause for any duration while
# processing a large input context before the first token arrives.
async with httpx.AsyncClient(timeout=httpx.Timeout(connect=30.0, read=None, write=None, pool=30.0)) as client:
async with client.stream(
"POST",
f"{Config.OLLAMA_URL}/api/chat",
@@ -164,7 +166,8 @@ async def stream_chat_with_tools(
}
if tools:
payload["tools"] = tools
async with httpx.AsyncClient(timeout=httpx.Timeout(1800.0, connect=30.0, read=300.0)) as client:
# read=None: no per-chunk timeout for the same reason as stream_chat.
async with httpx.AsyncClient(timeout=httpx.Timeout(connect=30.0, read=None, write=None, pool=30.0)) as client:
async with client.stream(
"POST",
f"{Config.OLLAMA_URL}/api/chat",