Fix streaming timeout and false-offline status indicator
- stream_chat and stream_chat_with_tools: remove read=300s per-chunk timeout, replace with read=None. In httpx streaming mode, the read timeout applies per-chunk — if Ollama pauses >300s while processing a large input context before the first token, it raises ReadTimeout, killing generation and leaving the assistant message as an empty stub. With read=None the stream is unbounded; connect=30s still guards the initial connection. - chat_status_route: increase Ollama status check timeout 5s → 10s. When Ollama is busy processing a large prompt it can be slow to respond to /api/tags, causing the status indicator to briefly flip to "offline" even though generation is running normally. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -328,7 +328,7 @@ async def chat_status_route():
|
|||||||
"default_model": default_model,
|
"default_model": default_model,
|
||||||
}
|
}
|
||||||
try:
|
try:
|
||||||
async with httpx.AsyncClient(timeout=5.0) as client:
|
async with httpx.AsyncClient(timeout=10.0) as client:
|
||||||
# Query installed models and loaded models in parallel
|
# Query installed models and loaded models in parallel
|
||||||
tags_task = asyncio.create_task(client.get(f"{Config.OLLAMA_URL}/api/tags"))
|
tags_task = asyncio.create_task(client.get(f"{Config.OLLAMA_URL}/api/tags"))
|
||||||
ps_task = asyncio.create_task(client.get(f"{Config.OLLAMA_URL}/api/ps"))
|
ps_task = asyncio.create_task(client.get(f"{Config.OLLAMA_URL}/api/ps"))
|
||||||
|
|||||||
@@ -110,7 +110,9 @@ async def stream_chat(
|
|||||||
if options:
|
if options:
|
||||||
merged_options.update(options)
|
merged_options.update(options)
|
||||||
payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options}
|
payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options}
|
||||||
async with httpx.AsyncClient(timeout=httpx.Timeout(1800.0, connect=30.0, read=300.0)) as client:
|
# read=None: no per-chunk timeout — Ollama may pause for any duration while
|
||||||
|
# processing a large input context before the first token arrives.
|
||||||
|
async with httpx.AsyncClient(timeout=httpx.Timeout(connect=30.0, read=None, write=None, pool=30.0)) as client:
|
||||||
async with client.stream(
|
async with client.stream(
|
||||||
"POST",
|
"POST",
|
||||||
f"{Config.OLLAMA_URL}/api/chat",
|
f"{Config.OLLAMA_URL}/api/chat",
|
||||||
@@ -164,7 +166,8 @@ async def stream_chat_with_tools(
|
|||||||
}
|
}
|
||||||
if tools:
|
if tools:
|
||||||
payload["tools"] = tools
|
payload["tools"] = tools
|
||||||
async with httpx.AsyncClient(timeout=httpx.Timeout(1800.0, connect=30.0, read=300.0)) as client:
|
# read=None: no per-chunk timeout for the same reason as stream_chat.
|
||||||
|
async with httpx.AsyncClient(timeout=httpx.Timeout(connect=30.0, read=None, write=None, pool=30.0)) as client:
|
||||||
async with client.stream(
|
async with client.stream(
|
||||||
"POST",
|
"POST",
|
||||||
f"{Config.OLLAMA_URL}/api/chat",
|
f"{Config.OLLAMA_URL}/api/chat",
|
||||||
|
|||||||
Reference in New Issue
Block a user