Fix writing assist: disable thinking mode, drop stuck-buffer 409
- stream_chat: add think=False parameter passed through to Ollama payload. qwen3 models have thinking enabled by default; without this flag the model spends minutes generating internal thinking tokens that stream_chat silently discards, leaving the frontend spinner blank until the SSE connection times out and the widget disappears. - create_assist_buffer: orphan (overwrite) a still-running buffer instead of raising. The old asyncio task holds a direct reference and completes harmlessly against the stale buffer. New requests always win. - assist_route: remove the 409 guard that blocked new requests when a previous generation got stuck. create_assist_buffer now handles this transparently. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -288,11 +288,6 @@ async def assist_route():
|
|||||||
if not target_section or not instruction:
|
if not target_section or not instruction:
|
||||||
return jsonify({"error": "target_section and instruction are required"}), 400
|
return jsonify({"error": "target_section and instruction are required"}), 400
|
||||||
|
|
||||||
# Reject if an assist generation is already running for this user
|
|
||||||
existing = get_assist_buffer(uid)
|
|
||||||
if existing and existing.state == GenerationState.RUNNING:
|
|
||||||
return jsonify({"error": "Assist generation already running"}), 409
|
|
||||||
|
|
||||||
model = await get_setting(uid, "default_model", Config.OLLAMA_MODEL) or Config.OLLAMA_MODEL
|
model = await get_setting(uid, "default_model", Config.OLLAMA_MODEL) or Config.OLLAMA_MODEL
|
||||||
messages = build_assist_messages(body, target_section, instruction)
|
messages = build_assist_messages(body, target_section, instruction)
|
||||||
|
|
||||||
|
|||||||
@@ -100,7 +100,9 @@ def create_assist_buffer(user_id: int) -> GenerationBuffer:
|
|||||||
key = f"assist:{user_id}"
|
key = f"assist:{user_id}"
|
||||||
existing = _buffers.get(key)
|
existing = _buffers.get(key)
|
||||||
if existing and existing.state == GenerationState.RUNNING:
|
if existing and existing.state == GenerationState.RUNNING:
|
||||||
raise RuntimeError(f"Assist generation already running for user {user_id}")
|
# Orphan the old buffer — the background task holds a direct reference
|
||||||
|
# and will complete against it harmlessly. A new request always wins.
|
||||||
|
logger.warning("Assist generation still running for user %d; orphaning old buffer", user_id)
|
||||||
buf = GenerationBuffer(conversation_id=0, assistant_message_id=0)
|
buf = GenerationBuffer(conversation_id=0, assistant_message_id=0)
|
||||||
_buffers[key] = buf
|
_buffers[key] = buf
|
||||||
return buf
|
return buf
|
||||||
|
|||||||
@@ -108,12 +108,18 @@ async def stream_chat(
|
|||||||
messages: list[dict],
|
messages: list[dict],
|
||||||
model: str,
|
model: str,
|
||||||
options: dict | None = None,
|
options: dict | None = None,
|
||||||
|
think: bool = False,
|
||||||
) -> AsyncGenerator[str, None]:
|
) -> AsyncGenerator[str, None]:
|
||||||
"""Stream chat completion from Ollama, yielding content chunks."""
|
"""Stream chat completion from Ollama, yielding content chunks.
|
||||||
|
|
||||||
|
Set think=False (default) to disable chain-of-thought on qwen3+ models.
|
||||||
|
Thinking tokens are silently discarded anyway, but disabling avoids the
|
||||||
|
multi-minute delay before the first content token arrives.
|
||||||
|
"""
|
||||||
merged_options = {"num_ctx": Config.OLLAMA_NUM_CTX}
|
merged_options = {"num_ctx": Config.OLLAMA_NUM_CTX}
|
||||||
if options:
|
if options:
|
||||||
merged_options.update(options)
|
merged_options.update(options)
|
||||||
payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options}
|
payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options, "think": think}
|
||||||
# read=None: no per-chunk timeout — Ollama may pause for any duration while
|
# read=None: no per-chunk timeout — Ollama may pause for any duration while
|
||||||
# processing a large input context before the first token arrives.
|
# processing a large input context before the first token arrives.
|
||||||
async with httpx.AsyncClient(timeout=httpx.Timeout(connect=30.0, read=None, write=None, pool=30.0)) as client:
|
async with httpx.AsyncClient(timeout=httpx.Timeout(connect=30.0, read=None, write=None, pool=30.0)) as client:
|
||||||
|
|||||||
Reference in New Issue
Block a user