From b11a92f32d7fef74de430271443eb2e20b5ca2f0 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 5 Mar 2026 13:28:59 -0500 Subject: [PATCH] Fix writing assist: disable thinking mode, drop stuck-buffer 409 - stream_chat: add think=False parameter passed through to Ollama payload. qwen3 models have thinking enabled by default; without this flag the model spends minutes generating internal thinking tokens that stream_chat silently discards, leaving the frontend spinner blank until the SSE connection times out and the widget disappears. - create_assist_buffer: orphan (overwrite) a still-running buffer instead of raising. The old asyncio task holds a direct reference and completes harmlessly against the stale buffer. New requests always win. - assist_route: remove the 409 guard that blocked new requests when a previous generation got stuck. create_assist_buffer now handles this transparently. Co-Authored-By: Claude Sonnet 4.6 --- src/fabledassistant/routes/notes.py | 5 ----- src/fabledassistant/services/generation_buffer.py | 4 +++- src/fabledassistant/services/llm.py | 10 ++++++++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/fabledassistant/routes/notes.py b/src/fabledassistant/routes/notes.py index 0bb911c..5c4d835 100644 --- a/src/fabledassistant/routes/notes.py +++ b/src/fabledassistant/routes/notes.py @@ -288,11 +288,6 @@ async def assist_route(): if not target_section or not instruction: return jsonify({"error": "target_section and instruction are required"}), 400 - # Reject if an assist generation is already running for this user - existing = get_assist_buffer(uid) - if existing and existing.state == GenerationState.RUNNING: - return jsonify({"error": "Assist generation already running"}), 409 - model = await get_setting(uid, "default_model", Config.OLLAMA_MODEL) or Config.OLLAMA_MODEL messages = build_assist_messages(body, target_section, instruction) diff --git a/src/fabledassistant/services/generation_buffer.py b/src/fabledassistant/services/generation_buffer.py index ce289ad..dc90e07 100644 --- a/src/fabledassistant/services/generation_buffer.py +++ b/src/fabledassistant/services/generation_buffer.py @@ -100,7 +100,9 @@ def create_assist_buffer(user_id: int) -> GenerationBuffer: key = f"assist:{user_id}" existing = _buffers.get(key) if existing and existing.state == GenerationState.RUNNING: - raise RuntimeError(f"Assist generation already running for user {user_id}") + # Orphan the old buffer — the background task holds a direct reference + # and will complete against it harmlessly. A new request always wins. + logger.warning("Assist generation still running for user %d; orphaning old buffer", user_id) buf = GenerationBuffer(conversation_id=0, assistant_message_id=0) _buffers[key] = buf return buf diff --git a/src/fabledassistant/services/llm.py b/src/fabledassistant/services/llm.py index 5de6fb2..ac217f3 100644 --- a/src/fabledassistant/services/llm.py +++ b/src/fabledassistant/services/llm.py @@ -108,12 +108,18 @@ async def stream_chat( messages: list[dict], model: str, options: dict | None = None, + think: bool = False, ) -> AsyncGenerator[str, None]: - """Stream chat completion from Ollama, yielding content chunks.""" + """Stream chat completion from Ollama, yielding content chunks. + + Set think=False (default) to disable chain-of-thought on qwen3+ models. + Thinking tokens are silently discarded anyway, but disabling avoids the + multi-minute delay before the first content token arrives. + """ merged_options = {"num_ctx": Config.OLLAMA_NUM_CTX} if options: merged_options.update(options) - payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options} + payload: dict = {"model": model, "messages": messages, "stream": True, "options": merged_options, "think": think} # read=None: no per-chunk timeout — Ollama may pause for any duration while # processing a large input context before the first token arrives. async with httpx.AsyncClient(timeout=httpx.Timeout(connect=30.0, read=None, write=None, pool=30.0)) as client: