From 92bf2768b61fece5a89fbd34d4e816858dee1ad1 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 18 Feb 2026 18:50:37 -0500 Subject: [PATCH] Reduce perceived latency: move context build into task, title fire-and-forget, think:False on aux calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - build_context() moved from route handler into run_generation() background task. The 202 response now returns immediately; client connects to SSE before note search / URL fetch begins, so 'Building context...' status is visible. - _generate_title() runs in a fire-and-forget asyncio.create_task() after the 'done' SSE event fires. Users see their response complete 2–5s sooner on new conversations; title appears later in the sidebar without blocking the stream. - generate_completion() now sets think:False and accepts a max_tokens limit. Intent classifier passes max_tokens=200 (JSON only), title generator passes max_tokens=30 (short title), eliminating qwen3 thinking-mode overhead on these auxiliary calls. Co-Authored-By: Claude Sonnet 4.6 --- src/fabledassistant/routes/chat.py | 12 ++-- .../services/generation_task.py | 63 +++++++++++-------- src/fabledassistant/services/intent.py | 2 +- src/fabledassistant/services/llm.py | 10 ++- 4 files changed, 49 insertions(+), 38 deletions(-) diff --git a/src/fabledassistant/routes/chat.py b/src/fabledassistant/routes/chat.py index 5bf781c..45be87d 100644 --- a/src/fabledassistant/routes/chat.py +++ b/src/fabledassistant/routes/chat.py @@ -23,7 +23,6 @@ from fabledassistant.services.generation_buffer import ( get_buffer, ) from fabledassistant.services.generation_task import run_generation -from fabledassistant.services.llm import build_context from fabledassistant.services.settings import get_setting logger = logging.getLogger(__name__) @@ -127,17 +126,14 @@ async def send_message_route(conv_id: int): if msg.role != "system": history.append({"role": msg.role, "content": msg.content}) - # Build context with note search, URL fetching, etc. - messages, context_meta = await build_context( - uid, history, context_note_id, content, exclude_note_ids=exclude_note_ids - ) - model = conv.model or await get_setting(uid, "default_model", Config.OLLAMA_MODEL) - # Launch background generation task + # Launch background generation task (context building happens inside the task) asyncio.create_task(run_generation( - buf, messages, model, context_meta, + buf, history, model, uid, conv_id, conv.title, content, + context_note_id=context_note_id, + exclude_note_ids=exclude_note_ids, )) return jsonify({ diff --git a/src/fabledassistant/services/generation_task.py b/src/fabledassistant/services/generation_task.py index 780300b..c3d80fb 100644 --- a/src/fabledassistant/services/generation_task.py +++ b/src/fabledassistant/services/generation_task.py @@ -16,7 +16,7 @@ from fabledassistant.config import Config from fabledassistant.models import async_session from fabledassistant.models.conversation import Message from fabledassistant.services.generation_buffer import GenerationBuffer, GenerationState -from fabledassistant.services.llm import ChatChunk, generate_completion, stream_chat, stream_chat_with_tools +from fabledassistant.services.llm import ChatChunk, build_context, generate_completion, stream_chat, stream_chat_with_tools from fabledassistant.services.chat import update_conversation_title from fabledassistant.services.intent import classify_intent from fabledassistant.services.logging import log_generation @@ -73,7 +73,7 @@ async def _generate_title(messages: list[dict], model: str) -> str: }, {"role": "user", "content": "\n\n".join(conv_lines)}, ] - title = await generate_completion(prompt_messages, model) + title = await generate_completion(prompt_messages, model, max_tokens=30) title = title.strip().strip('"\'').strip() return title[:100] if title else "" @@ -98,18 +98,25 @@ async def _update_message( async def run_generation( buf: GenerationBuffer, - messages: list[dict], + history: list[dict], model: str, - context_meta: dict, user_id: int, conv_id: int, conv_title: str, user_content: str, + context_note_id: int | None = None, + exclude_note_ids: list[int] | None = None, ) -> None: """Stream LLM response into buffer with periodic DB flushes.""" MAX_TOOL_ROUNDS = 5 msg_id = buf.assistant_message_id + # Build context inside the background task so the 202 response returns immediately + buf.append_event("status", {"status": "Building context..."}) + messages, context_meta = await build_context( + user_id, history, context_note_id, user_content, exclude_note_ids=exclude_note_ids + ) + # Emit context event buf.append_event("context", {"context": context_meta}) @@ -296,29 +303,6 @@ async def run_generation( tool_calls=all_tool_calls if all_tool_calls else None, ) - # Count non-system messages to decide on title generation - non_system = [m for m in messages if m["role"] != "system"] - msg_count = len(non_system) - should_gen_title = not conv_title or (msg_count > 0 and msg_count % 10 == 0) - - if should_gen_title: - # Include the just-generated assistant reply for context - title_messages = messages + [ - {"role": "assistant", "content": buf.content_so_far} - ] - try: - title = await _generate_title(title_messages, model) - if title: - await update_conversation_title(user_id, conv_id, title) - except Exception: - logger.warning("Failed to generate title for conversation %d", conv_id, exc_info=True) - # Fallback for first message only - if not conv_title: - fallback = user_content[:80] - if len(user_content) > 80: - fallback += "..." - await update_conversation_title(user_id, conv_id, fallback) - timing["total_ms"] = int((time.monotonic() - t_start) * 1000) logger.info( "Generation timing for conv %d: total=%dms ttft=%s intent=%s tools=%s generation=%s", @@ -334,6 +318,31 @@ async def run_generation( buf.finished_at = time.monotonic() buf.append_event("done", {"done": True, "message_id": msg_id, "timing": timing}) + # Title generation is non-critical — fire-and-forget so done fires immediately + non_system = [m for m in messages if m["role"] != "system"] + msg_count = len(non_system) + should_gen_title = not conv_title or (msg_count > 0 and msg_count % 10 == 0) + + if should_gen_title: + title_messages = messages + [ + {"role": "assistant", "content": buf.content_so_far} + ] + + async def _bg_title() -> None: + try: + title = await _generate_title(title_messages, model) + if title: + await update_conversation_title(user_id, conv_id, title) + except Exception: + logger.warning("Failed to generate title for conversation %d", conv_id, exc_info=True) + if not conv_title: + fallback = user_content[:80] + if len(user_content) > 80: + fallback += "..." + await update_conversation_title(user_id, conv_id, fallback) + + asyncio.create_task(_bg_title()) + except Exception as e: logger.exception("Error in generation task for conversation %d", conv_id) # Save partial content with error status diff --git a/src/fabledassistant/services/intent.py b/src/fabledassistant/services/intent.py index d52f3fc..1fd897b 100644 --- a/src/fabledassistant/services/intent.py +++ b/src/fabledassistant/services/intent.py @@ -136,7 +136,7 @@ async def classify_intent( messages.append({"role": "user", "content": user_message}) try: - raw = await generate_completion(messages, model) + raw = await generate_completion(messages, model, max_tokens=200) except Exception: logger.warning("Intent classification LLM call failed", exc_info=True) return IntentResult() diff --git a/src/fabledassistant/services/llm.py b/src/fabledassistant/services/llm.py index b705c03..5a9f0fe 100644 --- a/src/fabledassistant/services/llm.py +++ b/src/fabledassistant/services/llm.py @@ -173,12 +173,18 @@ async def stream_chat_with_tools( break -async def generate_completion(messages: list[dict], model: str) -> str: +async def generate_completion(messages: list[dict], model: str, max_tokens: int = 4096) -> str: """Non-streaming chat completion, returns full response text.""" async with httpx.AsyncClient(timeout=httpx.Timeout(1800.0, connect=30.0, read=300.0)) as client: resp = await client.post( f"{Config.OLLAMA_URL}/api/chat", - json={"model": model, "messages": messages, "stream": False}, + json={ + "model": model, + "messages": messages, + "stream": False, + "think": False, + "options": {"num_predict": max_tokens}, + }, ) resp.raise_for_status() data = resp.json()