Reduce perceived latency: move context build into task, title fire-and-forget, think:False on aux calls
- build_context() moved from route handler into run_generation() background task. The 202 response now returns immediately; client connects to SSE before note search / URL fetch begins, so 'Building context...' status is visible. - _generate_title() runs in a fire-and-forget asyncio.create_task() after the 'done' SSE event fires. Users see their response complete 2–5s sooner on new conversations; title appears later in the sidebar without blocking the stream. - generate_completion() now sets think:False and accepts a max_tokens limit. Intent classifier passes max_tokens=200 (JSON only), title generator passes max_tokens=30 (short title), eliminating qwen3 thinking-mode overhead on these auxiliary calls. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -16,7 +16,7 @@ from fabledassistant.config import Config
|
||||
from fabledassistant.models import async_session
|
||||
from fabledassistant.models.conversation import Message
|
||||
from fabledassistant.services.generation_buffer import GenerationBuffer, GenerationState
|
||||
from fabledassistant.services.llm import ChatChunk, generate_completion, stream_chat, stream_chat_with_tools
|
||||
from fabledassistant.services.llm import ChatChunk, build_context, generate_completion, stream_chat, stream_chat_with_tools
|
||||
from fabledassistant.services.chat import update_conversation_title
|
||||
from fabledassistant.services.intent import classify_intent
|
||||
from fabledassistant.services.logging import log_generation
|
||||
@@ -73,7 +73,7 @@ async def _generate_title(messages: list[dict], model: str) -> str:
|
||||
},
|
||||
{"role": "user", "content": "\n\n".join(conv_lines)},
|
||||
]
|
||||
title = await generate_completion(prompt_messages, model)
|
||||
title = await generate_completion(prompt_messages, model, max_tokens=30)
|
||||
title = title.strip().strip('"\'').strip()
|
||||
return title[:100] if title else ""
|
||||
|
||||
@@ -98,18 +98,25 @@ async def _update_message(
|
||||
|
||||
async def run_generation(
|
||||
buf: GenerationBuffer,
|
||||
messages: list[dict],
|
||||
history: list[dict],
|
||||
model: str,
|
||||
context_meta: dict,
|
||||
user_id: int,
|
||||
conv_id: int,
|
||||
conv_title: str,
|
||||
user_content: str,
|
||||
context_note_id: int | None = None,
|
||||
exclude_note_ids: list[int] | None = None,
|
||||
) -> None:
|
||||
"""Stream LLM response into buffer with periodic DB flushes."""
|
||||
MAX_TOOL_ROUNDS = 5
|
||||
msg_id = buf.assistant_message_id
|
||||
|
||||
# Build context inside the background task so the 202 response returns immediately
|
||||
buf.append_event("status", {"status": "Building context..."})
|
||||
messages, context_meta = await build_context(
|
||||
user_id, history, context_note_id, user_content, exclude_note_ids=exclude_note_ids
|
||||
)
|
||||
|
||||
# Emit context event
|
||||
buf.append_event("context", {"context": context_meta})
|
||||
|
||||
@@ -296,29 +303,6 @@ async def run_generation(
|
||||
tool_calls=all_tool_calls if all_tool_calls else None,
|
||||
)
|
||||
|
||||
# Count non-system messages to decide on title generation
|
||||
non_system = [m for m in messages if m["role"] != "system"]
|
||||
msg_count = len(non_system)
|
||||
should_gen_title = not conv_title or (msg_count > 0 and msg_count % 10 == 0)
|
||||
|
||||
if should_gen_title:
|
||||
# Include the just-generated assistant reply for context
|
||||
title_messages = messages + [
|
||||
{"role": "assistant", "content": buf.content_so_far}
|
||||
]
|
||||
try:
|
||||
title = await _generate_title(title_messages, model)
|
||||
if title:
|
||||
await update_conversation_title(user_id, conv_id, title)
|
||||
except Exception:
|
||||
logger.warning("Failed to generate title for conversation %d", conv_id, exc_info=True)
|
||||
# Fallback for first message only
|
||||
if not conv_title:
|
||||
fallback = user_content[:80]
|
||||
if len(user_content) > 80:
|
||||
fallback += "..."
|
||||
await update_conversation_title(user_id, conv_id, fallback)
|
||||
|
||||
timing["total_ms"] = int((time.monotonic() - t_start) * 1000)
|
||||
logger.info(
|
||||
"Generation timing for conv %d: total=%dms ttft=%s intent=%s tools=%s generation=%s",
|
||||
@@ -334,6 +318,31 @@ async def run_generation(
|
||||
buf.finished_at = time.monotonic()
|
||||
buf.append_event("done", {"done": True, "message_id": msg_id, "timing": timing})
|
||||
|
||||
# Title generation is non-critical — fire-and-forget so done fires immediately
|
||||
non_system = [m for m in messages if m["role"] != "system"]
|
||||
msg_count = len(non_system)
|
||||
should_gen_title = not conv_title or (msg_count > 0 and msg_count % 10 == 0)
|
||||
|
||||
if should_gen_title:
|
||||
title_messages = messages + [
|
||||
{"role": "assistant", "content": buf.content_so_far}
|
||||
]
|
||||
|
||||
async def _bg_title() -> None:
|
||||
try:
|
||||
title = await _generate_title(title_messages, model)
|
||||
if title:
|
||||
await update_conversation_title(user_id, conv_id, title)
|
||||
except Exception:
|
||||
logger.warning("Failed to generate title for conversation %d", conv_id, exc_info=True)
|
||||
if not conv_title:
|
||||
fallback = user_content[:80]
|
||||
if len(user_content) > 80:
|
||||
fallback += "..."
|
||||
await update_conversation_title(user_id, conv_id, fallback)
|
||||
|
||||
asyncio.create_task(_bg_title())
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("Error in generation task for conversation %d", conv_id)
|
||||
# Save partial content with error status
|
||||
|
||||
Reference in New Issue
Block a user