Reduce perceived latency: move context build into task, title fire-and-forget, think:False on aux calls
- build_context() moved from route handler into run_generation() background task. The 202 response now returns immediately; client connects to SSE before note search / URL fetch begins, so 'Building context...' status is visible. - _generate_title() runs in a fire-and-forget asyncio.create_task() after the 'done' SSE event fires. Users see their response complete 2–5s sooner on new conversations; title appears later in the sidebar without blocking the stream. - generate_completion() now sets think:False and accepts a max_tokens limit. Intent classifier passes max_tokens=200 (JSON only), title generator passes max_tokens=30 (short title), eliminating qwen3 thinking-mode overhead on these auxiliary calls. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -23,7 +23,6 @@ from fabledassistant.services.generation_buffer import (
|
||||
get_buffer,
|
||||
)
|
||||
from fabledassistant.services.generation_task import run_generation
|
||||
from fabledassistant.services.llm import build_context
|
||||
from fabledassistant.services.settings import get_setting
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -127,17 +126,14 @@ async def send_message_route(conv_id: int):
|
||||
if msg.role != "system":
|
||||
history.append({"role": msg.role, "content": msg.content})
|
||||
|
||||
# Build context with note search, URL fetching, etc.
|
||||
messages, context_meta = await build_context(
|
||||
uid, history, context_note_id, content, exclude_note_ids=exclude_note_ids
|
||||
)
|
||||
|
||||
model = conv.model or await get_setting(uid, "default_model", Config.OLLAMA_MODEL)
|
||||
|
||||
# Launch background generation task
|
||||
# Launch background generation task (context building happens inside the task)
|
||||
asyncio.create_task(run_generation(
|
||||
buf, messages, model, context_meta,
|
||||
buf, history, model,
|
||||
uid, conv_id, conv.title, content,
|
||||
context_note_id=context_note_id,
|
||||
exclude_note_ids=exclude_note_ids,
|
||||
))
|
||||
|
||||
return jsonify({
|
||||
|
||||
Reference in New Issue
Block a user