Phase 22b: Parallel research fetching, streaming synthesis, intent optimizations
research.py:
- Parallelize all 5 SearXNG queries concurrently (200ms stagger via asyncio.gather)
- Parallelize all URL fetches in parallel (asyncio.gather) — up to 15 URLs at once
instead of sequential fetches; biggest performance win (was O(n) × 15s, now ~15s flat)
- _synthesize_note accepts buf: when provided uses stream_chat (num_ctx=16384,
num_predict=8192) to emit tokens into the chat buffer in real time so users see
the note being written; falls back to generate_completion when buf=None
- Added \n\n---\n\n separator before "Research complete!" to cleanly mark boundary
after streamed synthesis content
intent.py:
- classify_intent passes num_ctx=4096 to generate_completion — reduces VRAM pressure
and prefill time for the intent model call on every single request
generation_task.py:
- _INTENT_TRIGGER_WORDS frozenset (~50 action/object/date words) + _should_skip_intent()
skips intent classification for short messages (≤10 words) with no trigger words;
saves 400-800ms model call for conversational replies ("thanks", "okay", etc.)
- Added \n\n---\n\n separator before research "done" text in research_topic branch
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -97,6 +97,42 @@ _TOOL_ACTIONS: dict[str, str] = {
|
||||
}
|
||||
|
||||
|
||||
# Words that strongly suggest a tool call is needed.
|
||||
# If none of these appear in a short message, skip intent classification.
|
||||
_INTENT_TRIGGER_WORDS: frozenset[str] = frozenset({
|
||||
# Creation
|
||||
"create", "add", "make", "new", "write", "set",
|
||||
# Objects / tools
|
||||
"note", "notes", "task", "tasks", "event", "calendar", "reminder", "todo",
|
||||
"meeting", "appointment", "schedule", "due", "deadline",
|
||||
# Read / search
|
||||
"find", "search", "look", "show", "list", "get", "read", "open", "fetch",
|
||||
# Research / web
|
||||
"research", "investigate", "compile", "report", "google", "web",
|
||||
# Mutation
|
||||
"update", "edit", "change", "rename", "move", "reschedule", "delete",
|
||||
"remove", "cancel", "complete", "finish", "mark", "tag", "untag", "append",
|
||||
# Dates / times (might trigger calendar tools)
|
||||
"today", "tomorrow", "yesterday", "next", "last", "week", "month",
|
||||
"monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday",
|
||||
# Misc triggers
|
||||
"overdue", "priority", "high", "urgent", "remind", "alert",
|
||||
})
|
||||
|
||||
|
||||
def _should_skip_intent(message: str) -> bool:
|
||||
"""Return True if the message is clearly conversational and needs no tool.
|
||||
|
||||
Skips intent classification for short messages (≤ 10 words) that contain
|
||||
none of the trigger words. This saves a model call (~400-800ms) for simple
|
||||
exchanges like "thanks", "okay", "can you explain that more?", etc.
|
||||
"""
|
||||
words = message.lower().split()
|
||||
if len(words) > 10:
|
||||
return False
|
||||
return not any(w in _INTENT_TRIGGER_WORDS for w in words)
|
||||
|
||||
|
||||
async def _generate_title(messages: list[dict], model: str) -> str:
|
||||
"""Ask the LLM for a concise conversation title."""
|
||||
# Build conversation text like summarize_conversation_as_note
|
||||
@@ -227,7 +263,7 @@ async def run_generation(
|
||||
|
||||
intent_task: asyncio.Task[IntentResult] | None = None
|
||||
t_intent = time.monotonic()
|
||||
if tools:
|
||||
if tools and not _should_skip_intent(user_content):
|
||||
intent_history = [
|
||||
m for m in history_to_use
|
||||
if m.get("role") in ("user", "assistant") and m.get("content")
|
||||
@@ -235,6 +271,8 @@ async def run_generation(
|
||||
intent_task = asyncio.create_task(
|
||||
classify_intent(user_content, tools, intent_model, history=intent_history)
|
||||
)
|
||||
elif tools:
|
||||
logger.debug("Skipping intent classification for short/conversational message")
|
||||
|
||||
messages, context_meta = await context_task
|
||||
|
||||
@@ -301,7 +339,7 @@ async def run_generation(
|
||||
topic, user_id, model, intent_model, buf
|
||||
)
|
||||
done_text = (
|
||||
f"Research complete! I've compiled a note: "
|
||||
f"\n\n---\n\nResearch complete! I've compiled a note: "
|
||||
f"**[{note.title}](/notes/{note.id})**."
|
||||
)
|
||||
buf.append_event("chunk", {"chunk": done_text})
|
||||
|
||||
Reference in New Issue
Block a user