Phase 22b: Parallel research fetching, streaming synthesis, intent optimizations

research.py:
- Parallelize all 5 SearXNG queries concurrently (200ms stagger via asyncio.gather)
- Parallelize all URL fetches in parallel (asyncio.gather) — up to 15 URLs at once
  instead of sequential fetches; biggest performance win (was O(n) × 15s, now ~15s flat)
- _synthesize_note accepts buf: when provided uses stream_chat (num_ctx=16384,
  num_predict=8192) to emit tokens into the chat buffer in real time so users see
  the note being written; falls back to generate_completion when buf=None
- Added \n\n---\n\n separator before "Research complete!" to cleanly mark boundary
  after streamed synthesis content

intent.py:
- classify_intent passes num_ctx=4096 to generate_completion — reduces VRAM pressure
  and prefill time for the intent model call on every single request

generation_task.py:
- _INTENT_TRIGGER_WORDS frozenset (~50 action/object/date words) + _should_skip_intent()
  skips intent classification for short messages (≤10 words) with no trigger words;
  saves 400-800ms model call for conversational replies ("thanks", "okay", etc.)
- Added \n\n---\n\n separator before research "done" text in research_topic branch

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-02-27 18:24:15 -05:00
parent 590682a5d2
commit df4c52412d
4 changed files with 133 additions and 40 deletions
@@ -97,6 +97,42 @@ _TOOL_ACTIONS: dict[str, str] = {
}
# Words that strongly suggest a tool call is needed.
# If none of these appear in a short message, skip intent classification.
_INTENT_TRIGGER_WORDS: frozenset[str] = frozenset({
# Creation
"create", "add", "make", "new", "write", "set",
# Objects / tools
"note", "notes", "task", "tasks", "event", "calendar", "reminder", "todo",
"meeting", "appointment", "schedule", "due", "deadline",
# Read / search
"find", "search", "look", "show", "list", "get", "read", "open", "fetch",
# Research / web
"research", "investigate", "compile", "report", "google", "web",
# Mutation
"update", "edit", "change", "rename", "move", "reschedule", "delete",
"remove", "cancel", "complete", "finish", "mark", "tag", "untag", "append",
# Dates / times (might trigger calendar tools)
"today", "tomorrow", "yesterday", "next", "last", "week", "month",
"monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday",
# Misc triggers
"overdue", "priority", "high", "urgent", "remind", "alert",
})
def _should_skip_intent(message: str) -> bool:
"""Return True if the message is clearly conversational and needs no tool.
Skips intent classification for short messages (≤ 10 words) that contain
none of the trigger words. This saves a model call (~400-800ms) for simple
exchanges like "thanks", "okay", "can you explain that more?", etc.
"""
words = message.lower().split()
if len(words) > 10:
return False
return not any(w in _INTENT_TRIGGER_WORDS for w in words)
async def _generate_title(messages: list[dict], model: str) -> str:
"""Ask the LLM for a concise conversation title."""
# Build conversation text like summarize_conversation_as_note
@@ -227,7 +263,7 @@ async def run_generation(
intent_task: asyncio.Task[IntentResult] | None = None
t_intent = time.monotonic()
if tools:
if tools and not _should_skip_intent(user_content):
intent_history = [
m for m in history_to_use
if m.get("role") in ("user", "assistant") and m.get("content")
@@ -235,6 +271,8 @@ async def run_generation(
intent_task = asyncio.create_task(
classify_intent(user_content, tools, intent_model, history=intent_history)
)
elif tools:
logger.debug("Skipping intent classification for short/conversational message")
messages, context_meta = await context_task
@@ -301,7 +339,7 @@ async def run_generation(
topic, user_id, model, intent_model, buf
)
done_text = (
f"Research complete! I've compiled a note: "
f"\n\n---\n\nResearch complete! I've compiled a note: "
f"**[{note.title}](/notes/{note.id})**."
)
buf.append_event("chunk", {"chunk": done_text})