bvandeusen/FabledScribe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 50 Wiki Activity

Phase 22b: Parallel research fetching, streaming synthesis, intent optimizations

Browse Source 
research.py:
- Parallelize all 5 SearXNG queries concurrently (200ms stagger via asyncio.gather)
- Parallelize all URL fetches in parallel (asyncio.gather) — up to 15 URLs at once
  instead of sequential fetches; biggest performance win (was O(n) × 15s, now ~15s flat)
- _synthesize_note accepts buf: when provided uses stream_chat (num_ctx=16384,
  num_predict=8192) to emit tokens into the chat buffer in real time so users see
  the note being written; falls back to generate_completion when buf=None
- Added \n\n---\n\n separator before "Research complete!" to cleanly mark boundary
  after streamed synthesis content

intent.py:
- classify_intent passes num_ctx=4096 to generate_completion — reduces VRAM pressure
  and prefill time for the intent model call on every single request

generation_task.py:
- _INTENT_TRIGGER_WORDS frozenset (~50 action/object/date words) + _should_skip_intent()
  skips intent classification for short messages (≤10 words) with no trigger words;
  saves 400-800ms model call for conversational replies ("thanks", "okay", etc.)
- Added \n\n---\n\n separator before research "done" text in research_topic branch

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Bryan Van Deusen
2026-02-27 18:24:15 -05:00
parent 590682a5d2
commit df4c52412d
4 changed files with 133 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/fabledassistant/services/research.py
+56 -30
View File
@@ -8,7 +8,7 @@ import re
import httpx
from fabledassistant.config import Config
from fabledassistant.services.llm import fetch_url_content, generate_completion
from fabledassistant.services.llm import fetch_url_content, generate_completion, stream_chat
from fabledassistant.services.notes import create_note
from fabledassistant.models.note import Note
@@ -38,32 +38,45 @@ async def run_research_pipeline(
queries = await _generate_sub_queries(topic, intent_model)
logger.info("Research: generated %d sub-queries for topic '%s'", len(queries), topic)
# Step 2: Search and fetch
all_sources: list[dict] = []
seen_urls: set[str] = set()
for i, query in enumerate(queries):
# Step 2: Search all queries in parallel (200 ms stagger to avoid hammering SearXNG)
async def _search_with_stagger(i: int, query: str) -> tuple[str, list[dict]]:
if i > 0:
await asyncio.sleep(1.0) # avoid hammering SearXNG
await asyncio.sleep(0.2 * i)
buf.append_event("status", {"status": f"Searching: {query}..."})
results = await _search_searxng(query)
logger.info("Research: query '%s' → %d results", query, len(results))
return query, results
search_results = await asyncio.gather(
*[_search_with_stagger(i, q) for i, q in enumerate(queries)]
)
# Deduplicate URLs across all queries
seen_urls: set[str] = set()
url_tasks: list[tuple[str, dict, str]] = [] # (url, result_dict, query)
for query, results in search_results:
for result in results[:PAGES_PER_QUERY]:
url = result.get("url", "")
if not url or url in seen_urls:
continue
seen_urls.add(url)
title = result.get("title", url)
buf.append_event("status", {"status": f"Reading: {title[:60]}..."})
content = await fetch_url_content(url)
all_sources.append({
"url": url,
"title": title,
"query": query,
"snippet": result.get("snippet", ""),
"content": content,
})
if url and url not in seen_urls:
seen_urls.add(url)
url_tasks.append((url, result, query))
# Fetch all unique URLs in parallel
async def _fetch_source(url: str, result: dict, query: str) -> dict:
title = result.get("title", url)
buf.append_event("status", {"status": f"Reading: {title[:60]}..."})
content = await fetch_url_content(url)
return {
"url": url,
"title": title,
"query": query,
"snippet": result.get("snippet", ""),
"content": content,
}
all_sources: list[dict] = list(await asyncio.gather(
*[_fetch_source(url, result, query) for url, result, query in url_tasks]
))
if not all_sources:
raise ValueError(f"No results found for '{topic}'")
@@ -84,9 +97,9 @@ async def run_research_pipeline(
len(good_sources), len(all_sources), len(synthesis_sources),
)
# Step 4: Synthesize
# Step 4: Synthesize (streams tokens into chat as the note is being written)
buf.append_event("status", {"status": f"Synthesizing report from {len(synthesis_sources)} sources..."})
title, body = await _synthesize_note(topic, synthesis_sources, model)
title, body = await _synthesize_note(topic, synthesis_sources, model, buf)
# Step 5: Create note
buf.append_event("status", {"status": "Saving note..."})
@@ -175,11 +188,13 @@ async def _synthesize_note(
topic: str,
sources: list[dict],
model: str,
buf=None,
) -> tuple[str, str]:
"""Synthesize a comprehensive markdown research document from fetched sources.
Returns (title, body_markdown).
Uses an extended context window so the output can be several thousand words.
When buf is provided, tokens are streamed into the chat buffer in real time
so the user can see the note being written. Uses an extended context window.
"""
sources_text_parts = []
for i, s in enumerate(sources, 1):
@@ -222,13 +237,24 @@ async def _synthesize_note(
},
]
raw = await generate_completion(
messages,
model,
max_tokens=8192,
num_ctx=16384,
)
raw = raw.strip()
if buf is not None:
# Stream tokens into the chat buffer so the user sees the note being written
raw_parts: list[str] = []
async for token in stream_chat(
messages, model, options={"num_ctx": 16384, "num_predict": 8192}
):
raw_parts.append(token)
buf.append_event("chunk", {"chunk": token})
buf.content_so_far += token
raw = "".join(raw_parts).strip()
else:
raw = await generate_completion(
messages,
model,
max_tokens=8192,
num_ctx=16384,
)
raw = raw.strip()
# Extract title from first # heading
lines = raw.splitlines()