feat: add Wikipedia as research pipeline source alongside SearXNG

Runs wiki_search in parallel with SearXNG queries; Wikipedia results
(which already carry content via their extract field) are merged into
the source pool before outline generation, skipping a separate fetch
step. Also fixes a pre-existing F811 ruff violation in the test file.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-04-17 21:59:35 -04:00
parent d5e6a8f6da
commit 8db6b4d230
2 changed files with 69 additions and 6 deletions
+30 -4
View File
@@ -10,6 +10,7 @@ import httpx
from fabledassistant.config import Config
from fabledassistant.services.llm import fetch_url_content, generate_completion, stream_chat
from fabledassistant.services.notes import create_note, update_note
from fabledassistant.services.wikipedia import wiki_search
from fabledassistant.models.note import Note
logger = logging.getLogger(__name__)
@@ -205,7 +206,7 @@ async def run_research_pipeline(
queries = await _generate_sub_queries(topic, model)
logger.info("Research: generated %d sub-queries for topic '%s'", len(queries), topic)
# Step 2: Search all queries in parallel (200 ms stagger to avoid hammering SearXNG)
# Step 2: Search all queries in parallel (SearXNG + Wikipedia)
async def _search_with_stagger(i: int, query: str) -> tuple[str, list[dict]]:
if i > 0:
await asyncio.sleep(0.2 * i)
@@ -214,13 +215,22 @@ async def run_research_pipeline(
logger.info("Research: query '%s'%d results", query, len(results))
return query, results
search_results = await asyncio.gather(
async def _wiki_for_query(query: str) -> list[dict]:
return await wiki_search(query, limit=1)
searxng_task = asyncio.gather(
*[_search_with_stagger(i, q) for i, q in enumerate(queries)]
)
wiki_task = asyncio.gather(
*[_wiki_for_query(q) for q in queries]
)
search_results, wiki_results = await asyncio.gather(searxng_task, wiki_task)
# Deduplicate URLs across all queries
seen_urls: set[str] = set()
url_tasks: list[tuple[str, dict, str]] = [] # (url, result_dict, query)
wiki_sources: list[dict] = [] # Wikipedia articles (already have content)
for query, results in search_results:
for result in results[:PAGES_PER_QUERY]:
url = result.get("url", "")
@@ -228,7 +238,21 @@ async def run_research_pipeline(
seen_urls.add(url)
url_tasks.append((url, result, query))
# Fetch all unique URLs in parallel
# Add Wikipedia results (they already have content via extract)
for query, wiki_hits in zip(queries, wiki_results):
for hit in wiki_hits:
url = hit.get("url", "")
if url and url not in seen_urls:
seen_urls.add(url)
wiki_sources.append({
"url": url,
"title": hit["title"],
"query": query,
"snippet": hit["extract"][:200],
"content": hit["extract"],
})
# Fetch all unique SearXNG URLs in parallel
async def _fetch_source(url: str, result: dict, query: str) -> dict:
title = result.get("title", url)
_status(f"Reading: {title[:60]}...")
@@ -241,10 +265,12 @@ async def run_research_pipeline(
"content": content,
}
all_sources: list[dict] = list(await asyncio.gather(
fetched_sources: list[dict] = list(await asyncio.gather(
*[_fetch_source(url, result, query) for url, result, query in url_tasks]
))
all_sources = wiki_sources + fetched_sources
if not all_sources:
raise ValueError(f"No results found for '{topic}'")