Phase 22: SearXNG web research pipeline + settings layout overhaul
Research pipeline (research_topic tool): - New service: services/research.py — sub-query generation, SearXNG search, URL fetch, deduplication, and LLM synthesis into a note - 5 sub-queries × 3 pages = up to 15 sources, capped at 12 for synthesis - Synthesis uses num_ctx=16384 + max_tokens=8192 for long-form output - Prompt demands 2500+ words, 6+ topic-appropriate sections, detailed prose - 429 retry with backoff; 1s inter-query sleep; raw_decode JSON parsing search_web tool (new): - Lightweight single-query SearXNG search, results returned inline in chat - LLM answers conversationally in round 1; no note created - web_search result type with external links in ToolCallCard Infrastructure: - llm.py: generate_completion accepts num_ctx override - config.py: SEARXNG_URL + Config.searxng_enabled() - docker-compose: OLLAMA_NUM_PARALLEL=2, commented SEARXNG_URL example - intent.py: search_web and research_topic routing rules Settings UI: - 2-column grid layout (small sections pair up, complex span full width) - Search Test section: live SearXNG query with result preview - GET /api/settings/search?q= proxy endpoint - Research button (magnifier) in ChatView input toolbar → popover modal Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,242 @@
|
||||
"""Web research pipeline: sub-queries → SearXNG → fetch → synthesize → note."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
|
||||
import httpx
|
||||
|
||||
from fabledassistant.config import Config
|
||||
from fabledassistant.services.llm import fetch_url_content, generate_completion
|
||||
from fabledassistant.services.notes import create_note
|
||||
from fabledassistant.models.note import Note
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SEARXNG_QUERIES = 5 # sub-queries to generate
|
||||
RESULTS_PER_QUERY = 3 # results fetched from SearXNG per query
|
||||
PAGES_PER_QUERY = 3 # pages actually read per sub-query (top N results)
|
||||
MAX_SYNTHESIS_SOURCES = 12 # deduplicated sources passed to synthesis LLM
|
||||
CHARS_PER_SOURCE = 2000 # content chars per source sent to synthesis
|
||||
|
||||
|
||||
async def run_research_pipeline(
|
||||
topic: str,
|
||||
user_id: int,
|
||||
model: str,
|
||||
intent_model: str,
|
||||
buf,
|
||||
) -> Note:
|
||||
"""Full research pipeline: search → fetch → synthesize → create note.
|
||||
|
||||
Emits status events via buf.append_event throughout.
|
||||
Returns the created Note.
|
||||
"""
|
||||
# Step 1: Generate sub-queries
|
||||
buf.append_event("status", {"status": "Generating search queries..."})
|
||||
queries = await _generate_sub_queries(topic, intent_model)
|
||||
logger.info("Research: generated %d sub-queries for topic '%s'", len(queries), topic)
|
||||
|
||||
# Step 2: Search and fetch
|
||||
all_sources: list[dict] = []
|
||||
seen_urls: set[str] = set()
|
||||
|
||||
for i, query in enumerate(queries):
|
||||
if i > 0:
|
||||
await asyncio.sleep(1.0) # avoid hammering SearXNG
|
||||
buf.append_event("status", {"status": f"Searching: {query}..."})
|
||||
results = await _search_searxng(query)
|
||||
logger.info("Research: query '%s' → %d results", query, len(results))
|
||||
|
||||
for result in results[:PAGES_PER_QUERY]:
|
||||
url = result.get("url", "")
|
||||
if not url or url in seen_urls:
|
||||
continue
|
||||
seen_urls.add(url)
|
||||
title = result.get("title", url)
|
||||
buf.append_event("status", {"status": f"Reading: {title[:60]}..."})
|
||||
content = await fetch_url_content(url)
|
||||
all_sources.append({
|
||||
"url": url,
|
||||
"title": title,
|
||||
"query": query,
|
||||
"snippet": result.get("snippet", ""),
|
||||
"content": content,
|
||||
})
|
||||
|
||||
if not all_sources:
|
||||
raise ValueError(f"No results found for '{topic}'")
|
||||
|
||||
# Step 3: Filter failed fetches
|
||||
good_sources = [
|
||||
s for s in all_sources
|
||||
if not s["content"].startswith("[Failed to fetch")
|
||||
]
|
||||
|
||||
if not good_sources:
|
||||
raise ValueError(f"Could not read any sources for '{topic}'")
|
||||
|
||||
# Limit to top N sources for synthesis (already deduplicated by URL)
|
||||
synthesis_sources = good_sources[:MAX_SYNTHESIS_SOURCES]
|
||||
logger.info(
|
||||
"Research: %d/%d sources successfully fetched, using %d for synthesis",
|
||||
len(good_sources), len(all_sources), len(synthesis_sources),
|
||||
)
|
||||
|
||||
# Step 4: Synthesize
|
||||
buf.append_event("status", {"status": f"Synthesizing report from {len(synthesis_sources)} sources..."})
|
||||
title, body = await _synthesize_note(topic, synthesis_sources, model)
|
||||
|
||||
# Step 5: Create note
|
||||
buf.append_event("status", {"status": "Saving note..."})
|
||||
note = await create_note(
|
||||
user_id=user_id,
|
||||
title=title,
|
||||
body=body,
|
||||
tags=["research"],
|
||||
)
|
||||
logger.info("Research: created note id=%d title='%s'", note.id, note.title)
|
||||
return note
|
||||
|
||||
|
||||
async def _generate_sub_queries(topic: str, intent_model: str) -> list[str]:
|
||||
"""Ask the intent model for focused search queries for the topic."""
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
f"You are a research assistant. Given a research topic, generate exactly {SEARXNG_QUERIES} "
|
||||
"focused web search queries that together would provide comprehensive coverage of the topic. "
|
||||
"Vary the angle of each query: include overview, implementation details, best practices, "
|
||||
"common problems, and real-world examples. "
|
||||
"Respond with ONLY a JSON array of strings, no other text. "
|
||||
'Example: ["query one", "query two", "query three"]'
|
||||
),
|
||||
},
|
||||
{"role": "user", "content": f"Topic: {topic}"},
|
||||
]
|
||||
try:
|
||||
raw = await generate_completion(messages, intent_model, max_tokens=200)
|
||||
raw = raw.strip()
|
||||
raw = re.sub(r"^```(?:json)?\s*", "", raw)
|
||||
raw = re.sub(r"\s*```$", "", raw)
|
||||
idx = raw.find("[")
|
||||
if idx >= 0:
|
||||
parsed, _ = json.JSONDecoder().raw_decode(raw[idx:])
|
||||
if isinstance(parsed, list) and parsed:
|
||||
queries = [str(q).strip() for q in parsed if str(q).strip()]
|
||||
if queries:
|
||||
return queries[:SEARXNG_QUERIES]
|
||||
except Exception:
|
||||
logger.warning("Sub-query generation failed, falling back to topic", exc_info=True)
|
||||
return [topic]
|
||||
|
||||
|
||||
async def _search_searxng(query: str) -> list[dict]:
|
||||
"""Search SearXNG and return top results as [{url, title, snippet}]."""
|
||||
url = Config.SEARXNG_URL.rstrip("/") + "/search"
|
||||
params = {"q": query, "format": "json", "categories": "general"}
|
||||
for attempt in range(3):
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=10.0) as client:
|
||||
resp = await client.get(url, params=params)
|
||||
if resp.status_code == 429:
|
||||
retry_after = int(resp.headers.get("Retry-After", "5"))
|
||||
wait = min(retry_after, 10) * (attempt + 1)
|
||||
logger.warning(
|
||||
"SearXNG 429 for query '%s' (attempt %d/3), waiting %ds",
|
||||
query, attempt + 1, wait,
|
||||
)
|
||||
await asyncio.sleep(wait)
|
||||
continue
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
results = data.get("results", [])
|
||||
out = []
|
||||
for r in results[:RESULTS_PER_QUERY]:
|
||||
out.append({
|
||||
"url": r.get("url", ""),
|
||||
"title": r.get("title", ""),
|
||||
"snippet": r.get("content", ""),
|
||||
})
|
||||
return out
|
||||
except httpx.HTTPStatusError:
|
||||
logger.warning("SearXNG search failed for query '%s'", query, exc_info=True)
|
||||
return []
|
||||
except Exception:
|
||||
logger.warning("SearXNG search failed for query '%s'", query, exc_info=True)
|
||||
return []
|
||||
logger.warning("SearXNG search gave up after 3 attempts for query '%s'", query)
|
||||
return []
|
||||
|
||||
|
||||
async def _synthesize_note(
|
||||
topic: str,
|
||||
sources: list[dict],
|
||||
model: str,
|
||||
) -> tuple[str, str]:
|
||||
"""Synthesize a comprehensive markdown research document from fetched sources.
|
||||
|
||||
Returns (title, body_markdown).
|
||||
Uses an extended context window so the output can be several thousand words.
|
||||
"""
|
||||
sources_text_parts = []
|
||||
for i, s in enumerate(sources, 1):
|
||||
content = (s.get("content") or s.get("snippet") or "")[:CHARS_PER_SOURCE]
|
||||
sources_text_parts.append(
|
||||
f"[Source {i}] {s['title']}\nURL: {s['url']}\nSearch query: {s['query']}\n\n{content}"
|
||||
)
|
||||
sources_block = "\n\n" + ("─" * 60) + "\n\n".join(sources_text_parts)
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
"You are a thorough researcher and writer. "
|
||||
"Your task is to write an exhaustive, well-structured document on the given topic — "
|
||||
"not a brief summary or intro paragraph.\n\n"
|
||||
"Requirements:\n"
|
||||
"- Write at least 2500 words of substantive content (excluding the Sources section)\n"
|
||||
"- Choose sections (##) that make sense for the topic — let the subject matter determine the structure. "
|
||||
"A technical topic might need implementation, configuration, and troubleshooting sections. "
|
||||
"A comparison topic might need dedicated sections per subject being compared plus a summary. "
|
||||
"A scientific topic might need background, mechanisms, research findings, and implications. "
|
||||
"Use your judgment — minimum 6 major sections.\n"
|
||||
"- Use ### for subsections where they add clarity\n"
|
||||
"- Write in detailed prose paragraphs — do not reduce sections to bullet-point lists\n"
|
||||
"- Include specific details, examples, data points, comparisons, and nuance from the sources\n"
|
||||
"- Do not pad with vague generalities — every paragraph should say something concrete\n"
|
||||
"- The first line must be the document title starting with '# '\n"
|
||||
"- End with a '## Sources' section listing every source as a markdown hyperlink\n\n"
|
||||
"The reader wants to finish this document with a thorough understanding of the topic, "
|
||||
"not just an overview."
|
||||
),
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
f"Write a comprehensive reference document on: {topic}\n\n"
|
||||
f"Sources ({len(sources)} pages fetched):\n{sources_block}"
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
raw = await generate_completion(
|
||||
messages,
|
||||
model,
|
||||
max_tokens=8192,
|
||||
num_ctx=16384,
|
||||
)
|
||||
raw = raw.strip()
|
||||
|
||||
# Extract title from first # heading
|
||||
lines = raw.splitlines()
|
||||
title = f"Research: {topic}"
|
||||
body_lines = lines
|
||||
if lines and lines[0].startswith("# "):
|
||||
title = lines[0][2:].strip()
|
||||
body_lines = lines[1:]
|
||||
|
||||
body = "\n".join(body_lines).strip()
|
||||
return title, body
|
||||
Reference in New Issue
Block a user