Phase 22: SearXNG web research pipeline + settings layout overhaul
Research pipeline (research_topic tool): - New service: services/research.py — sub-query generation, SearXNG search, URL fetch, deduplication, and LLM synthesis into a note - 5 sub-queries × 3 pages = up to 15 sources, capped at 12 for synthesis - Synthesis uses num_ctx=16384 + max_tokens=8192 for long-form output - Prompt demands 2500+ words, 6+ topic-appropriate sections, detailed prose - 429 retry with backoff; 1s inter-query sleep; raw_decode JSON parsing search_web tool (new): - Lightweight single-query SearXNG search, results returned inline in chat - LLM answers conversationally in round 1; no note created - web_search result type with external links in ToolCallCard Infrastructure: - llm.py: generate_completion accepts num_ctx override - config.py: SEARXNG_URL + Config.searxng_enabled() - docker-compose: OLLAMA_NUM_PARALLEL=2, commented SEARXNG_URL example - intent.py: search_web and research_topic routing rules Settings UI: - 2-column grid layout (small sections pair up, complex span full width) - Search Test section: live SearXNG query with result preview - GET /api/settings/search?q= proxy endpoint - Research button (magnifier) in ChatView input toolbar → popover modal Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -13,6 +13,8 @@ services:
|
||||
OLLAMA_URL: "http://ollama:11434"
|
||||
OLLAMA_MODEL: "${OLLAMA_MODEL:-llama3.1}"
|
||||
SECRET_KEY: "${SECRET_KEY:-dev-secret-change-me}"
|
||||
# Uncomment and set to enable web research via SearXNG:
|
||||
# SEARXNG_URL: "http://searxng:8080"
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:5000/api/health')"]
|
||||
interval: 10s
|
||||
@@ -40,6 +42,7 @@ services:
|
||||
- ollama_models:/root/.ollama
|
||||
environment:
|
||||
OLLAMA_MAX_LOADED_MODELS: "2"
|
||||
OLLAMA_NUM_PARALLEL: "2"
|
||||
OLLAMA_KEEP_ALIVE: "30m"
|
||||
OLLAMA_FLASH_ATTENTION: "1"
|
||||
deploy:
|
||||
|
||||
@@ -52,6 +52,12 @@ const label = computed(() => {
|
||||
return "Completed todo";
|
||||
case "todo_deleted":
|
||||
return "Deleted todo";
|
||||
case "web_search":
|
||||
return "Web search";
|
||||
case "research_pending":
|
||||
return "Research started";
|
||||
case "research_note":
|
||||
return "Research note";
|
||||
default:
|
||||
return "Tool call";
|
||||
}
|
||||
@@ -167,6 +173,12 @@ const taskCount = computed(() => {
|
||||
return (data.total as number) ?? 0;
|
||||
});
|
||||
|
||||
const webSearch = computed(() => {
|
||||
const data = props.toolCall.result.data;
|
||||
if (!data || props.toolCall.result.type !== "web_search") return null;
|
||||
return data as { query: string; results: { url: string; title: string; snippet: string }[]; count: number };
|
||||
});
|
||||
|
||||
const searchResults = computed(() => {
|
||||
const data = props.toolCall.result.data;
|
||||
if (!data || props.toolCall.result.type !== "search") return null;
|
||||
@@ -317,6 +329,19 @@ async function applyTag(tag: string) {
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
<template v-else-if="webSearch">
|
||||
<span class="tool-search-info">{{ webSearch.count }} result{{ webSearch.count !== 1 ? "s" : "" }}</span>
|
||||
<div class="tool-search-results web-search-results">
|
||||
<a
|
||||
v-for="(r, i) in webSearch.results"
|
||||
:key="i"
|
||||
:href="r.url"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
class="tool-search-item"
|
||||
>{{ r.title || r.url }}</a>
|
||||
</div>
|
||||
</template>
|
||||
<template v-else-if="linkTo">
|
||||
<router-link :to="linkTo" class="tool-link">{{ title }}</router-link>
|
||||
</template>
|
||||
@@ -400,6 +425,13 @@ async function applyTag(tag: string) {
|
||||
color: var(--color-text-muted);
|
||||
margin-right: 0.1rem;
|
||||
}
|
||||
.web-search-results {
|
||||
flex-direction: column;
|
||||
flex-wrap: nowrap;
|
||||
}
|
||||
.web-search-results .tool-search-item::after {
|
||||
content: none;
|
||||
}
|
||||
.tool-event-title {
|
||||
font-weight: 600;
|
||||
color: var(--color-text);
|
||||
|
||||
@@ -22,6 +22,10 @@ const sending = ref(false);
|
||||
const summarizing = ref(false);
|
||||
const sidebarOpen = ref(false);
|
||||
|
||||
// Research modal state
|
||||
const showResearchModal = ref(false);
|
||||
const researchTopic = ref("");
|
||||
|
||||
// Note picker state
|
||||
const attachedNote = ref<{ id: number; title: string } | null>(null);
|
||||
const showNotePicker = ref(false);
|
||||
@@ -239,6 +243,27 @@ function onInputKeydown(e: KeyboardEvent) {
|
||||
}
|
||||
}
|
||||
|
||||
// Research modal
|
||||
function toggleResearchModal() {
|
||||
showResearchModal.value = !showResearchModal.value;
|
||||
if (showResearchModal.value) {
|
||||
researchTopic.value = "";
|
||||
nextTick(() => {
|
||||
const el = document.querySelector(".research-topic-input") as HTMLInputElement;
|
||||
el?.focus();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function submitResearch() {
|
||||
const topic = researchTopic.value.trim();
|
||||
if (!topic) return;
|
||||
messageInput.value = `Research: ${topic}`;
|
||||
showResearchModal.value = false;
|
||||
researchTopic.value = "";
|
||||
sendMessage();
|
||||
}
|
||||
|
||||
// Note picker
|
||||
function toggleNotePicker() {
|
||||
showNotePicker.value = !showNotePicker.value;
|
||||
@@ -302,7 +327,12 @@ function removeIncludedNote(noteId: number) {
|
||||
// Keyboard shortcuts
|
||||
function onGlobalKeydown(e: KeyboardEvent) {
|
||||
if (e.key !== "Escape") return;
|
||||
// Close note picker first
|
||||
// Close research modal first
|
||||
if (showResearchModal.value) {
|
||||
showResearchModal.value = false;
|
||||
return;
|
||||
}
|
||||
// Close note picker
|
||||
if (showNotePicker.value) {
|
||||
showNotePicker.value = false;
|
||||
return;
|
||||
@@ -479,6 +509,37 @@ onUnmounted(() => {
|
||||
|
||||
<div class="input-wrapper">
|
||||
<div class="input-area">
|
||||
<!-- Research button -->
|
||||
<div class="research-wrapper">
|
||||
<button
|
||||
class="btn-attach"
|
||||
@click="toggleResearchModal"
|
||||
:disabled="store.streaming || !store.chatReady"
|
||||
title="Research a topic"
|
||||
>
|
||||
<svg width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||||
<circle cx="11" cy="11" r="8"/>
|
||||
<line x1="21" y1="21" x2="16.65" y2="16.65"/>
|
||||
</svg>
|
||||
</button>
|
||||
|
||||
<!-- Research modal -->
|
||||
<div v-if="showResearchModal" class="research-modal">
|
||||
<div class="research-modal-header">Research topic</div>
|
||||
<input
|
||||
class="research-topic-input"
|
||||
v-model="researchTopic"
|
||||
placeholder="e.g. quantum computing"
|
||||
@keydown.enter="submitResearch"
|
||||
@keydown.escape="showResearchModal = false"
|
||||
/>
|
||||
<div class="research-modal-actions">
|
||||
<button class="btn-research-cancel" @click="showResearchModal = false">Cancel</button>
|
||||
<button class="btn-research-go" @click="submitResearch" :disabled="!researchTopic.trim()">Go</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Note picker button -->
|
||||
<div class="note-picker-wrapper">
|
||||
<button
|
||||
@@ -1005,6 +1066,84 @@ details[open] .thinking-summary::before {
|
||||
color: var(--color-text-muted);
|
||||
font-size: 0.85rem;
|
||||
}
|
||||
|
||||
/* Research modal */
|
||||
.research-wrapper {
|
||||
position: relative;
|
||||
}
|
||||
.research-modal {
|
||||
position: absolute;
|
||||
bottom: calc(100% + 8px);
|
||||
left: 0;
|
||||
width: 280px;
|
||||
background: var(--color-bg-card);
|
||||
border: 1px solid var(--color-border);
|
||||
border-radius: var(--radius-md);
|
||||
box-shadow: 0 4px 16px var(--color-shadow);
|
||||
z-index: 10;
|
||||
padding: 0.75rem;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
.research-modal-header {
|
||||
font-size: 0.8rem;
|
||||
font-weight: 600;
|
||||
color: var(--color-text-muted);
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.04em;
|
||||
}
|
||||
.research-topic-input {
|
||||
width: 100%;
|
||||
padding: 0.4rem 0.6rem;
|
||||
border: 1px solid var(--color-border);
|
||||
border-radius: var(--radius-sm);
|
||||
background: var(--color-bg-secondary);
|
||||
color: var(--color-text);
|
||||
font-size: 0.9rem;
|
||||
outline: none;
|
||||
font-family: inherit;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
.research-topic-input:focus {
|
||||
border-color: var(--color-primary);
|
||||
}
|
||||
.research-modal-actions {
|
||||
display: flex;
|
||||
justify-content: flex-end;
|
||||
gap: 0.4rem;
|
||||
}
|
||||
.btn-research-cancel {
|
||||
padding: 0.3rem 0.65rem;
|
||||
background: none;
|
||||
border: 1px solid var(--color-border);
|
||||
border-radius: var(--radius-sm);
|
||||
color: var(--color-text-muted);
|
||||
font-size: 0.85rem;
|
||||
cursor: pointer;
|
||||
font-family: inherit;
|
||||
}
|
||||
.btn-research-cancel:hover {
|
||||
background: var(--color-bg-secondary);
|
||||
}
|
||||
.btn-research-go {
|
||||
padding: 0.3rem 0.75rem;
|
||||
background: var(--color-primary);
|
||||
border: none;
|
||||
border-radius: var(--radius-sm);
|
||||
color: #fff;
|
||||
font-size: 0.85rem;
|
||||
cursor: pointer;
|
||||
font-family: inherit;
|
||||
}
|
||||
.btn-research-go:disabled {
|
||||
opacity: 0.4;
|
||||
cursor: default;
|
||||
}
|
||||
.btn-research-go:not(:disabled):hover {
|
||||
opacity: 0.9;
|
||||
}
|
||||
|
||||
.input-area textarea {
|
||||
flex: 1;
|
||||
resize: none;
|
||||
|
||||
+560
-410
File diff suppressed because it is too large
Load Diff
@@ -56,6 +56,13 @@ class Config:
|
||||
OIDC_SCOPES: str = os.environ.get("OIDC_SCOPES", "openid profile email")
|
||||
LOCAL_AUTH_ENABLED: bool = os.environ.get("LOCAL_AUTH_ENABLED", "true").lower() not in ("0", "false", "no")
|
||||
|
||||
# SearXNG web search (external instance)
|
||||
SEARXNG_URL: str = os.environ.get("SEARXNG_URL", "")
|
||||
|
||||
@classmethod
|
||||
def oidc_enabled(cls) -> bool:
|
||||
return bool(cls.OIDC_ISSUER and cls.OIDC_CLIENT_ID and cls.OIDC_CLIENT_SECRET)
|
||||
|
||||
@classmethod
|
||||
def searxng_enabled(cls) -> bool:
|
||||
return bool(cls.SEARXNG_URL)
|
||||
|
||||
@@ -84,3 +84,17 @@ async def test_caldav():
|
||||
uid = get_current_user_id()
|
||||
result = await test_connection(uid)
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@settings_bp.route("/search", methods=["GET"])
|
||||
@login_required
|
||||
async def test_search():
|
||||
"""Test SearXNG connectivity and preview results for a query."""
|
||||
if not Config.searxng_enabled():
|
||||
return jsonify({"configured": False, "results": [], "searxng_url": ""})
|
||||
q = request.args.get("q", "").strip()
|
||||
if not q:
|
||||
return jsonify({"configured": True, "results": [], "searxng_url": Config.SEARXNG_URL})
|
||||
from fabledassistant.services.research import _search_searxng
|
||||
results = await _search_searxng(q)
|
||||
return jsonify({"configured": True, "results": results, "query": q, "searxng_url": Config.SEARXNG_URL})
|
||||
|
||||
@@ -28,6 +28,7 @@ from fabledassistant.services.intent import IntentResult, classify_intent
|
||||
from fabledassistant.services.logging import log_generation
|
||||
from fabledassistant.services.settings import get_setting
|
||||
from fabledassistant.services.tools import get_tools_for_user, execute_tool
|
||||
from fabledassistant.services.research import run_research_pipeline
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -59,6 +60,8 @@ _TOOL_LABELS: dict[str, str] = {
|
||||
"update_todo": "Updating todo",
|
||||
"complete_todo": "Completing todo",
|
||||
"delete_todo": "Removing todo",
|
||||
"search_web": "Searching the web",
|
||||
"research_topic": "Researching topic",
|
||||
}
|
||||
|
||||
# Tools that write data and require explicit user confirmation before executing.
|
||||
@@ -285,6 +288,43 @@ async def run_generation(
|
||||
if timing["ttft_ms"] is None:
|
||||
timing["ttft_ms"] = int((time.monotonic() - t_start) * 1000)
|
||||
|
||||
if tool_name == "research_topic":
|
||||
topic = intent.arguments.get("topic", "")
|
||||
if not ack_text:
|
||||
fallback_ack = f"I'll research '{topic}' and compile a note.\n\n"
|
||||
buf.append_event("chunk", {"chunk": fallback_ack})
|
||||
buf.content_so_far += fallback_ack
|
||||
if timing["ttft_ms"] is None:
|
||||
timing["ttft_ms"] = int((time.monotonic() - t_start) * 1000)
|
||||
try:
|
||||
note = await run_research_pipeline(
|
||||
topic, user_id, model, intent_model, buf
|
||||
)
|
||||
done_text = (
|
||||
f"Research complete! I've compiled a note: "
|
||||
f"**[{note.title}](/notes/{note.id})**."
|
||||
)
|
||||
buf.append_event("chunk", {"chunk": done_text})
|
||||
buf.content_so_far += done_text
|
||||
tool_record = {
|
||||
"function": "research_topic",
|
||||
"arguments": {"topic": topic},
|
||||
"result": {
|
||||
"success": True,
|
||||
"type": "research_note",
|
||||
"data": {"id": note.id, "title": note.title},
|
||||
},
|
||||
"status": "success",
|
||||
}
|
||||
all_tool_calls.append(tool_record)
|
||||
buf.append_event("tool_call", {"tool_call": tool_record})
|
||||
except Exception as e:
|
||||
logger.exception("Research pipeline failed for topic: %s", topic)
|
||||
err_text = f"\nResearch failed: {e}"
|
||||
buf.append_event("chunk", {"chunk": err_text})
|
||||
buf.content_so_far += err_text
|
||||
break # research IS the full response
|
||||
|
||||
confirmed = True
|
||||
if tool_name in _WRITE_TOOLS:
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
@@ -100,6 +100,12 @@ Rules:
|
||||
- "read", "open", "show me", "what does X say", "display", "pull up" a specific note → use get_note with query=<note name>.
|
||||
- "list my notes", "show notes", "recent notes", "browse notes", "notes tagged X" → use list_notes (with optional q or tags).
|
||||
- "tag X with Y", "add tag Y to X", "untag Y from X", "remove tag Y from X" → use update_note with tags=[Y] and tag_mode="add" or "remove".
|
||||
- search_web: user wants a quick web search to answer a factual question
|
||||
("search for X", "look up X", "what is the latest version of X", "find X online",
|
||||
"google X", "what is X" for quick factual answers — NOT when they want a comprehensive note)
|
||||
- research_topic: user wants to research a topic and create a comprehensive note from web sources
|
||||
("research X", "research X and make a note", "compile notes on X", "write a report on X",
|
||||
"deep dive into X", "find everything about X", "comprehensive guide to X")
|
||||
- "ack": one short, natural sentence confirming the action (tool path only). Vary phrasing — do not always start with "Let me". Omit (null) for chat-only responses.
|
||||
- Do NOT wrap the JSON in markdown code fences."""
|
||||
|
||||
|
||||
@@ -208,12 +208,21 @@ async def stream_chat_with_tools(
|
||||
break
|
||||
|
||||
|
||||
async def generate_completion(messages: list[dict], model: str, max_tokens: int = 4096) -> str:
|
||||
async def generate_completion(
|
||||
messages: list[dict],
|
||||
model: str,
|
||||
max_tokens: int = 4096,
|
||||
num_ctx: int | None = None,
|
||||
) -> str:
|
||||
"""Non-streaming chat completion, returns full response text.
|
||||
|
||||
Retries up to 2 times on Ollama 500 errors (cold model loading race).
|
||||
num_ctx overrides the model's context window for this call only.
|
||||
"""
|
||||
last_exc: Exception | None = None
|
||||
options: dict = {"num_predict": max_tokens}
|
||||
if num_ctx is not None:
|
||||
options["num_ctx"] = num_ctx
|
||||
for attempt in range(3):
|
||||
if attempt > 0:
|
||||
delay = 3.0 * attempt
|
||||
@@ -230,7 +239,7 @@ async def generate_completion(messages: list[dict], model: str, max_tokens: int
|
||||
"messages": messages,
|
||||
"stream": False,
|
||||
"think": False,
|
||||
"options": {"num_predict": max_tokens},
|
||||
"options": options,
|
||||
},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
"""Web research pipeline: sub-queries → SearXNG → fetch → synthesize → note."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
|
||||
import httpx
|
||||
|
||||
from fabledassistant.config import Config
|
||||
from fabledassistant.services.llm import fetch_url_content, generate_completion
|
||||
from fabledassistant.services.notes import create_note
|
||||
from fabledassistant.models.note import Note
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SEARXNG_QUERIES = 5 # sub-queries to generate
|
||||
RESULTS_PER_QUERY = 3 # results fetched from SearXNG per query
|
||||
PAGES_PER_QUERY = 3 # pages actually read per sub-query (top N results)
|
||||
MAX_SYNTHESIS_SOURCES = 12 # deduplicated sources passed to synthesis LLM
|
||||
CHARS_PER_SOURCE = 2000 # content chars per source sent to synthesis
|
||||
|
||||
|
||||
async def run_research_pipeline(
|
||||
topic: str,
|
||||
user_id: int,
|
||||
model: str,
|
||||
intent_model: str,
|
||||
buf,
|
||||
) -> Note:
|
||||
"""Full research pipeline: search → fetch → synthesize → create note.
|
||||
|
||||
Emits status events via buf.append_event throughout.
|
||||
Returns the created Note.
|
||||
"""
|
||||
# Step 1: Generate sub-queries
|
||||
buf.append_event("status", {"status": "Generating search queries..."})
|
||||
queries = await _generate_sub_queries(topic, intent_model)
|
||||
logger.info("Research: generated %d sub-queries for topic '%s'", len(queries), topic)
|
||||
|
||||
# Step 2: Search and fetch
|
||||
all_sources: list[dict] = []
|
||||
seen_urls: set[str] = set()
|
||||
|
||||
for i, query in enumerate(queries):
|
||||
if i > 0:
|
||||
await asyncio.sleep(1.0) # avoid hammering SearXNG
|
||||
buf.append_event("status", {"status": f"Searching: {query}..."})
|
||||
results = await _search_searxng(query)
|
||||
logger.info("Research: query '%s' → %d results", query, len(results))
|
||||
|
||||
for result in results[:PAGES_PER_QUERY]:
|
||||
url = result.get("url", "")
|
||||
if not url or url in seen_urls:
|
||||
continue
|
||||
seen_urls.add(url)
|
||||
title = result.get("title", url)
|
||||
buf.append_event("status", {"status": f"Reading: {title[:60]}..."})
|
||||
content = await fetch_url_content(url)
|
||||
all_sources.append({
|
||||
"url": url,
|
||||
"title": title,
|
||||
"query": query,
|
||||
"snippet": result.get("snippet", ""),
|
||||
"content": content,
|
||||
})
|
||||
|
||||
if not all_sources:
|
||||
raise ValueError(f"No results found for '{topic}'")
|
||||
|
||||
# Step 3: Filter failed fetches
|
||||
good_sources = [
|
||||
s for s in all_sources
|
||||
if not s["content"].startswith("[Failed to fetch")
|
||||
]
|
||||
|
||||
if not good_sources:
|
||||
raise ValueError(f"Could not read any sources for '{topic}'")
|
||||
|
||||
# Limit to top N sources for synthesis (already deduplicated by URL)
|
||||
synthesis_sources = good_sources[:MAX_SYNTHESIS_SOURCES]
|
||||
logger.info(
|
||||
"Research: %d/%d sources successfully fetched, using %d for synthesis",
|
||||
len(good_sources), len(all_sources), len(synthesis_sources),
|
||||
)
|
||||
|
||||
# Step 4: Synthesize
|
||||
buf.append_event("status", {"status": f"Synthesizing report from {len(synthesis_sources)} sources..."})
|
||||
title, body = await _synthesize_note(topic, synthesis_sources, model)
|
||||
|
||||
# Step 5: Create note
|
||||
buf.append_event("status", {"status": "Saving note..."})
|
||||
note = await create_note(
|
||||
user_id=user_id,
|
||||
title=title,
|
||||
body=body,
|
||||
tags=["research"],
|
||||
)
|
||||
logger.info("Research: created note id=%d title='%s'", note.id, note.title)
|
||||
return note
|
||||
|
||||
|
||||
async def _generate_sub_queries(topic: str, intent_model: str) -> list[str]:
|
||||
"""Ask the intent model for focused search queries for the topic."""
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
f"You are a research assistant. Given a research topic, generate exactly {SEARXNG_QUERIES} "
|
||||
"focused web search queries that together would provide comprehensive coverage of the topic. "
|
||||
"Vary the angle of each query: include overview, implementation details, best practices, "
|
||||
"common problems, and real-world examples. "
|
||||
"Respond with ONLY a JSON array of strings, no other text. "
|
||||
'Example: ["query one", "query two", "query three"]'
|
||||
),
|
||||
},
|
||||
{"role": "user", "content": f"Topic: {topic}"},
|
||||
]
|
||||
try:
|
||||
raw = await generate_completion(messages, intent_model, max_tokens=200)
|
||||
raw = raw.strip()
|
||||
raw = re.sub(r"^```(?:json)?\s*", "", raw)
|
||||
raw = re.sub(r"\s*```$", "", raw)
|
||||
idx = raw.find("[")
|
||||
if idx >= 0:
|
||||
parsed, _ = json.JSONDecoder().raw_decode(raw[idx:])
|
||||
if isinstance(parsed, list) and parsed:
|
||||
queries = [str(q).strip() for q in parsed if str(q).strip()]
|
||||
if queries:
|
||||
return queries[:SEARXNG_QUERIES]
|
||||
except Exception:
|
||||
logger.warning("Sub-query generation failed, falling back to topic", exc_info=True)
|
||||
return [topic]
|
||||
|
||||
|
||||
async def _search_searxng(query: str) -> list[dict]:
|
||||
"""Search SearXNG and return top results as [{url, title, snippet}]."""
|
||||
url = Config.SEARXNG_URL.rstrip("/") + "/search"
|
||||
params = {"q": query, "format": "json", "categories": "general"}
|
||||
for attempt in range(3):
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=10.0) as client:
|
||||
resp = await client.get(url, params=params)
|
||||
if resp.status_code == 429:
|
||||
retry_after = int(resp.headers.get("Retry-After", "5"))
|
||||
wait = min(retry_after, 10) * (attempt + 1)
|
||||
logger.warning(
|
||||
"SearXNG 429 for query '%s' (attempt %d/3), waiting %ds",
|
||||
query, attempt + 1, wait,
|
||||
)
|
||||
await asyncio.sleep(wait)
|
||||
continue
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
results = data.get("results", [])
|
||||
out = []
|
||||
for r in results[:RESULTS_PER_QUERY]:
|
||||
out.append({
|
||||
"url": r.get("url", ""),
|
||||
"title": r.get("title", ""),
|
||||
"snippet": r.get("content", ""),
|
||||
})
|
||||
return out
|
||||
except httpx.HTTPStatusError:
|
||||
logger.warning("SearXNG search failed for query '%s'", query, exc_info=True)
|
||||
return []
|
||||
except Exception:
|
||||
logger.warning("SearXNG search failed for query '%s'", query, exc_info=True)
|
||||
return []
|
||||
logger.warning("SearXNG search gave up after 3 attempts for query '%s'", query)
|
||||
return []
|
||||
|
||||
|
||||
async def _synthesize_note(
|
||||
topic: str,
|
||||
sources: list[dict],
|
||||
model: str,
|
||||
) -> tuple[str, str]:
|
||||
"""Synthesize a comprehensive markdown research document from fetched sources.
|
||||
|
||||
Returns (title, body_markdown).
|
||||
Uses an extended context window so the output can be several thousand words.
|
||||
"""
|
||||
sources_text_parts = []
|
||||
for i, s in enumerate(sources, 1):
|
||||
content = (s.get("content") or s.get("snippet") or "")[:CHARS_PER_SOURCE]
|
||||
sources_text_parts.append(
|
||||
f"[Source {i}] {s['title']}\nURL: {s['url']}\nSearch query: {s['query']}\n\n{content}"
|
||||
)
|
||||
sources_block = "\n\n" + ("─" * 60) + "\n\n".join(sources_text_parts)
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
"You are a thorough researcher and writer. "
|
||||
"Your task is to write an exhaustive, well-structured document on the given topic — "
|
||||
"not a brief summary or intro paragraph.\n\n"
|
||||
"Requirements:\n"
|
||||
"- Write at least 2500 words of substantive content (excluding the Sources section)\n"
|
||||
"- Choose sections (##) that make sense for the topic — let the subject matter determine the structure. "
|
||||
"A technical topic might need implementation, configuration, and troubleshooting sections. "
|
||||
"A comparison topic might need dedicated sections per subject being compared plus a summary. "
|
||||
"A scientific topic might need background, mechanisms, research findings, and implications. "
|
||||
"Use your judgment — minimum 6 major sections.\n"
|
||||
"- Use ### for subsections where they add clarity\n"
|
||||
"- Write in detailed prose paragraphs — do not reduce sections to bullet-point lists\n"
|
||||
"- Include specific details, examples, data points, comparisons, and nuance from the sources\n"
|
||||
"- Do not pad with vague generalities — every paragraph should say something concrete\n"
|
||||
"- The first line must be the document title starting with '# '\n"
|
||||
"- End with a '## Sources' section listing every source as a markdown hyperlink\n\n"
|
||||
"The reader wants to finish this document with a thorough understanding of the topic, "
|
||||
"not just an overview."
|
||||
),
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
f"Write a comprehensive reference document on: {topic}\n\n"
|
||||
f"Sources ({len(sources)} pages fetched):\n{sources_block}"
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
raw = await generate_completion(
|
||||
messages,
|
||||
model,
|
||||
max_tokens=8192,
|
||||
num_ctx=16384,
|
||||
)
|
||||
raw = raw.strip()
|
||||
|
||||
# Extract title from first # heading
|
||||
lines = raw.splitlines()
|
||||
title = f"Research: {topic}"
|
||||
body_lines = lines
|
||||
if lines and lines[0].startswith("# "):
|
||||
title = lines[0][2:].strip()
|
||||
body_lines = lines[1:]
|
||||
|
||||
body = "\n".join(body_lines).strip()
|
||||
return title, body
|
||||
@@ -19,6 +19,7 @@ from fabledassistant.services.caldav import (
|
||||
update_event,
|
||||
update_todo,
|
||||
)
|
||||
from fabledassistant.config import Config
|
||||
from fabledassistant.services.notes import create_note, delete_note, get_note_by_title, list_notes, update_note
|
||||
from fabledassistant.services.tag_suggestions import suggest_tags
|
||||
|
||||
@@ -663,11 +664,61 @@ _CALDAV_TOOLS = [
|
||||
]
|
||||
|
||||
|
||||
_SEARCH_TOOLS = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "search_web",
|
||||
"description": (
|
||||
"Search the web for quick information and answer the user's question from results. "
|
||||
"Use for factual lookups, current events, version numbers, quick definitions, or any question "
|
||||
"where a fast web answer is needed without creating a note. "
|
||||
"Use research_topic instead when the user explicitly wants a comprehensive note or deep research."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string", "description": "The search query"}
|
||||
},
|
||||
"required": ["query"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
_RESEARCH_TOOLS = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "research_topic",
|
||||
"description": (
|
||||
"Research a topic by searching the web and compiling a note with cited sources. "
|
||||
"Use for 'research X', 'look up X', 'find information about X', "
|
||||
"'compile notes on X'. Takes 30–120 seconds."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"topic": {
|
||||
"type": "string",
|
||||
"description": "The topic or question to research",
|
||||
}
|
||||
},
|
||||
"required": ["topic"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
async def get_tools_for_user(user_id: int) -> list[dict]:
|
||||
"""Build the tool list for a user based on their configured integrations."""
|
||||
tools = list(_CORE_TOOLS)
|
||||
if await is_caldav_configured(user_id):
|
||||
tools.extend(_CALDAV_TOOLS)
|
||||
if Config.searxng_enabled():
|
||||
tools.extend(_SEARCH_TOOLS)
|
||||
tools.extend(_RESEARCH_TOOLS)
|
||||
logger.debug("User %d: %d tools available", user_id, len(tools))
|
||||
return tools
|
||||
|
||||
@@ -1128,6 +1179,32 @@ async def execute_tool(user_id: int, tool_name: str, arguments: dict) -> dict:
|
||||
"data": {"id": note.id, "title": note.title},
|
||||
}
|
||||
|
||||
elif tool_name == "search_web":
|
||||
from fabledassistant.services.research import _search_searxng
|
||||
query = arguments.get("query", "")
|
||||
results = await _search_searxng(query)
|
||||
if not results:
|
||||
return {"success": False, "error": f"No results found for '{query}'"}
|
||||
return {
|
||||
"success": True,
|
||||
"type": "web_search",
|
||||
"data": {
|
||||
"query": query,
|
||||
"results": results,
|
||||
"count": len(results),
|
||||
},
|
||||
}
|
||||
|
||||
elif tool_name == "research_topic":
|
||||
# Research is always handled upstream in generation_task.py (round 0).
|
||||
# This fallback exists in case it somehow reaches execute_tool.
|
||||
topic = arguments.get("topic", "")
|
||||
return {
|
||||
"success": True,
|
||||
"type": "research_pending",
|
||||
"data": {"topic": topic},
|
||||
}
|
||||
|
||||
else:
|
||||
return {"success": False, "error": f"Unknown tool: {tool_name}"}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user