- Connect to a CalDAV server (Nextcloud, iCloud, Radicale, Baikal) to create and view calendar events from chat.
-
-
-
-
- {{ caldavTestResult.message }}
-
- Calendars: {{ caldavTestResult.calendars.join(", ") }}
+
+
+ Assistant
+
+
+
+
+
The name used in chat messages and LLM context.
+
+
+
+
Model used for new conversations.
+
+
+
+
+
Smaller/faster model for intent routing before the main model.
+
+
+
+ Saved!
+
+
+
+
+
+ Email Address
+ Used for password resets and notifications.
+
+
+
+
+
+
+
+
Required to confirm the change.
+
+
+
+
+
+
+
+ Change Password
+
+
+
+
+
+
+
+
Must be at least 8 characters
+
+
+
+
+ Passwords do not match
+
+
+
+
+
+
+
+
+ Notifications
+
+ Email notifications when SMTP is configured by an admin.
+
+
+
+
Daily email for tasks due or overdue.
+
+
+
Emails for logins, logouts, and password changes.
+
+
+ Saved!
+
+
+
+
+
+
+
+
+ Calendar (CalDAV)
+
+ Connect to a CalDAV server (Nextcloud, iCloud, Radicale, Baikal) to create and view calendar events from chat.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Optional. Exact calendar name to use.
+
+
+
+ Saved!
+
+
+
+
+ {{ caldavTestResult.message }}
+
+ Calendars: {{ caldavTestResult.calendars.join(", ") }}
+
+
+
{{ caldavTestResult.error }}
+
+
+
+
+ Web Search (SearXNG)
+
+
+ Connected to {{ searxngUrl }}.
+ Test a query below to verify results and rate limiting.
+
+
+
+
+
+ {{ searchError }}
+
+ -
+
+
{{ r.snippet }}
+
+
- {{ caldavTestResult.error }}
+
+ Not configured. Set SEARXNG_URL in docker-compose to enable web research from chat.
+
- Application URL
-
- The public URL used in email links (invitations, password resets). Example: https://notes.example.com
-
-
-
-
-
-
-
- Saved!
-
-
-
-
- Email / SMTP
-
- Configure SMTP settings to enable email notifications for all users.
-
-
-
-
-
-
-
Enable STARTTLS encryption (recommended for port 587). Implicit TLS is used automatically for port 465.
-
-
- Saved!
-
-
-
-
Test Email
-
-
-
-
- Data
- Export your data or restore from a backup.
-
-
-
-
-
-
+
Test Email
+
+
+
+
+
+
+
diff --git a/src/fabledassistant/config.py b/src/fabledassistant/config.py
index 34e5210..909be5f 100644
--- a/src/fabledassistant/config.py
+++ b/src/fabledassistant/config.py
@@ -56,6 +56,13 @@ class Config:
OIDC_SCOPES: str = os.environ.get("OIDC_SCOPES", "openid profile email")
LOCAL_AUTH_ENABLED: bool = os.environ.get("LOCAL_AUTH_ENABLED", "true").lower() not in ("0", "false", "no")
+ # SearXNG web search (external instance)
+ SEARXNG_URL: str = os.environ.get("SEARXNG_URL", "")
+
@classmethod
def oidc_enabled(cls) -> bool:
return bool(cls.OIDC_ISSUER and cls.OIDC_CLIENT_ID and cls.OIDC_CLIENT_SECRET)
+
+ @classmethod
+ def searxng_enabled(cls) -> bool:
+ return bool(cls.SEARXNG_URL)
diff --git a/src/fabledassistant/routes/settings.py b/src/fabledassistant/routes/settings.py
index 8070aaf..e555000 100644
--- a/src/fabledassistant/routes/settings.py
+++ b/src/fabledassistant/routes/settings.py
@@ -84,3 +84,17 @@ async def test_caldav():
uid = get_current_user_id()
result = await test_connection(uid)
return jsonify(result)
+
+
+@settings_bp.route("/search", methods=["GET"])
+@login_required
+async def test_search():
+ """Test SearXNG connectivity and preview results for a query."""
+ if not Config.searxng_enabled():
+ return jsonify({"configured": False, "results": [], "searxng_url": ""})
+ q = request.args.get("q", "").strip()
+ if not q:
+ return jsonify({"configured": True, "results": [], "searxng_url": Config.SEARXNG_URL})
+ from fabledassistant.services.research import _search_searxng
+ results = await _search_searxng(q)
+ return jsonify({"configured": True, "results": results, "query": q, "searxng_url": Config.SEARXNG_URL})
diff --git a/src/fabledassistant/services/generation_task.py b/src/fabledassistant/services/generation_task.py
index fa22cac..b688541 100644
--- a/src/fabledassistant/services/generation_task.py
+++ b/src/fabledassistant/services/generation_task.py
@@ -28,6 +28,7 @@ from fabledassistant.services.intent import IntentResult, classify_intent
from fabledassistant.services.logging import log_generation
from fabledassistant.services.settings import get_setting
from fabledassistant.services.tools import get_tools_for_user, execute_tool
+from fabledassistant.services.research import run_research_pipeline
logger = logging.getLogger(__name__)
@@ -59,6 +60,8 @@ _TOOL_LABELS: dict[str, str] = {
"update_todo": "Updating todo",
"complete_todo": "Completing todo",
"delete_todo": "Removing todo",
+ "search_web": "Searching the web",
+ "research_topic": "Researching topic",
}
# Tools that write data and require explicit user confirmation before executing.
@@ -285,6 +288,43 @@ async def run_generation(
if timing["ttft_ms"] is None:
timing["ttft_ms"] = int((time.monotonic() - t_start) * 1000)
+ if tool_name == "research_topic":
+ topic = intent.arguments.get("topic", "")
+ if not ack_text:
+ fallback_ack = f"I'll research '{topic}' and compile a note.\n\n"
+ buf.append_event("chunk", {"chunk": fallback_ack})
+ buf.content_so_far += fallback_ack
+ if timing["ttft_ms"] is None:
+ timing["ttft_ms"] = int((time.monotonic() - t_start) * 1000)
+ try:
+ note = await run_research_pipeline(
+ topic, user_id, model, intent_model, buf
+ )
+ done_text = (
+ f"Research complete! I've compiled a note: "
+ f"**[{note.title}](/notes/{note.id})**."
+ )
+ buf.append_event("chunk", {"chunk": done_text})
+ buf.content_so_far += done_text
+ tool_record = {
+ "function": "research_topic",
+ "arguments": {"topic": topic},
+ "result": {
+ "success": True,
+ "type": "research_note",
+ "data": {"id": note.id, "title": note.title},
+ },
+ "status": "success",
+ }
+ all_tool_calls.append(tool_record)
+ buf.append_event("tool_call", {"tool_call": tool_record})
+ except Exception as e:
+ logger.exception("Research pipeline failed for topic: %s", topic)
+ err_text = f"\nResearch failed: {e}"
+ buf.append_event("chunk", {"chunk": err_text})
+ buf.content_so_far += err_text
+ break # research IS the full response
+
confirmed = True
if tool_name in _WRITE_TOOLS:
loop = asyncio.get_running_loop()
diff --git a/src/fabledassistant/services/intent.py b/src/fabledassistant/services/intent.py
index abc5efd..f7f2484 100644
--- a/src/fabledassistant/services/intent.py
+++ b/src/fabledassistant/services/intent.py
@@ -100,6 +100,12 @@ Rules:
- "read", "open", "show me", "what does X say", "display", "pull up" a specific note → use get_note with query=
.
- "list my notes", "show notes", "recent notes", "browse notes", "notes tagged X" → use list_notes (with optional q or tags).
- "tag X with Y", "add tag Y to X", "untag Y from X", "remove tag Y from X" → use update_note with tags=[Y] and tag_mode="add" or "remove".
+- search_web: user wants a quick web search to answer a factual question
+ ("search for X", "look up X", "what is the latest version of X", "find X online",
+ "google X", "what is X" for quick factual answers — NOT when they want a comprehensive note)
+- research_topic: user wants to research a topic and create a comprehensive note from web sources
+ ("research X", "research X and make a note", "compile notes on X", "write a report on X",
+ "deep dive into X", "find everything about X", "comprehensive guide to X")
- "ack": one short, natural sentence confirming the action (tool path only). Vary phrasing — do not always start with "Let me". Omit (null) for chat-only responses.
- Do NOT wrap the JSON in markdown code fences."""
diff --git a/src/fabledassistant/services/llm.py b/src/fabledassistant/services/llm.py
index 045494d..8a04e79 100644
--- a/src/fabledassistant/services/llm.py
+++ b/src/fabledassistant/services/llm.py
@@ -208,12 +208,21 @@ async def stream_chat_with_tools(
break
-async def generate_completion(messages: list[dict], model: str, max_tokens: int = 4096) -> str:
+async def generate_completion(
+ messages: list[dict],
+ model: str,
+ max_tokens: int = 4096,
+ num_ctx: int | None = None,
+) -> str:
"""Non-streaming chat completion, returns full response text.
Retries up to 2 times on Ollama 500 errors (cold model loading race).
+ num_ctx overrides the model's context window for this call only.
"""
last_exc: Exception | None = None
+ options: dict = {"num_predict": max_tokens}
+ if num_ctx is not None:
+ options["num_ctx"] = num_ctx
for attempt in range(3):
if attempt > 0:
delay = 3.0 * attempt
@@ -230,7 +239,7 @@ async def generate_completion(messages: list[dict], model: str, max_tokens: int
"messages": messages,
"stream": False,
"think": False,
- "options": {"num_predict": max_tokens},
+ "options": options,
},
)
resp.raise_for_status()
diff --git a/src/fabledassistant/services/research.py b/src/fabledassistant/services/research.py
new file mode 100644
index 0000000..16e8e83
--- /dev/null
+++ b/src/fabledassistant/services/research.py
@@ -0,0 +1,242 @@
+"""Web research pipeline: sub-queries → SearXNG → fetch → synthesize → note."""
+
+import asyncio
+import json
+import logging
+import re
+
+import httpx
+
+from fabledassistant.config import Config
+from fabledassistant.services.llm import fetch_url_content, generate_completion
+from fabledassistant.services.notes import create_note
+from fabledassistant.models.note import Note
+
+logger = logging.getLogger(__name__)
+
+SEARXNG_QUERIES = 5 # sub-queries to generate
+RESULTS_PER_QUERY = 3 # results fetched from SearXNG per query
+PAGES_PER_QUERY = 3 # pages actually read per sub-query (top N results)
+MAX_SYNTHESIS_SOURCES = 12 # deduplicated sources passed to synthesis LLM
+CHARS_PER_SOURCE = 2000 # content chars per source sent to synthesis
+
+
+async def run_research_pipeline(
+ topic: str,
+ user_id: int,
+ model: str,
+ intent_model: str,
+ buf,
+) -> Note:
+ """Full research pipeline: search → fetch → synthesize → create note.
+
+ Emits status events via buf.append_event throughout.
+ Returns the created Note.
+ """
+ # Step 1: Generate sub-queries
+ buf.append_event("status", {"status": "Generating search queries..."})
+ queries = await _generate_sub_queries(topic, intent_model)
+ logger.info("Research: generated %d sub-queries for topic '%s'", len(queries), topic)
+
+ # Step 2: Search and fetch
+ all_sources: list[dict] = []
+ seen_urls: set[str] = set()
+
+ for i, query in enumerate(queries):
+ if i > 0:
+ await asyncio.sleep(1.0) # avoid hammering SearXNG
+ buf.append_event("status", {"status": f"Searching: {query}..."})
+ results = await _search_searxng(query)
+ logger.info("Research: query '%s' → %d results", query, len(results))
+
+ for result in results[:PAGES_PER_QUERY]:
+ url = result.get("url", "")
+ if not url or url in seen_urls:
+ continue
+ seen_urls.add(url)
+ title = result.get("title", url)
+ buf.append_event("status", {"status": f"Reading: {title[:60]}..."})
+ content = await fetch_url_content(url)
+ all_sources.append({
+ "url": url,
+ "title": title,
+ "query": query,
+ "snippet": result.get("snippet", ""),
+ "content": content,
+ })
+
+ if not all_sources:
+ raise ValueError(f"No results found for '{topic}'")
+
+ # Step 3: Filter failed fetches
+ good_sources = [
+ s for s in all_sources
+ if not s["content"].startswith("[Failed to fetch")
+ ]
+
+ if not good_sources:
+ raise ValueError(f"Could not read any sources for '{topic}'")
+
+ # Limit to top N sources for synthesis (already deduplicated by URL)
+ synthesis_sources = good_sources[:MAX_SYNTHESIS_SOURCES]
+ logger.info(
+ "Research: %d/%d sources successfully fetched, using %d for synthesis",
+ len(good_sources), len(all_sources), len(synthesis_sources),
+ )
+
+ # Step 4: Synthesize
+ buf.append_event("status", {"status": f"Synthesizing report from {len(synthesis_sources)} sources..."})
+ title, body = await _synthesize_note(topic, synthesis_sources, model)
+
+ # Step 5: Create note
+ buf.append_event("status", {"status": "Saving note..."})
+ note = await create_note(
+ user_id=user_id,
+ title=title,
+ body=body,
+ tags=["research"],
+ )
+ logger.info("Research: created note id=%d title='%s'", note.id, note.title)
+ return note
+
+
+async def _generate_sub_queries(topic: str, intent_model: str) -> list[str]:
+ """Ask the intent model for focused search queries for the topic."""
+ messages = [
+ {
+ "role": "system",
+ "content": (
+ f"You are a research assistant. Given a research topic, generate exactly {SEARXNG_QUERIES} "
+ "focused web search queries that together would provide comprehensive coverage of the topic. "
+ "Vary the angle of each query: include overview, implementation details, best practices, "
+ "common problems, and real-world examples. "
+ "Respond with ONLY a JSON array of strings, no other text. "
+ 'Example: ["query one", "query two", "query three"]'
+ ),
+ },
+ {"role": "user", "content": f"Topic: {topic}"},
+ ]
+ try:
+ raw = await generate_completion(messages, intent_model, max_tokens=200)
+ raw = raw.strip()
+ raw = re.sub(r"^```(?:json)?\s*", "", raw)
+ raw = re.sub(r"\s*```$", "", raw)
+ idx = raw.find("[")
+ if idx >= 0:
+ parsed, _ = json.JSONDecoder().raw_decode(raw[idx:])
+ if isinstance(parsed, list) and parsed:
+ queries = [str(q).strip() for q in parsed if str(q).strip()]
+ if queries:
+ return queries[:SEARXNG_QUERIES]
+ except Exception:
+ logger.warning("Sub-query generation failed, falling back to topic", exc_info=True)
+ return [topic]
+
+
+async def _search_searxng(query: str) -> list[dict]:
+ """Search SearXNG and return top results as [{url, title, snippet}]."""
+ url = Config.SEARXNG_URL.rstrip("/") + "/search"
+ params = {"q": query, "format": "json", "categories": "general"}
+ for attempt in range(3):
+ try:
+ async with httpx.AsyncClient(timeout=10.0) as client:
+ resp = await client.get(url, params=params)
+ if resp.status_code == 429:
+ retry_after = int(resp.headers.get("Retry-After", "5"))
+ wait = min(retry_after, 10) * (attempt + 1)
+ logger.warning(
+ "SearXNG 429 for query '%s' (attempt %d/3), waiting %ds",
+ query, attempt + 1, wait,
+ )
+ await asyncio.sleep(wait)
+ continue
+ resp.raise_for_status()
+ data = resp.json()
+ results = data.get("results", [])
+ out = []
+ for r in results[:RESULTS_PER_QUERY]:
+ out.append({
+ "url": r.get("url", ""),
+ "title": r.get("title", ""),
+ "snippet": r.get("content", ""),
+ })
+ return out
+ except httpx.HTTPStatusError:
+ logger.warning("SearXNG search failed for query '%s'", query, exc_info=True)
+ return []
+ except Exception:
+ logger.warning("SearXNG search failed for query '%s'", query, exc_info=True)
+ return []
+ logger.warning("SearXNG search gave up after 3 attempts for query '%s'", query)
+ return []
+
+
+async def _synthesize_note(
+ topic: str,
+ sources: list[dict],
+ model: str,
+) -> tuple[str, str]:
+ """Synthesize a comprehensive markdown research document from fetched sources.
+
+ Returns (title, body_markdown).
+ Uses an extended context window so the output can be several thousand words.
+ """
+ sources_text_parts = []
+ for i, s in enumerate(sources, 1):
+ content = (s.get("content") or s.get("snippet") or "")[:CHARS_PER_SOURCE]
+ sources_text_parts.append(
+ f"[Source {i}] {s['title']}\nURL: {s['url']}\nSearch query: {s['query']}\n\n{content}"
+ )
+ sources_block = "\n\n" + ("─" * 60) + "\n\n".join(sources_text_parts)
+
+ messages = [
+ {
+ "role": "system",
+ "content": (
+ "You are a thorough researcher and writer. "
+ "Your task is to write an exhaustive, well-structured document on the given topic — "
+ "not a brief summary or intro paragraph.\n\n"
+ "Requirements:\n"
+ "- Write at least 2500 words of substantive content (excluding the Sources section)\n"
+ "- Choose sections (##) that make sense for the topic — let the subject matter determine the structure. "
+ "A technical topic might need implementation, configuration, and troubleshooting sections. "
+ "A comparison topic might need dedicated sections per subject being compared plus a summary. "
+ "A scientific topic might need background, mechanisms, research findings, and implications. "
+ "Use your judgment — minimum 6 major sections.\n"
+ "- Use ### for subsections where they add clarity\n"
+ "- Write in detailed prose paragraphs — do not reduce sections to bullet-point lists\n"
+ "- Include specific details, examples, data points, comparisons, and nuance from the sources\n"
+ "- Do not pad with vague generalities — every paragraph should say something concrete\n"
+ "- The first line must be the document title starting with '# '\n"
+ "- End with a '## Sources' section listing every source as a markdown hyperlink\n\n"
+ "The reader wants to finish this document with a thorough understanding of the topic, "
+ "not just an overview."
+ ),
+ },
+ {
+ "role": "user",
+ "content": (
+ f"Write a comprehensive reference document on: {topic}\n\n"
+ f"Sources ({len(sources)} pages fetched):\n{sources_block}"
+ ),
+ },
+ ]
+
+ raw = await generate_completion(
+ messages,
+ model,
+ max_tokens=8192,
+ num_ctx=16384,
+ )
+ raw = raw.strip()
+
+ # Extract title from first # heading
+ lines = raw.splitlines()
+ title = f"Research: {topic}"
+ body_lines = lines
+ if lines and lines[0].startswith("# "):
+ title = lines[0][2:].strip()
+ body_lines = lines[1:]
+
+ body = "\n".join(body_lines).strip()
+ return title, body
diff --git a/src/fabledassistant/services/tools.py b/src/fabledassistant/services/tools.py
index b530435..0810239 100644
--- a/src/fabledassistant/services/tools.py
+++ b/src/fabledassistant/services/tools.py
@@ -19,6 +19,7 @@ from fabledassistant.services.caldav import (
update_event,
update_todo,
)
+from fabledassistant.config import Config
from fabledassistant.services.notes import create_note, delete_note, get_note_by_title, list_notes, update_note
from fabledassistant.services.tag_suggestions import suggest_tags
@@ -663,11 +664,61 @@ _CALDAV_TOOLS = [
]
+_SEARCH_TOOLS = [
+ {
+ "type": "function",
+ "function": {
+ "name": "search_web",
+ "description": (
+ "Search the web for quick information and answer the user's question from results. "
+ "Use for factual lookups, current events, version numbers, quick definitions, or any question "
+ "where a fast web answer is needed without creating a note. "
+ "Use research_topic instead when the user explicitly wants a comprehensive note or deep research."
+ ),
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "query": {"type": "string", "description": "The search query"}
+ },
+ "required": ["query"],
+ },
+ },
+ }
+]
+
+_RESEARCH_TOOLS = [
+ {
+ "type": "function",
+ "function": {
+ "name": "research_topic",
+ "description": (
+ "Research a topic by searching the web and compiling a note with cited sources. "
+ "Use for 'research X', 'look up X', 'find information about X', "
+ "'compile notes on X'. Takes 30–120 seconds."
+ ),
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "topic": {
+ "type": "string",
+ "description": "The topic or question to research",
+ }
+ },
+ "required": ["topic"],
+ },
+ },
+ }
+]
+
+
async def get_tools_for_user(user_id: int) -> list[dict]:
"""Build the tool list for a user based on their configured integrations."""
tools = list(_CORE_TOOLS)
if await is_caldav_configured(user_id):
tools.extend(_CALDAV_TOOLS)
+ if Config.searxng_enabled():
+ tools.extend(_SEARCH_TOOLS)
+ tools.extend(_RESEARCH_TOOLS)
logger.debug("User %d: %d tools available", user_id, len(tools))
return tools
@@ -1128,6 +1179,32 @@ async def execute_tool(user_id: int, tool_name: str, arguments: dict) -> dict:
"data": {"id": note.id, "title": note.title},
}
+ elif tool_name == "search_web":
+ from fabledassistant.services.research import _search_searxng
+ query = arguments.get("query", "")
+ results = await _search_searxng(query)
+ if not results:
+ return {"success": False, "error": f"No results found for '{query}'"}
+ return {
+ "success": True,
+ "type": "web_search",
+ "data": {
+ "query": query,
+ "results": results,
+ "count": len(results),
+ },
+ }
+
+ elif tool_name == "research_topic":
+ # Research is always handled upstream in generation_task.py (round 0).
+ # This fallback exists in case it somehow reaches execute_tool.
+ topic = arguments.get("topic", "")
+ return {
+ "success": True,
+ "type": "research_pending",
+ "data": {"topic": topic},
+ }
+
else:
return {"success": False, "error": f"Unknown tool: {tool_name}"}