Phase 22: SearXNG web research pipeline + settings layout overhaul
Research pipeline (research_topic tool): - New service: services/research.py — sub-query generation, SearXNG search, URL fetch, deduplication, and LLM synthesis into a note - 5 sub-queries × 3 pages = up to 15 sources, capped at 12 for synthesis - Synthesis uses num_ctx=16384 + max_tokens=8192 for long-form output - Prompt demands 2500+ words, 6+ topic-appropriate sections, detailed prose - 429 retry with backoff; 1s inter-query sleep; raw_decode JSON parsing search_web tool (new): - Lightweight single-query SearXNG search, results returned inline in chat - LLM answers conversationally in round 1; no note created - web_search result type with external links in ToolCallCard Infrastructure: - llm.py: generate_completion accepts num_ctx override - config.py: SEARXNG_URL + Config.searxng_enabled() - docker-compose: OLLAMA_NUM_PARALLEL=2, commented SEARXNG_URL example - intent.py: search_web and research_topic routing rules Settings UI: - 2-column grid layout (small sections pair up, complex span full width) - Search Test section: live SearXNG query with result preview - GET /api/settings/search?q= proxy endpoint - Research button (magnifier) in ChatView input toolbar → popover modal Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -13,6 +13,8 @@ services:
|
|||||||
OLLAMA_URL: "http://ollama:11434"
|
OLLAMA_URL: "http://ollama:11434"
|
||||||
OLLAMA_MODEL: "${OLLAMA_MODEL:-llama3.1}"
|
OLLAMA_MODEL: "${OLLAMA_MODEL:-llama3.1}"
|
||||||
SECRET_KEY: "${SECRET_KEY:-dev-secret-change-me}"
|
SECRET_KEY: "${SECRET_KEY:-dev-secret-change-me}"
|
||||||
|
# Uncomment and set to enable web research via SearXNG:
|
||||||
|
# SEARXNG_URL: "http://searxng:8080"
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:5000/api/health')"]
|
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:5000/api/health')"]
|
||||||
interval: 10s
|
interval: 10s
|
||||||
@@ -40,6 +42,7 @@ services:
|
|||||||
- ollama_models:/root/.ollama
|
- ollama_models:/root/.ollama
|
||||||
environment:
|
environment:
|
||||||
OLLAMA_MAX_LOADED_MODELS: "2"
|
OLLAMA_MAX_LOADED_MODELS: "2"
|
||||||
|
OLLAMA_NUM_PARALLEL: "2"
|
||||||
OLLAMA_KEEP_ALIVE: "30m"
|
OLLAMA_KEEP_ALIVE: "30m"
|
||||||
OLLAMA_FLASH_ATTENTION: "1"
|
OLLAMA_FLASH_ATTENTION: "1"
|
||||||
deploy:
|
deploy:
|
||||||
|
|||||||
@@ -52,6 +52,12 @@ const label = computed(() => {
|
|||||||
return "Completed todo";
|
return "Completed todo";
|
||||||
case "todo_deleted":
|
case "todo_deleted":
|
||||||
return "Deleted todo";
|
return "Deleted todo";
|
||||||
|
case "web_search":
|
||||||
|
return "Web search";
|
||||||
|
case "research_pending":
|
||||||
|
return "Research started";
|
||||||
|
case "research_note":
|
||||||
|
return "Research note";
|
||||||
default:
|
default:
|
||||||
return "Tool call";
|
return "Tool call";
|
||||||
}
|
}
|
||||||
@@ -167,6 +173,12 @@ const taskCount = computed(() => {
|
|||||||
return (data.total as number) ?? 0;
|
return (data.total as number) ?? 0;
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const webSearch = computed(() => {
|
||||||
|
const data = props.toolCall.result.data;
|
||||||
|
if (!data || props.toolCall.result.type !== "web_search") return null;
|
||||||
|
return data as { query: string; results: { url: string; title: string; snippet: string }[]; count: number };
|
||||||
|
});
|
||||||
|
|
||||||
const searchResults = computed(() => {
|
const searchResults = computed(() => {
|
||||||
const data = props.toolCall.result.data;
|
const data = props.toolCall.result.data;
|
||||||
if (!data || props.toolCall.result.type !== "search") return null;
|
if (!data || props.toolCall.result.type !== "search") return null;
|
||||||
@@ -317,6 +329,19 @@ async function applyTag(tag: string) {
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
|
<template v-else-if="webSearch">
|
||||||
|
<span class="tool-search-info">{{ webSearch.count }} result{{ webSearch.count !== 1 ? "s" : "" }}</span>
|
||||||
|
<div class="tool-search-results web-search-results">
|
||||||
|
<a
|
||||||
|
v-for="(r, i) in webSearch.results"
|
||||||
|
:key="i"
|
||||||
|
:href="r.url"
|
||||||
|
target="_blank"
|
||||||
|
rel="noopener noreferrer"
|
||||||
|
class="tool-search-item"
|
||||||
|
>{{ r.title || r.url }}</a>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
<template v-else-if="linkTo">
|
<template v-else-if="linkTo">
|
||||||
<router-link :to="linkTo" class="tool-link">{{ title }}</router-link>
|
<router-link :to="linkTo" class="tool-link">{{ title }}</router-link>
|
||||||
</template>
|
</template>
|
||||||
@@ -400,6 +425,13 @@ async function applyTag(tag: string) {
|
|||||||
color: var(--color-text-muted);
|
color: var(--color-text-muted);
|
||||||
margin-right: 0.1rem;
|
margin-right: 0.1rem;
|
||||||
}
|
}
|
||||||
|
.web-search-results {
|
||||||
|
flex-direction: column;
|
||||||
|
flex-wrap: nowrap;
|
||||||
|
}
|
||||||
|
.web-search-results .tool-search-item::after {
|
||||||
|
content: none;
|
||||||
|
}
|
||||||
.tool-event-title {
|
.tool-event-title {
|
||||||
font-weight: 600;
|
font-weight: 600;
|
||||||
color: var(--color-text);
|
color: var(--color-text);
|
||||||
|
|||||||
@@ -22,6 +22,10 @@ const sending = ref(false);
|
|||||||
const summarizing = ref(false);
|
const summarizing = ref(false);
|
||||||
const sidebarOpen = ref(false);
|
const sidebarOpen = ref(false);
|
||||||
|
|
||||||
|
// Research modal state
|
||||||
|
const showResearchModal = ref(false);
|
||||||
|
const researchTopic = ref("");
|
||||||
|
|
||||||
// Note picker state
|
// Note picker state
|
||||||
const attachedNote = ref<{ id: number; title: string } | null>(null);
|
const attachedNote = ref<{ id: number; title: string } | null>(null);
|
||||||
const showNotePicker = ref(false);
|
const showNotePicker = ref(false);
|
||||||
@@ -239,6 +243,27 @@ function onInputKeydown(e: KeyboardEvent) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Research modal
|
||||||
|
function toggleResearchModal() {
|
||||||
|
showResearchModal.value = !showResearchModal.value;
|
||||||
|
if (showResearchModal.value) {
|
||||||
|
researchTopic.value = "";
|
||||||
|
nextTick(() => {
|
||||||
|
const el = document.querySelector(".research-topic-input") as HTMLInputElement;
|
||||||
|
el?.focus();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function submitResearch() {
|
||||||
|
const topic = researchTopic.value.trim();
|
||||||
|
if (!topic) return;
|
||||||
|
messageInput.value = `Research: ${topic}`;
|
||||||
|
showResearchModal.value = false;
|
||||||
|
researchTopic.value = "";
|
||||||
|
sendMessage();
|
||||||
|
}
|
||||||
|
|
||||||
// Note picker
|
// Note picker
|
||||||
function toggleNotePicker() {
|
function toggleNotePicker() {
|
||||||
showNotePicker.value = !showNotePicker.value;
|
showNotePicker.value = !showNotePicker.value;
|
||||||
@@ -302,7 +327,12 @@ function removeIncludedNote(noteId: number) {
|
|||||||
// Keyboard shortcuts
|
// Keyboard shortcuts
|
||||||
function onGlobalKeydown(e: KeyboardEvent) {
|
function onGlobalKeydown(e: KeyboardEvent) {
|
||||||
if (e.key !== "Escape") return;
|
if (e.key !== "Escape") return;
|
||||||
// Close note picker first
|
// Close research modal first
|
||||||
|
if (showResearchModal.value) {
|
||||||
|
showResearchModal.value = false;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Close note picker
|
||||||
if (showNotePicker.value) {
|
if (showNotePicker.value) {
|
||||||
showNotePicker.value = false;
|
showNotePicker.value = false;
|
||||||
return;
|
return;
|
||||||
@@ -479,6 +509,37 @@ onUnmounted(() => {
|
|||||||
|
|
||||||
<div class="input-wrapper">
|
<div class="input-wrapper">
|
||||||
<div class="input-area">
|
<div class="input-area">
|
||||||
|
<!-- Research button -->
|
||||||
|
<div class="research-wrapper">
|
||||||
|
<button
|
||||||
|
class="btn-attach"
|
||||||
|
@click="toggleResearchModal"
|
||||||
|
:disabled="store.streaming || !store.chatReady"
|
||||||
|
title="Research a topic"
|
||||||
|
>
|
||||||
|
<svg width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||||||
|
<circle cx="11" cy="11" r="8"/>
|
||||||
|
<line x1="21" y1="21" x2="16.65" y2="16.65"/>
|
||||||
|
</svg>
|
||||||
|
</button>
|
||||||
|
|
||||||
|
<!-- Research modal -->
|
||||||
|
<div v-if="showResearchModal" class="research-modal">
|
||||||
|
<div class="research-modal-header">Research topic</div>
|
||||||
|
<input
|
||||||
|
class="research-topic-input"
|
||||||
|
v-model="researchTopic"
|
||||||
|
placeholder="e.g. quantum computing"
|
||||||
|
@keydown.enter="submitResearch"
|
||||||
|
@keydown.escape="showResearchModal = false"
|
||||||
|
/>
|
||||||
|
<div class="research-modal-actions">
|
||||||
|
<button class="btn-research-cancel" @click="showResearchModal = false">Cancel</button>
|
||||||
|
<button class="btn-research-go" @click="submitResearch" :disabled="!researchTopic.trim()">Go</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<!-- Note picker button -->
|
<!-- Note picker button -->
|
||||||
<div class="note-picker-wrapper">
|
<div class="note-picker-wrapper">
|
||||||
<button
|
<button
|
||||||
@@ -1005,6 +1066,84 @@ details[open] .thinking-summary::before {
|
|||||||
color: var(--color-text-muted);
|
color: var(--color-text-muted);
|
||||||
font-size: 0.85rem;
|
font-size: 0.85rem;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Research modal */
|
||||||
|
.research-wrapper {
|
||||||
|
position: relative;
|
||||||
|
}
|
||||||
|
.research-modal {
|
||||||
|
position: absolute;
|
||||||
|
bottom: calc(100% + 8px);
|
||||||
|
left: 0;
|
||||||
|
width: 280px;
|
||||||
|
background: var(--color-bg-card);
|
||||||
|
border: 1px solid var(--color-border);
|
||||||
|
border-radius: var(--radius-md);
|
||||||
|
box-shadow: 0 4px 16px var(--color-shadow);
|
||||||
|
z-index: 10;
|
||||||
|
padding: 0.75rem;
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 0.5rem;
|
||||||
|
}
|
||||||
|
.research-modal-header {
|
||||||
|
font-size: 0.8rem;
|
||||||
|
font-weight: 600;
|
||||||
|
color: var(--color-text-muted);
|
||||||
|
text-transform: uppercase;
|
||||||
|
letter-spacing: 0.04em;
|
||||||
|
}
|
||||||
|
.research-topic-input {
|
||||||
|
width: 100%;
|
||||||
|
padding: 0.4rem 0.6rem;
|
||||||
|
border: 1px solid var(--color-border);
|
||||||
|
border-radius: var(--radius-sm);
|
||||||
|
background: var(--color-bg-secondary);
|
||||||
|
color: var(--color-text);
|
||||||
|
font-size: 0.9rem;
|
||||||
|
outline: none;
|
||||||
|
font-family: inherit;
|
||||||
|
box-sizing: border-box;
|
||||||
|
}
|
||||||
|
.research-topic-input:focus {
|
||||||
|
border-color: var(--color-primary);
|
||||||
|
}
|
||||||
|
.research-modal-actions {
|
||||||
|
display: flex;
|
||||||
|
justify-content: flex-end;
|
||||||
|
gap: 0.4rem;
|
||||||
|
}
|
||||||
|
.btn-research-cancel {
|
||||||
|
padding: 0.3rem 0.65rem;
|
||||||
|
background: none;
|
||||||
|
border: 1px solid var(--color-border);
|
||||||
|
border-radius: var(--radius-sm);
|
||||||
|
color: var(--color-text-muted);
|
||||||
|
font-size: 0.85rem;
|
||||||
|
cursor: pointer;
|
||||||
|
font-family: inherit;
|
||||||
|
}
|
||||||
|
.btn-research-cancel:hover {
|
||||||
|
background: var(--color-bg-secondary);
|
||||||
|
}
|
||||||
|
.btn-research-go {
|
||||||
|
padding: 0.3rem 0.75rem;
|
||||||
|
background: var(--color-primary);
|
||||||
|
border: none;
|
||||||
|
border-radius: var(--radius-sm);
|
||||||
|
color: #fff;
|
||||||
|
font-size: 0.85rem;
|
||||||
|
cursor: pointer;
|
||||||
|
font-family: inherit;
|
||||||
|
}
|
||||||
|
.btn-research-go:disabled {
|
||||||
|
opacity: 0.4;
|
||||||
|
cursor: default;
|
||||||
|
}
|
||||||
|
.btn-research-go:not(:disabled):hover {
|
||||||
|
opacity: 0.9;
|
||||||
|
}
|
||||||
|
|
||||||
.input-area textarea {
|
.input-area textarea {
|
||||||
flex: 1;
|
flex: 1;
|
||||||
resize: none;
|
resize: none;
|
||||||
|
|||||||
+560
-410
File diff suppressed because it is too large
Load Diff
@@ -56,6 +56,13 @@ class Config:
|
|||||||
OIDC_SCOPES: str = os.environ.get("OIDC_SCOPES", "openid profile email")
|
OIDC_SCOPES: str = os.environ.get("OIDC_SCOPES", "openid profile email")
|
||||||
LOCAL_AUTH_ENABLED: bool = os.environ.get("LOCAL_AUTH_ENABLED", "true").lower() not in ("0", "false", "no")
|
LOCAL_AUTH_ENABLED: bool = os.environ.get("LOCAL_AUTH_ENABLED", "true").lower() not in ("0", "false", "no")
|
||||||
|
|
||||||
|
# SearXNG web search (external instance)
|
||||||
|
SEARXNG_URL: str = os.environ.get("SEARXNG_URL", "")
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def oidc_enabled(cls) -> bool:
|
def oidc_enabled(cls) -> bool:
|
||||||
return bool(cls.OIDC_ISSUER and cls.OIDC_CLIENT_ID and cls.OIDC_CLIENT_SECRET)
|
return bool(cls.OIDC_ISSUER and cls.OIDC_CLIENT_ID and cls.OIDC_CLIENT_SECRET)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def searxng_enabled(cls) -> bool:
|
||||||
|
return bool(cls.SEARXNG_URL)
|
||||||
|
|||||||
@@ -84,3 +84,17 @@ async def test_caldav():
|
|||||||
uid = get_current_user_id()
|
uid = get_current_user_id()
|
||||||
result = await test_connection(uid)
|
result = await test_connection(uid)
|
||||||
return jsonify(result)
|
return jsonify(result)
|
||||||
|
|
||||||
|
|
||||||
|
@settings_bp.route("/search", methods=["GET"])
|
||||||
|
@login_required
|
||||||
|
async def test_search():
|
||||||
|
"""Test SearXNG connectivity and preview results for a query."""
|
||||||
|
if not Config.searxng_enabled():
|
||||||
|
return jsonify({"configured": False, "results": [], "searxng_url": ""})
|
||||||
|
q = request.args.get("q", "").strip()
|
||||||
|
if not q:
|
||||||
|
return jsonify({"configured": True, "results": [], "searxng_url": Config.SEARXNG_URL})
|
||||||
|
from fabledassistant.services.research import _search_searxng
|
||||||
|
results = await _search_searxng(q)
|
||||||
|
return jsonify({"configured": True, "results": results, "query": q, "searxng_url": Config.SEARXNG_URL})
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ from fabledassistant.services.intent import IntentResult, classify_intent
|
|||||||
from fabledassistant.services.logging import log_generation
|
from fabledassistant.services.logging import log_generation
|
||||||
from fabledassistant.services.settings import get_setting
|
from fabledassistant.services.settings import get_setting
|
||||||
from fabledassistant.services.tools import get_tools_for_user, execute_tool
|
from fabledassistant.services.tools import get_tools_for_user, execute_tool
|
||||||
|
from fabledassistant.services.research import run_research_pipeline
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -59,6 +60,8 @@ _TOOL_LABELS: dict[str, str] = {
|
|||||||
"update_todo": "Updating todo",
|
"update_todo": "Updating todo",
|
||||||
"complete_todo": "Completing todo",
|
"complete_todo": "Completing todo",
|
||||||
"delete_todo": "Removing todo",
|
"delete_todo": "Removing todo",
|
||||||
|
"search_web": "Searching the web",
|
||||||
|
"research_topic": "Researching topic",
|
||||||
}
|
}
|
||||||
|
|
||||||
# Tools that write data and require explicit user confirmation before executing.
|
# Tools that write data and require explicit user confirmation before executing.
|
||||||
@@ -285,6 +288,43 @@ async def run_generation(
|
|||||||
if timing["ttft_ms"] is None:
|
if timing["ttft_ms"] is None:
|
||||||
timing["ttft_ms"] = int((time.monotonic() - t_start) * 1000)
|
timing["ttft_ms"] = int((time.monotonic() - t_start) * 1000)
|
||||||
|
|
||||||
|
if tool_name == "research_topic":
|
||||||
|
topic = intent.arguments.get("topic", "")
|
||||||
|
if not ack_text:
|
||||||
|
fallback_ack = f"I'll research '{topic}' and compile a note.\n\n"
|
||||||
|
buf.append_event("chunk", {"chunk": fallback_ack})
|
||||||
|
buf.content_so_far += fallback_ack
|
||||||
|
if timing["ttft_ms"] is None:
|
||||||
|
timing["ttft_ms"] = int((time.monotonic() - t_start) * 1000)
|
||||||
|
try:
|
||||||
|
note = await run_research_pipeline(
|
||||||
|
topic, user_id, model, intent_model, buf
|
||||||
|
)
|
||||||
|
done_text = (
|
||||||
|
f"Research complete! I've compiled a note: "
|
||||||
|
f"**[{note.title}](/notes/{note.id})**."
|
||||||
|
)
|
||||||
|
buf.append_event("chunk", {"chunk": done_text})
|
||||||
|
buf.content_so_far += done_text
|
||||||
|
tool_record = {
|
||||||
|
"function": "research_topic",
|
||||||
|
"arguments": {"topic": topic},
|
||||||
|
"result": {
|
||||||
|
"success": True,
|
||||||
|
"type": "research_note",
|
||||||
|
"data": {"id": note.id, "title": note.title},
|
||||||
|
},
|
||||||
|
"status": "success",
|
||||||
|
}
|
||||||
|
all_tool_calls.append(tool_record)
|
||||||
|
buf.append_event("tool_call", {"tool_call": tool_record})
|
||||||
|
except Exception as e:
|
||||||
|
logger.exception("Research pipeline failed for topic: %s", topic)
|
||||||
|
err_text = f"\nResearch failed: {e}"
|
||||||
|
buf.append_event("chunk", {"chunk": err_text})
|
||||||
|
buf.content_so_far += err_text
|
||||||
|
break # research IS the full response
|
||||||
|
|
||||||
confirmed = True
|
confirmed = True
|
||||||
if tool_name in _WRITE_TOOLS:
|
if tool_name in _WRITE_TOOLS:
|
||||||
loop = asyncio.get_running_loop()
|
loop = asyncio.get_running_loop()
|
||||||
|
|||||||
@@ -100,6 +100,12 @@ Rules:
|
|||||||
- "read", "open", "show me", "what does X say", "display", "pull up" a specific note → use get_note with query=<note name>.
|
- "read", "open", "show me", "what does X say", "display", "pull up" a specific note → use get_note with query=<note name>.
|
||||||
- "list my notes", "show notes", "recent notes", "browse notes", "notes tagged X" → use list_notes (with optional q or tags).
|
- "list my notes", "show notes", "recent notes", "browse notes", "notes tagged X" → use list_notes (with optional q or tags).
|
||||||
- "tag X with Y", "add tag Y to X", "untag Y from X", "remove tag Y from X" → use update_note with tags=[Y] and tag_mode="add" or "remove".
|
- "tag X with Y", "add tag Y to X", "untag Y from X", "remove tag Y from X" → use update_note with tags=[Y] and tag_mode="add" or "remove".
|
||||||
|
- search_web: user wants a quick web search to answer a factual question
|
||||||
|
("search for X", "look up X", "what is the latest version of X", "find X online",
|
||||||
|
"google X", "what is X" for quick factual answers — NOT when they want a comprehensive note)
|
||||||
|
- research_topic: user wants to research a topic and create a comprehensive note from web sources
|
||||||
|
("research X", "research X and make a note", "compile notes on X", "write a report on X",
|
||||||
|
"deep dive into X", "find everything about X", "comprehensive guide to X")
|
||||||
- "ack": one short, natural sentence confirming the action (tool path only). Vary phrasing — do not always start with "Let me". Omit (null) for chat-only responses.
|
- "ack": one short, natural sentence confirming the action (tool path only). Vary phrasing — do not always start with "Let me". Omit (null) for chat-only responses.
|
||||||
- Do NOT wrap the JSON in markdown code fences."""
|
- Do NOT wrap the JSON in markdown code fences."""
|
||||||
|
|
||||||
|
|||||||
@@ -208,12 +208,21 @@ async def stream_chat_with_tools(
|
|||||||
break
|
break
|
||||||
|
|
||||||
|
|
||||||
async def generate_completion(messages: list[dict], model: str, max_tokens: int = 4096) -> str:
|
async def generate_completion(
|
||||||
|
messages: list[dict],
|
||||||
|
model: str,
|
||||||
|
max_tokens: int = 4096,
|
||||||
|
num_ctx: int | None = None,
|
||||||
|
) -> str:
|
||||||
"""Non-streaming chat completion, returns full response text.
|
"""Non-streaming chat completion, returns full response text.
|
||||||
|
|
||||||
Retries up to 2 times on Ollama 500 errors (cold model loading race).
|
Retries up to 2 times on Ollama 500 errors (cold model loading race).
|
||||||
|
num_ctx overrides the model's context window for this call only.
|
||||||
"""
|
"""
|
||||||
last_exc: Exception | None = None
|
last_exc: Exception | None = None
|
||||||
|
options: dict = {"num_predict": max_tokens}
|
||||||
|
if num_ctx is not None:
|
||||||
|
options["num_ctx"] = num_ctx
|
||||||
for attempt in range(3):
|
for attempt in range(3):
|
||||||
if attempt > 0:
|
if attempt > 0:
|
||||||
delay = 3.0 * attempt
|
delay = 3.0 * attempt
|
||||||
@@ -230,7 +239,7 @@ async def generate_completion(messages: list[dict], model: str, max_tokens: int
|
|||||||
"messages": messages,
|
"messages": messages,
|
||||||
"stream": False,
|
"stream": False,
|
||||||
"think": False,
|
"think": False,
|
||||||
"options": {"num_predict": max_tokens},
|
"options": options,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
resp.raise_for_status()
|
resp.raise_for_status()
|
||||||
|
|||||||
@@ -0,0 +1,242 @@
|
|||||||
|
"""Web research pipeline: sub-queries → SearXNG → fetch → synthesize → note."""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
from fabledassistant.config import Config
|
||||||
|
from fabledassistant.services.llm import fetch_url_content, generate_completion
|
||||||
|
from fabledassistant.services.notes import create_note
|
||||||
|
from fabledassistant.models.note import Note
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
SEARXNG_QUERIES = 5 # sub-queries to generate
|
||||||
|
RESULTS_PER_QUERY = 3 # results fetched from SearXNG per query
|
||||||
|
PAGES_PER_QUERY = 3 # pages actually read per sub-query (top N results)
|
||||||
|
MAX_SYNTHESIS_SOURCES = 12 # deduplicated sources passed to synthesis LLM
|
||||||
|
CHARS_PER_SOURCE = 2000 # content chars per source sent to synthesis
|
||||||
|
|
||||||
|
|
||||||
|
async def run_research_pipeline(
|
||||||
|
topic: str,
|
||||||
|
user_id: int,
|
||||||
|
model: str,
|
||||||
|
intent_model: str,
|
||||||
|
buf,
|
||||||
|
) -> Note:
|
||||||
|
"""Full research pipeline: search → fetch → synthesize → create note.
|
||||||
|
|
||||||
|
Emits status events via buf.append_event throughout.
|
||||||
|
Returns the created Note.
|
||||||
|
"""
|
||||||
|
# Step 1: Generate sub-queries
|
||||||
|
buf.append_event("status", {"status": "Generating search queries..."})
|
||||||
|
queries = await _generate_sub_queries(topic, intent_model)
|
||||||
|
logger.info("Research: generated %d sub-queries for topic '%s'", len(queries), topic)
|
||||||
|
|
||||||
|
# Step 2: Search and fetch
|
||||||
|
all_sources: list[dict] = []
|
||||||
|
seen_urls: set[str] = set()
|
||||||
|
|
||||||
|
for i, query in enumerate(queries):
|
||||||
|
if i > 0:
|
||||||
|
await asyncio.sleep(1.0) # avoid hammering SearXNG
|
||||||
|
buf.append_event("status", {"status": f"Searching: {query}..."})
|
||||||
|
results = await _search_searxng(query)
|
||||||
|
logger.info("Research: query '%s' → %d results", query, len(results))
|
||||||
|
|
||||||
|
for result in results[:PAGES_PER_QUERY]:
|
||||||
|
url = result.get("url", "")
|
||||||
|
if not url or url in seen_urls:
|
||||||
|
continue
|
||||||
|
seen_urls.add(url)
|
||||||
|
title = result.get("title", url)
|
||||||
|
buf.append_event("status", {"status": f"Reading: {title[:60]}..."})
|
||||||
|
content = await fetch_url_content(url)
|
||||||
|
all_sources.append({
|
||||||
|
"url": url,
|
||||||
|
"title": title,
|
||||||
|
"query": query,
|
||||||
|
"snippet": result.get("snippet", ""),
|
||||||
|
"content": content,
|
||||||
|
})
|
||||||
|
|
||||||
|
if not all_sources:
|
||||||
|
raise ValueError(f"No results found for '{topic}'")
|
||||||
|
|
||||||
|
# Step 3: Filter failed fetches
|
||||||
|
good_sources = [
|
||||||
|
s for s in all_sources
|
||||||
|
if not s["content"].startswith("[Failed to fetch")
|
||||||
|
]
|
||||||
|
|
||||||
|
if not good_sources:
|
||||||
|
raise ValueError(f"Could not read any sources for '{topic}'")
|
||||||
|
|
||||||
|
# Limit to top N sources for synthesis (already deduplicated by URL)
|
||||||
|
synthesis_sources = good_sources[:MAX_SYNTHESIS_SOURCES]
|
||||||
|
logger.info(
|
||||||
|
"Research: %d/%d sources successfully fetched, using %d for synthesis",
|
||||||
|
len(good_sources), len(all_sources), len(synthesis_sources),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Step 4: Synthesize
|
||||||
|
buf.append_event("status", {"status": f"Synthesizing report from {len(synthesis_sources)} sources..."})
|
||||||
|
title, body = await _synthesize_note(topic, synthesis_sources, model)
|
||||||
|
|
||||||
|
# Step 5: Create note
|
||||||
|
buf.append_event("status", {"status": "Saving note..."})
|
||||||
|
note = await create_note(
|
||||||
|
user_id=user_id,
|
||||||
|
title=title,
|
||||||
|
body=body,
|
||||||
|
tags=["research"],
|
||||||
|
)
|
||||||
|
logger.info("Research: created note id=%d title='%s'", note.id, note.title)
|
||||||
|
return note
|
||||||
|
|
||||||
|
|
||||||
|
async def _generate_sub_queries(topic: str, intent_model: str) -> list[str]:
|
||||||
|
"""Ask the intent model for focused search queries for the topic."""
|
||||||
|
messages = [
|
||||||
|
{
|
||||||
|
"role": "system",
|
||||||
|
"content": (
|
||||||
|
f"You are a research assistant. Given a research topic, generate exactly {SEARXNG_QUERIES} "
|
||||||
|
"focused web search queries that together would provide comprehensive coverage of the topic. "
|
||||||
|
"Vary the angle of each query: include overview, implementation details, best practices, "
|
||||||
|
"common problems, and real-world examples. "
|
||||||
|
"Respond with ONLY a JSON array of strings, no other text. "
|
||||||
|
'Example: ["query one", "query two", "query three"]'
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{"role": "user", "content": f"Topic: {topic}"},
|
||||||
|
]
|
||||||
|
try:
|
||||||
|
raw = await generate_completion(messages, intent_model, max_tokens=200)
|
||||||
|
raw = raw.strip()
|
||||||
|
raw = re.sub(r"^```(?:json)?\s*", "", raw)
|
||||||
|
raw = re.sub(r"\s*```$", "", raw)
|
||||||
|
idx = raw.find("[")
|
||||||
|
if idx >= 0:
|
||||||
|
parsed, _ = json.JSONDecoder().raw_decode(raw[idx:])
|
||||||
|
if isinstance(parsed, list) and parsed:
|
||||||
|
queries = [str(q).strip() for q in parsed if str(q).strip()]
|
||||||
|
if queries:
|
||||||
|
return queries[:SEARXNG_QUERIES]
|
||||||
|
except Exception:
|
||||||
|
logger.warning("Sub-query generation failed, falling back to topic", exc_info=True)
|
||||||
|
return [topic]
|
||||||
|
|
||||||
|
|
||||||
|
async def _search_searxng(query: str) -> list[dict]:
|
||||||
|
"""Search SearXNG and return top results as [{url, title, snippet}]."""
|
||||||
|
url = Config.SEARXNG_URL.rstrip("/") + "/search"
|
||||||
|
params = {"q": query, "format": "json", "categories": "general"}
|
||||||
|
for attempt in range(3):
|
||||||
|
try:
|
||||||
|
async with httpx.AsyncClient(timeout=10.0) as client:
|
||||||
|
resp = await client.get(url, params=params)
|
||||||
|
if resp.status_code == 429:
|
||||||
|
retry_after = int(resp.headers.get("Retry-After", "5"))
|
||||||
|
wait = min(retry_after, 10) * (attempt + 1)
|
||||||
|
logger.warning(
|
||||||
|
"SearXNG 429 for query '%s' (attempt %d/3), waiting %ds",
|
||||||
|
query, attempt + 1, wait,
|
||||||
|
)
|
||||||
|
await asyncio.sleep(wait)
|
||||||
|
continue
|
||||||
|
resp.raise_for_status()
|
||||||
|
data = resp.json()
|
||||||
|
results = data.get("results", [])
|
||||||
|
out = []
|
||||||
|
for r in results[:RESULTS_PER_QUERY]:
|
||||||
|
out.append({
|
||||||
|
"url": r.get("url", ""),
|
||||||
|
"title": r.get("title", ""),
|
||||||
|
"snippet": r.get("content", ""),
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
except httpx.HTTPStatusError:
|
||||||
|
logger.warning("SearXNG search failed for query '%s'", query, exc_info=True)
|
||||||
|
return []
|
||||||
|
except Exception:
|
||||||
|
logger.warning("SearXNG search failed for query '%s'", query, exc_info=True)
|
||||||
|
return []
|
||||||
|
logger.warning("SearXNG search gave up after 3 attempts for query '%s'", query)
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
async def _synthesize_note(
|
||||||
|
topic: str,
|
||||||
|
sources: list[dict],
|
||||||
|
model: str,
|
||||||
|
) -> tuple[str, str]:
|
||||||
|
"""Synthesize a comprehensive markdown research document from fetched sources.
|
||||||
|
|
||||||
|
Returns (title, body_markdown).
|
||||||
|
Uses an extended context window so the output can be several thousand words.
|
||||||
|
"""
|
||||||
|
sources_text_parts = []
|
||||||
|
for i, s in enumerate(sources, 1):
|
||||||
|
content = (s.get("content") or s.get("snippet") or "")[:CHARS_PER_SOURCE]
|
||||||
|
sources_text_parts.append(
|
||||||
|
f"[Source {i}] {s['title']}\nURL: {s['url']}\nSearch query: {s['query']}\n\n{content}"
|
||||||
|
)
|
||||||
|
sources_block = "\n\n" + ("─" * 60) + "\n\n".join(sources_text_parts)
|
||||||
|
|
||||||
|
messages = [
|
||||||
|
{
|
||||||
|
"role": "system",
|
||||||
|
"content": (
|
||||||
|
"You are a thorough researcher and writer. "
|
||||||
|
"Your task is to write an exhaustive, well-structured document on the given topic — "
|
||||||
|
"not a brief summary or intro paragraph.\n\n"
|
||||||
|
"Requirements:\n"
|
||||||
|
"- Write at least 2500 words of substantive content (excluding the Sources section)\n"
|
||||||
|
"- Choose sections (##) that make sense for the topic — let the subject matter determine the structure. "
|
||||||
|
"A technical topic might need implementation, configuration, and troubleshooting sections. "
|
||||||
|
"A comparison topic might need dedicated sections per subject being compared plus a summary. "
|
||||||
|
"A scientific topic might need background, mechanisms, research findings, and implications. "
|
||||||
|
"Use your judgment — minimum 6 major sections.\n"
|
||||||
|
"- Use ### for subsections where they add clarity\n"
|
||||||
|
"- Write in detailed prose paragraphs — do not reduce sections to bullet-point lists\n"
|
||||||
|
"- Include specific details, examples, data points, comparisons, and nuance from the sources\n"
|
||||||
|
"- Do not pad with vague generalities — every paragraph should say something concrete\n"
|
||||||
|
"- The first line must be the document title starting with '# '\n"
|
||||||
|
"- End with a '## Sources' section listing every source as a markdown hyperlink\n\n"
|
||||||
|
"The reader wants to finish this document with a thorough understanding of the topic, "
|
||||||
|
"not just an overview."
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"role": "user",
|
||||||
|
"content": (
|
||||||
|
f"Write a comprehensive reference document on: {topic}\n\n"
|
||||||
|
f"Sources ({len(sources)} pages fetched):\n{sources_block}"
|
||||||
|
),
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
raw = await generate_completion(
|
||||||
|
messages,
|
||||||
|
model,
|
||||||
|
max_tokens=8192,
|
||||||
|
num_ctx=16384,
|
||||||
|
)
|
||||||
|
raw = raw.strip()
|
||||||
|
|
||||||
|
# Extract title from first # heading
|
||||||
|
lines = raw.splitlines()
|
||||||
|
title = f"Research: {topic}"
|
||||||
|
body_lines = lines
|
||||||
|
if lines and lines[0].startswith("# "):
|
||||||
|
title = lines[0][2:].strip()
|
||||||
|
body_lines = lines[1:]
|
||||||
|
|
||||||
|
body = "\n".join(body_lines).strip()
|
||||||
|
return title, body
|
||||||
@@ -19,6 +19,7 @@ from fabledassistant.services.caldav import (
|
|||||||
update_event,
|
update_event,
|
||||||
update_todo,
|
update_todo,
|
||||||
)
|
)
|
||||||
|
from fabledassistant.config import Config
|
||||||
from fabledassistant.services.notes import create_note, delete_note, get_note_by_title, list_notes, update_note
|
from fabledassistant.services.notes import create_note, delete_note, get_note_by_title, list_notes, update_note
|
||||||
from fabledassistant.services.tag_suggestions import suggest_tags
|
from fabledassistant.services.tag_suggestions import suggest_tags
|
||||||
|
|
||||||
@@ -663,11 +664,61 @@ _CALDAV_TOOLS = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
_SEARCH_TOOLS = [
|
||||||
|
{
|
||||||
|
"type": "function",
|
||||||
|
"function": {
|
||||||
|
"name": "search_web",
|
||||||
|
"description": (
|
||||||
|
"Search the web for quick information and answer the user's question from results. "
|
||||||
|
"Use for factual lookups, current events, version numbers, quick definitions, or any question "
|
||||||
|
"where a fast web answer is needed without creating a note. "
|
||||||
|
"Use research_topic instead when the user explicitly wants a comprehensive note or deep research."
|
||||||
|
),
|
||||||
|
"parameters": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"query": {"type": "string", "description": "The search query"}
|
||||||
|
},
|
||||||
|
"required": ["query"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
_RESEARCH_TOOLS = [
|
||||||
|
{
|
||||||
|
"type": "function",
|
||||||
|
"function": {
|
||||||
|
"name": "research_topic",
|
||||||
|
"description": (
|
||||||
|
"Research a topic by searching the web and compiling a note with cited sources. "
|
||||||
|
"Use for 'research X', 'look up X', 'find information about X', "
|
||||||
|
"'compile notes on X'. Takes 30–120 seconds."
|
||||||
|
),
|
||||||
|
"parameters": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"topic": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The topic or question to research",
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": ["topic"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
async def get_tools_for_user(user_id: int) -> list[dict]:
|
async def get_tools_for_user(user_id: int) -> list[dict]:
|
||||||
"""Build the tool list for a user based on their configured integrations."""
|
"""Build the tool list for a user based on their configured integrations."""
|
||||||
tools = list(_CORE_TOOLS)
|
tools = list(_CORE_TOOLS)
|
||||||
if await is_caldav_configured(user_id):
|
if await is_caldav_configured(user_id):
|
||||||
tools.extend(_CALDAV_TOOLS)
|
tools.extend(_CALDAV_TOOLS)
|
||||||
|
if Config.searxng_enabled():
|
||||||
|
tools.extend(_SEARCH_TOOLS)
|
||||||
|
tools.extend(_RESEARCH_TOOLS)
|
||||||
logger.debug("User %d: %d tools available", user_id, len(tools))
|
logger.debug("User %d: %d tools available", user_id, len(tools))
|
||||||
return tools
|
return tools
|
||||||
|
|
||||||
@@ -1128,6 +1179,32 @@ async def execute_tool(user_id: int, tool_name: str, arguments: dict) -> dict:
|
|||||||
"data": {"id": note.id, "title": note.title},
|
"data": {"id": note.id, "title": note.title},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
elif tool_name == "search_web":
|
||||||
|
from fabledassistant.services.research import _search_searxng
|
||||||
|
query = arguments.get("query", "")
|
||||||
|
results = await _search_searxng(query)
|
||||||
|
if not results:
|
||||||
|
return {"success": False, "error": f"No results found for '{query}'"}
|
||||||
|
return {
|
||||||
|
"success": True,
|
||||||
|
"type": "web_search",
|
||||||
|
"data": {
|
||||||
|
"query": query,
|
||||||
|
"results": results,
|
||||||
|
"count": len(results),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
elif tool_name == "research_topic":
|
||||||
|
# Research is always handled upstream in generation_task.py (round 0).
|
||||||
|
# This fallback exists in case it somehow reaches execute_tool.
|
||||||
|
topic = arguments.get("topic", "")
|
||||||
|
return {
|
||||||
|
"success": True,
|
||||||
|
"type": "research_pending",
|
||||||
|
"data": {"topic": topic},
|
||||||
|
}
|
||||||
|
|
||||||
else:
|
else:
|
||||||
return {"success": False, "error": f"Unknown tool: {tool_name}"}
|
return {"success": False, "error": f"Unknown tool: {tool_name}"}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user