Phase 22: SearXNG web research pipeline + settings layout overhaul

Research pipeline (research_topic tool):
- New service: services/research.py — sub-query generation, SearXNG
  search, URL fetch, deduplication, and LLM synthesis into a note
- 5 sub-queries × 3 pages = up to 15 sources, capped at 12 for synthesis
- Synthesis uses num_ctx=16384 + max_tokens=8192 for long-form output
- Prompt demands 2500+ words, 6+ topic-appropriate sections, detailed prose
- 429 retry with backoff; 1s inter-query sleep; raw_decode JSON parsing

search_web tool (new):
- Lightweight single-query SearXNG search, results returned inline in chat
- LLM answers conversationally in round 1; no note created
- web_search result type with external links in ToolCallCard

Infrastructure:
- llm.py: generate_completion accepts num_ctx override
- config.py: SEARXNG_URL + Config.searxng_enabled()
- docker-compose: OLLAMA_NUM_PARALLEL=2, commented SEARXNG_URL example
- intent.py: search_web and research_topic routing rules

Settings UI:
- 2-column grid layout (small sections pair up, complex span full width)
- Search Test section: live SearXNG query with result preview
- GET /api/settings/search?q= proxy endpoint
- Research button (magnifier) in ChatView input toolbar → popover modal

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-02-27 15:21:38 -05:00
parent 432e0bd2a0
commit 590682a5d2
11 changed files with 1132 additions and 413 deletions
+3
View File
@@ -13,6 +13,8 @@ services:
OLLAMA_URL: "http://ollama:11434" OLLAMA_URL: "http://ollama:11434"
OLLAMA_MODEL: "${OLLAMA_MODEL:-llama3.1}" OLLAMA_MODEL: "${OLLAMA_MODEL:-llama3.1}"
SECRET_KEY: "${SECRET_KEY:-dev-secret-change-me}" SECRET_KEY: "${SECRET_KEY:-dev-secret-change-me}"
# Uncomment and set to enable web research via SearXNG:
# SEARXNG_URL: "http://searxng:8080"
healthcheck: healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:5000/api/health')"] test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:5000/api/health')"]
interval: 10s interval: 10s
@@ -40,6 +42,7 @@ services:
- ollama_models:/root/.ollama - ollama_models:/root/.ollama
environment: environment:
OLLAMA_MAX_LOADED_MODELS: "2" OLLAMA_MAX_LOADED_MODELS: "2"
OLLAMA_NUM_PARALLEL: "2"
OLLAMA_KEEP_ALIVE: "30m" OLLAMA_KEEP_ALIVE: "30m"
OLLAMA_FLASH_ATTENTION: "1" OLLAMA_FLASH_ATTENTION: "1"
deploy: deploy:
+32
View File
@@ -52,6 +52,12 @@ const label = computed(() => {
return "Completed todo"; return "Completed todo";
case "todo_deleted": case "todo_deleted":
return "Deleted todo"; return "Deleted todo";
case "web_search":
return "Web search";
case "research_pending":
return "Research started";
case "research_note":
return "Research note";
default: default:
return "Tool call"; return "Tool call";
} }
@@ -167,6 +173,12 @@ const taskCount = computed(() => {
return (data.total as number) ?? 0; return (data.total as number) ?? 0;
}); });
const webSearch = computed(() => {
const data = props.toolCall.result.data;
if (!data || props.toolCall.result.type !== "web_search") return null;
return data as { query: string; results: { url: string; title: string; snippet: string }[]; count: number };
});
const searchResults = computed(() => { const searchResults = computed(() => {
const data = props.toolCall.result.data; const data = props.toolCall.result.data;
if (!data || props.toolCall.result.type !== "search") return null; if (!data || props.toolCall.result.type !== "search") return null;
@@ -317,6 +329,19 @@ async function applyTag(tag: string) {
</div> </div>
</div> </div>
</template> </template>
<template v-else-if="webSearch">
<span class="tool-search-info">{{ webSearch.count }} result{{ webSearch.count !== 1 ? "s" : "" }}</span>
<div class="tool-search-results web-search-results">
<a
v-for="(r, i) in webSearch.results"
:key="i"
:href="r.url"
target="_blank"
rel="noopener noreferrer"
class="tool-search-item"
>{{ r.title || r.url }}</a>
</div>
</template>
<template v-else-if="linkTo"> <template v-else-if="linkTo">
<router-link :to="linkTo" class="tool-link">{{ title }}</router-link> <router-link :to="linkTo" class="tool-link">{{ title }}</router-link>
</template> </template>
@@ -400,6 +425,13 @@ async function applyTag(tag: string) {
color: var(--color-text-muted); color: var(--color-text-muted);
margin-right: 0.1rem; margin-right: 0.1rem;
} }
.web-search-results {
flex-direction: column;
flex-wrap: nowrap;
}
.web-search-results .tool-search-item::after {
content: none;
}
.tool-event-title { .tool-event-title {
font-weight: 600; font-weight: 600;
color: var(--color-text); color: var(--color-text);
+140 -1
View File
@@ -22,6 +22,10 @@ const sending = ref(false);
const summarizing = ref(false); const summarizing = ref(false);
const sidebarOpen = ref(false); const sidebarOpen = ref(false);
// Research modal state
const showResearchModal = ref(false);
const researchTopic = ref("");
// Note picker state // Note picker state
const attachedNote = ref<{ id: number; title: string } | null>(null); const attachedNote = ref<{ id: number; title: string } | null>(null);
const showNotePicker = ref(false); const showNotePicker = ref(false);
@@ -239,6 +243,27 @@ function onInputKeydown(e: KeyboardEvent) {
} }
} }
// Research modal
function toggleResearchModal() {
showResearchModal.value = !showResearchModal.value;
if (showResearchModal.value) {
researchTopic.value = "";
nextTick(() => {
const el = document.querySelector(".research-topic-input") as HTMLInputElement;
el?.focus();
});
}
}
function submitResearch() {
const topic = researchTopic.value.trim();
if (!topic) return;
messageInput.value = `Research: ${topic}`;
showResearchModal.value = false;
researchTopic.value = "";
sendMessage();
}
// Note picker // Note picker
function toggleNotePicker() { function toggleNotePicker() {
showNotePicker.value = !showNotePicker.value; showNotePicker.value = !showNotePicker.value;
@@ -302,7 +327,12 @@ function removeIncludedNote(noteId: number) {
// Keyboard shortcuts // Keyboard shortcuts
function onGlobalKeydown(e: KeyboardEvent) { function onGlobalKeydown(e: KeyboardEvent) {
if (e.key !== "Escape") return; if (e.key !== "Escape") return;
// Close note picker first // Close research modal first
if (showResearchModal.value) {
showResearchModal.value = false;
return;
}
// Close note picker
if (showNotePicker.value) { if (showNotePicker.value) {
showNotePicker.value = false; showNotePicker.value = false;
return; return;
@@ -479,6 +509,37 @@ onUnmounted(() => {
<div class="input-wrapper"> <div class="input-wrapper">
<div class="input-area"> <div class="input-area">
<!-- Research button -->
<div class="research-wrapper">
<button
class="btn-attach"
@click="toggleResearchModal"
:disabled="store.streaming || !store.chatReady"
title="Research a topic"
>
<svg width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
<circle cx="11" cy="11" r="8"/>
<line x1="21" y1="21" x2="16.65" y2="16.65"/>
</svg>
</button>
<!-- Research modal -->
<div v-if="showResearchModal" class="research-modal">
<div class="research-modal-header">Research topic</div>
<input
class="research-topic-input"
v-model="researchTopic"
placeholder="e.g. quantum computing"
@keydown.enter="submitResearch"
@keydown.escape="showResearchModal = false"
/>
<div class="research-modal-actions">
<button class="btn-research-cancel" @click="showResearchModal = false">Cancel</button>
<button class="btn-research-go" @click="submitResearch" :disabled="!researchTopic.trim()">Go</button>
</div>
</div>
</div>
<!-- Note picker button --> <!-- Note picker button -->
<div class="note-picker-wrapper"> <div class="note-picker-wrapper">
<button <button
@@ -1005,6 +1066,84 @@ details[open] .thinking-summary::before {
color: var(--color-text-muted); color: var(--color-text-muted);
font-size: 0.85rem; font-size: 0.85rem;
} }
/* Research modal */
.research-wrapper {
position: relative;
}
.research-modal {
position: absolute;
bottom: calc(100% + 8px);
left: 0;
width: 280px;
background: var(--color-bg-card);
border: 1px solid var(--color-border);
border-radius: var(--radius-md);
box-shadow: 0 4px 16px var(--color-shadow);
z-index: 10;
padding: 0.75rem;
display: flex;
flex-direction: column;
gap: 0.5rem;
}
.research-modal-header {
font-size: 0.8rem;
font-weight: 600;
color: var(--color-text-muted);
text-transform: uppercase;
letter-spacing: 0.04em;
}
.research-topic-input {
width: 100%;
padding: 0.4rem 0.6rem;
border: 1px solid var(--color-border);
border-radius: var(--radius-sm);
background: var(--color-bg-secondary);
color: var(--color-text);
font-size: 0.9rem;
outline: none;
font-family: inherit;
box-sizing: border-box;
}
.research-topic-input:focus {
border-color: var(--color-primary);
}
.research-modal-actions {
display: flex;
justify-content: flex-end;
gap: 0.4rem;
}
.btn-research-cancel {
padding: 0.3rem 0.65rem;
background: none;
border: 1px solid var(--color-border);
border-radius: var(--radius-sm);
color: var(--color-text-muted);
font-size: 0.85rem;
cursor: pointer;
font-family: inherit;
}
.btn-research-cancel:hover {
background: var(--color-bg-secondary);
}
.btn-research-go {
padding: 0.3rem 0.75rem;
background: var(--color-primary);
border: none;
border-radius: var(--radius-sm);
color: #fff;
font-size: 0.85rem;
cursor: pointer;
font-family: inherit;
}
.btn-research-go:disabled {
opacity: 0.4;
cursor: default;
}
.btn-research-go:not(:disabled):hover {
opacity: 0.9;
}
.input-area textarea { .input-area textarea {
flex: 1; flex: 1;
resize: none; resize: none;
+314 -164
View File
@@ -65,6 +65,14 @@ const baseUrl = ref("");
const savingBaseUrl = ref(false); const savingBaseUrl = ref(false);
const baseUrlSaved = ref(false); const baseUrlSaved = ref(false);
// Search test (SearXNG)
const searxngConfigured = ref(false);
const searxngUrl = ref("");
const searchQuery = ref("");
const searchResults = ref<{ url: string; title: string; snippet: string }[]>([]);
const searchLoading = ref(false);
const searchError = ref("");
onMounted(async () => { onMounted(async () => {
await store.fetchSettings(); await store.fetchSettings();
assistantName.value = store.assistantName; assistantName.value = store.assistantName;
@@ -99,6 +107,15 @@ onMounted(async () => {
// CalDAV not configured yet // CalDAV not configured yet
} }
// Check SearXNG status
try {
const sr = await apiGet<{ configured: boolean; searxng_url: string }>("/api/settings/search");
searxngConfigured.value = sr.configured;
searxngUrl.value = sr.searxng_url;
} catch {
searxngConfigured.value = false;
}
// Load admin settings // Load admin settings
if (authStore.isAdmin) { if (authStore.isAdmin) {
try { try {
@@ -332,18 +349,52 @@ async function handleRestoreFile(event: Event) {
toastStore.show("Restore failed: " + (e as Error).message, "error"); toastStore.show("Restore failed: " + (e as Error).message, "error");
} finally { } finally {
restoring.value = false; restoring.value = false;
// Reset file input
if (restoreFileInput.value) restoreFileInput.value.value = ""; if (restoreFileInput.value) restoreFileInput.value.value = "";
} }
} }
async function testSearch() {
const q = searchQuery.value.trim();
if (!q) return;
searchLoading.value = true;
searchError.value = "";
searchResults.value = [];
try {
const data = await apiGet<{ configured: boolean; results: { url: string; title: string; snippet: string }[]; error?: string }>(
`/api/settings/search?q=${encodeURIComponent(q)}`
);
if (!data.configured) {
searchError.value = "SearXNG is not configured — set SEARXNG_URL in docker-compose.";
} else {
searchResults.value = data.results;
if (!data.results.length) searchError.value = "No results found.";
}
} catch {
searchError.value = "Search request failed.";
} finally {
searchLoading.value = false;
}
}
function onSearchKeydown(e: KeyboardEvent) {
if (e.key === "Enter") testSearch();
}
function hostname(url: string): string {
try { return new URL(url).hostname; } catch { return url; }
}
</script> </script>
<template> <template>
<main class="settings-page"> <main class="settings-page">
<h1>Settings</h1> <h1>Settings</h1>
<section class="settings-section"> <div class="settings-grid">
<!-- Assistant full width -->
<section class="settings-section full-width">
<h2>Assistant</h2> <h2>Assistant</h2>
<div class="assistant-grid">
<div class="field"> <div class="field">
<label for="assistant-name">Assistant Name</label> <label for="assistant-name">Assistant Name</label>
<input <input
@@ -353,11 +404,8 @@ async function handleRestoreFile(event: Event) {
placeholder="Fable" placeholder="Fable"
class="input" class="input"
/> />
<p class="field-hint"> <p class="field-hint">The name used in chat messages and LLM context.</p>
The name used for the AI assistant in chat messages and LLM context.
</p>
</div> </div>
<div class="field"> <div class="field">
<label for="default-model">Chat Model</label> <label for="default-model">Chat Model</label>
<select id="default-model" v-model="defaultModel" class="input"> <select id="default-model" v-model="defaultModel" class="input">
@@ -366,16 +414,15 @@ async function handleRestoreFile(event: Event) {
</select> </select>
<p class="field-hint">Model used for new conversations.</p> <p class="field-hint">Model used for new conversations.</p>
</div> </div>
<div class="field"> <div class="field">
<label for="intent-model">Intent Model</label> <label for="intent-model">Intent Model</label>
<select id="intent-model" v-model="intentModel" class="input"> <select id="intent-model" v-model="intentModel" class="input">
<option value="">Default ({{ defaultIntentModel || "qwen2.5:1.5b" }})</option> <option value="">Default ({{ defaultIntentModel || "qwen2.5:1.5b" }})</option>
<option v-for="m in installedModels" :key="m" :value="m">{{ m }}</option> <option v-for="m in installedModels" :key="m" :value="m">{{ m }}</option>
</select> </select>
<p class="field-hint">Smaller/faster model for intent routing. Runs before the main model to detect tool calls.</p> <p class="field-hint">Smaller/faster model for intent routing before the main model.</p>
</div>
</div> </div>
<div class="actions"> <div class="actions">
<button class="btn-save" @click="saveAssistant" :disabled="saving"> <button class="btn-save" @click="saveAssistant" :disabled="saving">
{{ saving ? "Saving..." : "Save" }} {{ saving ? "Saving..." : "Save" }}
@@ -384,6 +431,7 @@ async function handleRestoreFile(event: Event) {
</div> </div>
</section> </section>
<!-- Email Address half width -->
<section class="settings-section"> <section class="settings-section">
<h2>Email Address</h2> <h2>Email Address</h2>
<p class="section-desc">Used for password resets and notifications.</p> <p class="section-desc">Used for password resets and notifications.</p>
@@ -419,6 +467,7 @@ async function handleRestoreFile(event: Event) {
</div> </div>
</section> </section>
<!-- Change Password half width -->
<section class="settings-section"> <section class="settings-section">
<h2>Change Password</h2> <h2>Change Password</h2>
<div class="field"> <div class="field">
@@ -467,25 +516,25 @@ async function handleRestoreFile(event: Event) {
</div> </div>
</section> </section>
<!-- Notifications half width -->
<section class="settings-section"> <section class="settings-section">
<h2>Notifications</h2> <h2>Notifications</h2>
<p class="section-desc"> <p class="section-desc">
Choose which email notifications you'd like to receive. Requires SMTP to be configured by an admin. Email notifications when SMTP is configured by an admin.
</p> </p>
<div class="checkbox-field"> <div class="checkbox-field">
<label> <label>
<input type="checkbox" v-model="notifyTaskReminders" /> <input type="checkbox" v-model="notifyTaskReminders" />
Task due date reminders Task due date reminders
</label> </label>
<p class="field-hint">Receive a daily email when you have tasks due or overdue.</p> <p class="field-hint">Daily email for tasks due or overdue.</p>
</div> </div>
<div class="checkbox-field"> <div class="checkbox-field">
<label> <label>
<input type="checkbox" v-model="notifySecurityAlerts" /> <input type="checkbox" v-model="notifySecurityAlerts" />
Security alerts Security alerts
</label> </label>
<p class="field-hint">Receive emails for login, logout, failed login attempts, and password changes.</p> <p class="field-hint">Emails for logins, logouts, and password changes.</p>
</div> </div>
<div class="actions"> <div class="actions">
<button class="btn-save" @click="saveNotifications" :disabled="savingNotifications"> <button class="btn-save" @click="saveNotifications" :disabled="savingNotifications">
@@ -495,12 +544,38 @@ async function handleRestoreFile(event: Event) {
</div> </div>
</section> </section>
<!-- Data half width -->
<section class="settings-section"> <section class="settings-section">
<h2>Data</h2>
<p class="section-desc">Export your data or restore from a backup.</p>
<div class="data-actions">
<button class="btn-secondary" @click="exportData('user')" :disabled="exporting">
{{ exporting ? "Exporting..." : "Export My Data" }}
</button>
<template v-if="authStore.isAdmin">
<button class="btn-secondary" @click="exportData('full')" :disabled="exporting">
{{ exporting ? "Exporting..." : "Full Backup" }}
</button>
<button class="btn-secondary btn-warn" @click="triggerRestoreUpload" :disabled="restoring">
{{ restoring ? "Restoring..." : "Restore from Backup" }}
</button>
<input
ref="restoreFileInput"
type="file"
accept=".json"
class="hidden-file-input"
@change="handleRestoreFile"
/>
</template>
</div>
</section>
<!-- CalDAV full width -->
<section class="settings-section full-width">
<h2>Calendar (CalDAV)</h2> <h2>Calendar (CalDAV)</h2>
<p class="section-desc"> <p class="section-desc">
Connect to a CalDAV server (Nextcloud, iCloud, Radicale, Baikal) to create and view calendar events from chat. Connect to a CalDAV server (Nextcloud, iCloud, Radicale, Baikal) to create and view calendar events from chat.
</p> </p>
<div class="caldav-grid"> <div class="caldav-grid">
<div class="field"> <div class="field">
<label for="caldav-url">CalDAV URL</label> <label for="caldav-url">CalDAV URL</label>
@@ -517,20 +592,18 @@ async function handleRestoreFile(event: Event) {
<div class="field"> <div class="field">
<label for="caldav-calendar-name">Calendar Name</label> <label for="caldav-calendar-name">Calendar Name</label>
<input id="caldav-calendar-name" v-model="caldav.caldav_calendar_name" type="text" placeholder="Leave empty to use first available" class="input" /> <input id="caldav-calendar-name" v-model="caldav.caldav_calendar_name" type="text" placeholder="Leave empty to use first available" class="input" />
<p class="field-hint">Optional. The exact name of the calendar to use.</p> <p class="field-hint">Optional. Exact calendar name to use.</p>
</div> </div>
</div> </div>
<div class="actions" style="margin-bottom: 1rem;"> <div class="actions" style="margin-bottom: 1rem;">
<button class="btn-save" @click="saveCaldav" :disabled="savingCaldav"> <button class="btn-save" @click="saveCaldav" :disabled="savingCaldav">
{{ savingCaldav ? "Saving..." : "Save" }} {{ savingCaldav ? "Saving..." : "Save" }}
</button> </button>
<span v-if="caldavSaved" class="saved-msg">Saved!</span> <span v-if="caldavSaved" class="saved-msg">Saved!</span>
<button class="btn-save btn-test" @click="testCaldav" :disabled="testingCaldav"> <button class="btn-secondary" @click="testCaldav" :disabled="testingCaldav">
{{ testingCaldav ? "Testing..." : "Test Connection" }} {{ testingCaldav ? "Testing..." : "Test Connection" }}
</button> </button>
</div> </div>
<div v-if="caldavTestResult" class="test-result" :class="{ success: caldavTestResult.success, error: !caldavTestResult.success }"> <div v-if="caldavTestResult" class="test-result" :class="{ success: caldavTestResult.success, error: !caldavTestResult.success }">
<template v-if="caldavTestResult.success"> <template v-if="caldavTestResult.success">
{{ caldavTestResult.message }} {{ caldavTestResult.message }}
@@ -538,18 +611,55 @@ async function handleRestoreFile(event: Event) {
Calendars: {{ caldavTestResult.calendars.join(", ") }} Calendars: {{ caldavTestResult.calendars.join(", ") }}
</div> </div>
</template> </template>
<template v-else> <template v-else>{{ caldavTestResult.error }}</template>
{{ caldavTestResult.error }}
</template>
</div> </div>
</section> </section>
<section v-if="authStore.isAdmin" class="settings-section"> <!-- Web Search full width -->
<section class="settings-section full-width">
<h2>Web Search (SearXNG)</h2>
<template v-if="searxngConfigured">
<p class="section-desc">
Connected to <code class="url-chip">{{ searxngUrl }}</code>.
Test a query below to verify results and rate limiting.
</p>
<div class="search-row">
<input
v-model="searchQuery"
type="text"
class="input"
placeholder="Enter a search query..."
@keydown="onSearchKeydown"
/>
<button class="btn-save" @click="testSearch" :disabled="searchLoading || !searchQuery.trim()">
{{ searchLoading ? "Searching..." : "Search" }}
</button>
</div>
<p v-if="searchError" class="search-error">{{ searchError }}</p>
<ul v-if="searchResults.length" class="search-results">
<li v-for="(r, i) in searchResults" :key="i" class="search-result">
<div class="result-header">
<a :href="r.url" target="_blank" rel="noopener noreferrer" class="result-title">{{ r.title || r.url }}</a>
<span class="result-host">{{ hostname(r.url) }}</span>
</div>
<p v-if="r.snippet" class="result-snippet">{{ r.snippet }}</p>
</li>
</ul>
</template>
<template v-else>
<p class="section-desc not-configured">
Not configured. Set <code>SEARXNG_URL</code> in docker-compose to enable web research from chat.
</p>
</template>
</section>
<!-- Application URL (admin) full width -->
<section v-if="authStore.isAdmin" class="settings-section full-width">
<h2>Application URL</h2> <h2>Application URL</h2>
<p class="section-desc"> <p class="section-desc">
The public URL used in email links (invitations, password resets). Example: https://notes.example.com Public URL used in email links (invitations, password resets). Example: https://notes.example.com
</p> </p>
<div class="field"> <div class="field url-field">
<label for="base-url">Base URL</label> <label for="base-url">Base URL</label>
<input <input
id="base-url" id="base-url"
@@ -567,12 +677,10 @@ async function handleRestoreFile(event: Event) {
</div> </div>
</section> </section>
<section v-if="authStore.isAdmin" class="settings-section"> <!-- SMTP (admin) full width -->
<section v-if="authStore.isAdmin" class="settings-section full-width">
<h2>Email / SMTP</h2> <h2>Email / SMTP</h2>
<p class="section-desc"> <p class="section-desc">Configure SMTP to enable email notifications for all users.</p>
Configure SMTP settings to enable email notifications for all users.
</p>
<div class="smtp-grid"> <div class="smtp-grid">
<div class="field"> <div class="field">
<label for="smtp-host">SMTP Host</label> <label for="smtp-host">SMTP Host</label>
@@ -599,22 +707,19 @@ async function handleRestoreFile(event: Event) {
<input id="smtp-from-name" v-model="smtp.smtp_from_name" type="text" placeholder="Fabled Assistant" class="input" /> <input id="smtp-from-name" v-model="smtp.smtp_from_name" type="text" placeholder="Fabled Assistant" class="input" />
</div> </div>
</div> </div>
<div class="checkbox-field"> <div class="checkbox-field">
<label> <label>
<input type="checkbox" :checked="smtp.smtp_use_tls === 'true'" @change="smtp.smtp_use_tls = ($event.target as HTMLInputElement).checked ? 'true' : 'false'" /> <input type="checkbox" :checked="smtp.smtp_use_tls === 'true'" @change="smtp.smtp_use_tls = ($event.target as HTMLInputElement).checked ? 'true' : 'false'" />
Use STARTTLS Use STARTTLS
</label> </label>
<p class="field-hint">Enable STARTTLS encryption (recommended for port 587). Implicit TLS is used automatically for port 465.</p> <p class="field-hint">Recommended for port 587. Implicit TLS is used automatically for port 465.</p>
</div> </div>
<div class="actions" style="margin-bottom: 1.25rem;"> <div class="actions" style="margin-bottom: 1.25rem;">
<button class="btn-save" @click="saveSmtp" :disabled="savingSmtp"> <button class="btn-save" @click="saveSmtp" :disabled="savingSmtp">
{{ savingSmtp ? "Saving..." : "Save SMTP Settings" }} {{ savingSmtp ? "Saving..." : "Save SMTP Settings" }}
</button> </button>
<span v-if="smtpSaved" class="saved-msg">Saved!</span> <span v-if="smtpSaved" class="saved-msg">Saved!</span>
</div> </div>
<div class="test-email-section"> <div class="test-email-section">
<h3 class="subsection-label">Test Email</h3> <h3 class="subsection-label">Test Email</h3>
<div class="test-email-row"> <div class="test-email-row">
@@ -623,7 +728,6 @@ async function handleRestoreFile(event: Event) {
type="email" type="email"
placeholder="test@example.com" placeholder="test@example.com"
class="input" class="input"
style="flex: 1;"
/> />
<button class="btn-save" @click="sendTestEmail" :disabled="sendingTest || !testRecipient.trim()"> <button class="btn-save" @click="sendTestEmail" :disabled="sendingTest || !testRecipient.trim()">
{{ sendingTest ? "Sending..." : "Send Test" }} {{ sendingTest ? "Sending..." : "Send Test" }}
@@ -632,192 +736,158 @@ async function handleRestoreFile(event: Event) {
</div> </div>
</section> </section>
<section class="settings-section">
<h2>Data</h2>
<p class="section-desc">Export your data or restore from a backup.</p>
<div class="data-actions">
<button
class="btn-export"
@click="exportData('user')"
:disabled="exporting"
>
{{ exporting ? "Exporting..." : "Export My Data" }}
</button>
<template v-if="authStore.isAdmin">
<button
class="btn-export"
@click="exportData('full')"
:disabled="exporting"
>
{{ exporting ? "Exporting..." : "Export Full Backup" }}
</button>
<button
class="btn-restore"
@click="triggerRestoreUpload"
:disabled="restoring"
>
{{ restoring ? "Restoring..." : "Restore from Backup" }}
</button>
<input
ref="restoreFileInput"
type="file"
accept=".json"
class="hidden-file-input"
@change="handleRestoreFile"
/>
</template>
</div> </div>
</section>
</main> </main>
</template> </template>
<style scoped> <style scoped>
.settings-page { .settings-page {
max-width: 1200px; max-width: 1100px;
margin: 2rem auto; margin: 2rem auto;
padding: 0 1rem; padding: 0 1.5rem;
} }
.settings-page h1 { .settings-page h1 {
margin: 0 0 1.5rem; margin: 0 0 1.5rem;
} }
/* Two-column grid — small sections pair up, full-width sections span both */
.settings-grid {
display: grid;
grid-template-columns: 1fr 1fr;
gap: 1.25rem;
align-items: start;
}
.settings-section { .settings-section {
background: var(--color-bg-card); background: var(--color-bg-card);
border: 1px solid var(--color-border); border: 1px solid var(--color-border);
border-radius: var(--radius-md); border-radius: var(--radius-md);
padding: 1.25rem; padding: 1.25rem;
margin-bottom: 1.5rem; }
.settings-section.full-width {
grid-column: 1 / -1;
} }
.settings-section h2 { .settings-section h2 {
margin: 0 0 0.5rem; margin: 0 0 0.5rem;
font-size: 1.1rem; font-size: 1.05rem;
} }
.section-desc { .section-desc {
margin: 0 0 1rem; margin: 0 0 1rem;
font-size: 0.9rem; font-size: 0.875rem;
color: var(--color-text-secondary); color: var(--color-text-secondary);
line-height: 1.5;
} }
/* Assistant — 3-col internal grid */
.assistant-grid {
display: grid;
grid-template-columns: 1fr 1fr 1fr;
gap: 0 1rem;
}
@media (max-width: 700px) {
.assistant-grid {
grid-template-columns: 1fr;
}
}
.field { .field {
margin-bottom: 1rem; margin-bottom: 1rem;
} }
.field label { .field label {
display: block; display: block;
font-size: 0.9rem; font-size: 0.875rem;
font-weight: 600; font-weight: 600;
margin-bottom: 0.35rem; margin-bottom: 0.35rem;
color: var(--color-text); color: var(--color-text);
} }
.input { .input {
width: 100%; width: 100%;
padding: 0.5rem 0.75rem; padding: 0.45rem 0.7rem;
border: 1px solid var(--color-border); border: 1px solid var(--color-border);
border-radius: var(--radius-sm); border-radius: var(--radius-sm);
font-size: 0.95rem; font-size: 0.9rem;
background: var(--color-bg); background: var(--color-bg);
color: var(--color-text); color: var(--color-text);
box-sizing: border-box; box-sizing: border-box;
font-family: inherit;
} }
.input:focus { .input:focus {
outline: none; outline: none;
border-color: var(--color-primary); border-color: var(--color-primary);
} }
.field-hint { .field-hint {
margin: 0.35rem 0 0; margin: 0.3rem 0 0;
font-size: 0.8rem; font-size: 0.78rem;
color: var(--color-text-muted); color: var(--color-text-muted);
} }
.actions { .actions {
display: flex; display: flex;
align-items: center; align-items: center;
gap: 0.75rem; gap: 0.65rem;
flex-wrap: wrap;
} }
.btn-save { .btn-save {
padding: 0.45rem 1rem; padding: 0.4rem 0.9rem;
background: var(--color-primary); background: var(--color-primary);
color: #fff; color: #fff;
border: none; border: none;
border-radius: var(--radius-sm); border-radius: var(--radius-sm);
cursor: pointer; cursor: pointer;
font-size: 0.9rem; font-size: 0.875rem;
font-family: inherit;
white-space: nowrap;
} }
.btn-save:disabled { .btn-save:disabled { opacity: 0.6; cursor: default; }
opacity: 0.6; .btn-save:hover:not(:disabled) { opacity: 0.9; }
cursor: default;
.btn-secondary {
padding: 0.4rem 0.9rem;
background: var(--color-bg-secondary);
color: var(--color-text);
border: 1px solid var(--color-border);
border-radius: var(--radius-sm);
cursor: pointer;
font-size: 0.875rem;
font-family: inherit;
white-space: nowrap;
} }
.btn-save:hover:not(:disabled) { .btn-secondary:hover:not(:disabled) {
opacity: 0.9; border-color: var(--color-primary);
color: var(--color-primary);
} }
.btn-secondary:disabled { opacity: 0.6; cursor: default; }
.btn-warn:hover:not(:disabled) {
border-color: var(--color-warning);
color: var(--color-warning);
}
.saved-msg { .saved-msg {
color: var(--color-success); color: var(--color-success);
font-size: 0.9rem; font-size: 0.875rem;
font-weight: 600; font-weight: 600;
} }
.input-error { .input-error { border-color: var(--color-danger); }
border-color: var(--color-danger); .input-error:focus { border-color: var(--color-danger); }
}
.input-error:focus {
border-color: var(--color-danger);
}
.error-hint { .error-hint {
margin: 0.35rem 0 0; margin: 0.3rem 0 0;
font-size: 0.8rem; font-size: 0.78rem;
color: var(--color-danger); color: var(--color-danger);
} }
/* Data section */ /* Data buttons */
.data-actions { .data-actions {
display: flex; display: flex;
flex-wrap: wrap; flex-wrap: wrap;
gap: 0.5rem; gap: 0.5rem;
} }
.btn-export { .hidden-file-input { display: none; }
padding: 0.45rem 1rem;
background: var(--color-bg-secondary);
color: var(--color-text);
border: 1px solid var(--color-border);
border-radius: var(--radius-sm);
cursor: pointer;
font-size: 0.9rem;
}
.btn-export:hover:not(:disabled) {
border-color: var(--color-primary);
color: var(--color-primary);
}
.btn-export:disabled {
opacity: 0.6;
cursor: default;
}
.btn-restore {
padding: 0.45rem 1rem;
background: var(--color-bg-secondary);
color: var(--color-text);
border: 1px solid var(--color-border);
border-radius: var(--radius-sm);
cursor: pointer;
font-size: 0.9rem;
}
.btn-restore:hover:not(:disabled) {
border-color: var(--color-warning);
color: var(--color-warning);
}
.btn-restore:disabled {
opacity: 0.6;
cursor: default;
}
.hidden-file-input {
display: none;
}
/* Checkbox fields */ /* Checkboxes */
.checkbox-field { .checkbox-field { margin-bottom: 1rem; }
margin-bottom: 1rem;
}
.checkbox-field label { .checkbox-field label {
display: flex; display: flex;
align-items: center; align-items: center;
gap: 0.5rem; gap: 0.5rem;
font-size: 0.95rem; font-size: 0.9rem;
color: var(--color-text); color: var(--color-text);
cursor: pointer; cursor: pointer;
} }
@@ -830,28 +900,20 @@ async function handleRestoreFile(event: Event) {
/* CalDAV grid */ /* CalDAV grid */
.caldav-grid { .caldav-grid {
display: grid; display: grid;
grid-template-columns: 1fr 1fr; grid-template-columns: 1fr 1fr 1fr 1fr;
gap: 0 1rem; gap: 0 1rem;
} }
@media (max-width: 700px) {
.caldav-grid { grid-template-columns: 1fr 1fr; }
}
@media (max-width: 480px) { @media (max-width: 480px) {
.caldav-grid { .caldav-grid { grid-template-columns: 1fr; }
grid-template-columns: 1fr;
}
}
.btn-test {
background: var(--color-bg-secondary);
color: var(--color-text);
border: 1px solid var(--color-border);
}
.btn-test:hover:not(:disabled) {
border-color: var(--color-primary);
color: var(--color-primary);
opacity: 1;
} }
.test-result { .test-result {
padding: 0.75rem 1rem; padding: 0.65rem 0.9rem;
border-radius: var(--radius-sm); border-radius: var(--radius-sm);
font-size: 0.9rem; font-size: 0.875rem;
line-height: 1.4; line-height: 1.4;
} }
.test-result.success { .test-result.success {
@@ -865,27 +927,100 @@ async function handleRestoreFile(event: Event) {
color: var(--color-danger); color: var(--color-danger);
} }
.test-calendars { .test-calendars {
margin-top: 0.35rem; margin-top: 0.3rem;
font-size: 0.85rem; font-size: 0.82rem;
opacity: 0.85; opacity: 0.85;
} }
/* Search test */
.url-chip {
font-size: 0.8rem;
padding: 0.1rem 0.4rem;
background: var(--color-bg-secondary);
border: 1px solid var(--color-border);
border-radius: 4px;
}
.not-configured {
font-style: italic;
opacity: 0.75;
}
.search-row {
display: flex;
gap: 0.5rem;
align-items: center;
margin-bottom: 0.75rem;
}
.search-row .input { flex: 1; }
.search-error {
font-size: 0.875rem;
color: var(--color-danger);
margin: 0 0 0.5rem;
}
.search-results {
list-style: none;
margin: 0;
padding: 0;
display: flex;
flex-direction: column;
gap: 0.75rem;
}
.search-result {
padding: 0.65rem 0.85rem;
background: var(--color-bg-secondary);
border: 1px solid var(--color-border);
border-radius: var(--radius-sm);
}
.result-header {
display: flex;
align-items: baseline;
gap: 0.5rem;
flex-wrap: wrap;
margin-bottom: 0.25rem;
}
.result-title {
font-size: 0.9rem;
font-weight: 600;
color: var(--color-primary);
text-decoration: none;
word-break: break-word;
}
.result-title:hover { text-decoration: underline; }
.result-host {
font-size: 0.75rem;
color: var(--color-text-muted);
white-space: nowrap;
}
.result-snippet {
margin: 0;
font-size: 0.82rem;
color: var(--color-text-secondary);
line-height: 1.45;
display: -webkit-box;
-webkit-line-clamp: 2;
-webkit-box-orient: vertical;
overflow: hidden;
}
/* Application URL — constrain input width */
.url-field { max-width: 480px; }
/* SMTP grid */ /* SMTP grid */
.smtp-grid { .smtp-grid {
display: grid; display: grid;
grid-template-columns: 1fr 1fr; grid-template-columns: 1fr 1fr 1fr;
gap: 0 1rem; gap: 0 1rem;
} }
@media (max-width: 480px) { @media (max-width: 700px) {
.smtp-grid { .smtp-grid { grid-template-columns: 1fr 1fr; }
grid-template-columns: 1fr;
} }
@media (max-width: 480px) {
.smtp-grid { grid-template-columns: 1fr; }
} }
/* Test email */ /* Test email */
.subsection-label { .subsection-label {
margin: 0 0 0.5rem; margin: 0 0 0.5rem;
font-size: 0.9rem; font-size: 0.875rem;
font-weight: 600; font-weight: 600;
color: var(--color-text-secondary); color: var(--color-text-secondary);
} }
@@ -897,5 +1032,20 @@ async function handleRestoreFile(event: Event) {
display: flex; display: flex;
gap: 0.5rem; gap: 0.5rem;
align-items: center; align-items: center;
max-width: 480px;
}
/* Responsive — collapse to 1 column on narrow screens */
@media (max-width: 640px) {
.settings-grid {
grid-template-columns: 1fr;
}
.settings-section.full-width {
grid-column: 1;
}
.settings-page {
padding: 0 1rem;
margin: 1rem auto;
}
} }
</style> </style>
+7
View File
@@ -56,6 +56,13 @@ class Config:
OIDC_SCOPES: str = os.environ.get("OIDC_SCOPES", "openid profile email") OIDC_SCOPES: str = os.environ.get("OIDC_SCOPES", "openid profile email")
LOCAL_AUTH_ENABLED: bool = os.environ.get("LOCAL_AUTH_ENABLED", "true").lower() not in ("0", "false", "no") LOCAL_AUTH_ENABLED: bool = os.environ.get("LOCAL_AUTH_ENABLED", "true").lower() not in ("0", "false", "no")
# SearXNG web search (external instance)
SEARXNG_URL: str = os.environ.get("SEARXNG_URL", "")
@classmethod @classmethod
def oidc_enabled(cls) -> bool: def oidc_enabled(cls) -> bool:
return bool(cls.OIDC_ISSUER and cls.OIDC_CLIENT_ID and cls.OIDC_CLIENT_SECRET) return bool(cls.OIDC_ISSUER and cls.OIDC_CLIENT_ID and cls.OIDC_CLIENT_SECRET)
@classmethod
def searxng_enabled(cls) -> bool:
return bool(cls.SEARXNG_URL)
+14
View File
@@ -84,3 +84,17 @@ async def test_caldav():
uid = get_current_user_id() uid = get_current_user_id()
result = await test_connection(uid) result = await test_connection(uid)
return jsonify(result) return jsonify(result)
@settings_bp.route("/search", methods=["GET"])
@login_required
async def test_search():
"""Test SearXNG connectivity and preview results for a query."""
if not Config.searxng_enabled():
return jsonify({"configured": False, "results": [], "searxng_url": ""})
q = request.args.get("q", "").strip()
if not q:
return jsonify({"configured": True, "results": [], "searxng_url": Config.SEARXNG_URL})
from fabledassistant.services.research import _search_searxng
results = await _search_searxng(q)
return jsonify({"configured": True, "results": results, "query": q, "searxng_url": Config.SEARXNG_URL})
@@ -28,6 +28,7 @@ from fabledassistant.services.intent import IntentResult, classify_intent
from fabledassistant.services.logging import log_generation from fabledassistant.services.logging import log_generation
from fabledassistant.services.settings import get_setting from fabledassistant.services.settings import get_setting
from fabledassistant.services.tools import get_tools_for_user, execute_tool from fabledassistant.services.tools import get_tools_for_user, execute_tool
from fabledassistant.services.research import run_research_pipeline
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -59,6 +60,8 @@ _TOOL_LABELS: dict[str, str] = {
"update_todo": "Updating todo", "update_todo": "Updating todo",
"complete_todo": "Completing todo", "complete_todo": "Completing todo",
"delete_todo": "Removing todo", "delete_todo": "Removing todo",
"search_web": "Searching the web",
"research_topic": "Researching topic",
} }
# Tools that write data and require explicit user confirmation before executing. # Tools that write data and require explicit user confirmation before executing.
@@ -285,6 +288,43 @@ async def run_generation(
if timing["ttft_ms"] is None: if timing["ttft_ms"] is None:
timing["ttft_ms"] = int((time.monotonic() - t_start) * 1000) timing["ttft_ms"] = int((time.monotonic() - t_start) * 1000)
if tool_name == "research_topic":
topic = intent.arguments.get("topic", "")
if not ack_text:
fallback_ack = f"I'll research '{topic}' and compile a note.\n\n"
buf.append_event("chunk", {"chunk": fallback_ack})
buf.content_so_far += fallback_ack
if timing["ttft_ms"] is None:
timing["ttft_ms"] = int((time.monotonic() - t_start) * 1000)
try:
note = await run_research_pipeline(
topic, user_id, model, intent_model, buf
)
done_text = (
f"Research complete! I've compiled a note: "
f"**[{note.title}](/notes/{note.id})**."
)
buf.append_event("chunk", {"chunk": done_text})
buf.content_so_far += done_text
tool_record = {
"function": "research_topic",
"arguments": {"topic": topic},
"result": {
"success": True,
"type": "research_note",
"data": {"id": note.id, "title": note.title},
},
"status": "success",
}
all_tool_calls.append(tool_record)
buf.append_event("tool_call", {"tool_call": tool_record})
except Exception as e:
logger.exception("Research pipeline failed for topic: %s", topic)
err_text = f"\nResearch failed: {e}"
buf.append_event("chunk", {"chunk": err_text})
buf.content_so_far += err_text
break # research IS the full response
confirmed = True confirmed = True
if tool_name in _WRITE_TOOLS: if tool_name in _WRITE_TOOLS:
loop = asyncio.get_running_loop() loop = asyncio.get_running_loop()
+6
View File
@@ -100,6 +100,12 @@ Rules:
- "read", "open", "show me", "what does X say", "display", "pull up" a specific note → use get_note with query=<note name>. - "read", "open", "show me", "what does X say", "display", "pull up" a specific note → use get_note with query=<note name>.
- "list my notes", "show notes", "recent notes", "browse notes", "notes tagged X" → use list_notes (with optional q or tags). - "list my notes", "show notes", "recent notes", "browse notes", "notes tagged X" → use list_notes (with optional q or tags).
- "tag X with Y", "add tag Y to X", "untag Y from X", "remove tag Y from X" → use update_note with tags=[Y] and tag_mode="add" or "remove". - "tag X with Y", "add tag Y to X", "untag Y from X", "remove tag Y from X" → use update_note with tags=[Y] and tag_mode="add" or "remove".
- search_web: user wants a quick web search to answer a factual question
("search for X", "look up X", "what is the latest version of X", "find X online",
"google X", "what is X" for quick factual answers — NOT when they want a comprehensive note)
- research_topic: user wants to research a topic and create a comprehensive note from web sources
("research X", "research X and make a note", "compile notes on X", "write a report on X",
"deep dive into X", "find everything about X", "comprehensive guide to X")
- "ack": one short, natural sentence confirming the action (tool path only). Vary phrasing — do not always start with "Let me". Omit (null) for chat-only responses. - "ack": one short, natural sentence confirming the action (tool path only). Vary phrasing — do not always start with "Let me". Omit (null) for chat-only responses.
- Do NOT wrap the JSON in markdown code fences.""" - Do NOT wrap the JSON in markdown code fences."""
+11 -2
View File
@@ -208,12 +208,21 @@ async def stream_chat_with_tools(
break break
async def generate_completion(messages: list[dict], model: str, max_tokens: int = 4096) -> str: async def generate_completion(
messages: list[dict],
model: str,
max_tokens: int = 4096,
num_ctx: int | None = None,
) -> str:
"""Non-streaming chat completion, returns full response text. """Non-streaming chat completion, returns full response text.
Retries up to 2 times on Ollama 500 errors (cold model loading race). Retries up to 2 times on Ollama 500 errors (cold model loading race).
num_ctx overrides the model's context window for this call only.
""" """
last_exc: Exception | None = None last_exc: Exception | None = None
options: dict = {"num_predict": max_tokens}
if num_ctx is not None:
options["num_ctx"] = num_ctx
for attempt in range(3): for attempt in range(3):
if attempt > 0: if attempt > 0:
delay = 3.0 * attempt delay = 3.0 * attempt
@@ -230,7 +239,7 @@ async def generate_completion(messages: list[dict], model: str, max_tokens: int
"messages": messages, "messages": messages,
"stream": False, "stream": False,
"think": False, "think": False,
"options": {"num_predict": max_tokens}, "options": options,
}, },
) )
resp.raise_for_status() resp.raise_for_status()
+242
View File
@@ -0,0 +1,242 @@
"""Web research pipeline: sub-queries → SearXNG → fetch → synthesize → note."""
import asyncio
import json
import logging
import re
import httpx
from fabledassistant.config import Config
from fabledassistant.services.llm import fetch_url_content, generate_completion
from fabledassistant.services.notes import create_note
from fabledassistant.models.note import Note
logger = logging.getLogger(__name__)
SEARXNG_QUERIES = 5 # sub-queries to generate
RESULTS_PER_QUERY = 3 # results fetched from SearXNG per query
PAGES_PER_QUERY = 3 # pages actually read per sub-query (top N results)
MAX_SYNTHESIS_SOURCES = 12 # deduplicated sources passed to synthesis LLM
CHARS_PER_SOURCE = 2000 # content chars per source sent to synthesis
async def run_research_pipeline(
topic: str,
user_id: int,
model: str,
intent_model: str,
buf,
) -> Note:
"""Full research pipeline: search → fetch → synthesize → create note.
Emits status events via buf.append_event throughout.
Returns the created Note.
"""
# Step 1: Generate sub-queries
buf.append_event("status", {"status": "Generating search queries..."})
queries = await _generate_sub_queries(topic, intent_model)
logger.info("Research: generated %d sub-queries for topic '%s'", len(queries), topic)
# Step 2: Search and fetch
all_sources: list[dict] = []
seen_urls: set[str] = set()
for i, query in enumerate(queries):
if i > 0:
await asyncio.sleep(1.0) # avoid hammering SearXNG
buf.append_event("status", {"status": f"Searching: {query}..."})
results = await _search_searxng(query)
logger.info("Research: query '%s'%d results", query, len(results))
for result in results[:PAGES_PER_QUERY]:
url = result.get("url", "")
if not url or url in seen_urls:
continue
seen_urls.add(url)
title = result.get("title", url)
buf.append_event("status", {"status": f"Reading: {title[:60]}..."})
content = await fetch_url_content(url)
all_sources.append({
"url": url,
"title": title,
"query": query,
"snippet": result.get("snippet", ""),
"content": content,
})
if not all_sources:
raise ValueError(f"No results found for '{topic}'")
# Step 3: Filter failed fetches
good_sources = [
s for s in all_sources
if not s["content"].startswith("[Failed to fetch")
]
if not good_sources:
raise ValueError(f"Could not read any sources for '{topic}'")
# Limit to top N sources for synthesis (already deduplicated by URL)
synthesis_sources = good_sources[:MAX_SYNTHESIS_SOURCES]
logger.info(
"Research: %d/%d sources successfully fetched, using %d for synthesis",
len(good_sources), len(all_sources), len(synthesis_sources),
)
# Step 4: Synthesize
buf.append_event("status", {"status": f"Synthesizing report from {len(synthesis_sources)} sources..."})
title, body = await _synthesize_note(topic, synthesis_sources, model)
# Step 5: Create note
buf.append_event("status", {"status": "Saving note..."})
note = await create_note(
user_id=user_id,
title=title,
body=body,
tags=["research"],
)
logger.info("Research: created note id=%d title='%s'", note.id, note.title)
return note
async def _generate_sub_queries(topic: str, intent_model: str) -> list[str]:
"""Ask the intent model for focused search queries for the topic."""
messages = [
{
"role": "system",
"content": (
f"You are a research assistant. Given a research topic, generate exactly {SEARXNG_QUERIES} "
"focused web search queries that together would provide comprehensive coverage of the topic. "
"Vary the angle of each query: include overview, implementation details, best practices, "
"common problems, and real-world examples. "
"Respond with ONLY a JSON array of strings, no other text. "
'Example: ["query one", "query two", "query three"]'
),
},
{"role": "user", "content": f"Topic: {topic}"},
]
try:
raw = await generate_completion(messages, intent_model, max_tokens=200)
raw = raw.strip()
raw = re.sub(r"^```(?:json)?\s*", "", raw)
raw = re.sub(r"\s*```$", "", raw)
idx = raw.find("[")
if idx >= 0:
parsed, _ = json.JSONDecoder().raw_decode(raw[idx:])
if isinstance(parsed, list) and parsed:
queries = [str(q).strip() for q in parsed if str(q).strip()]
if queries:
return queries[:SEARXNG_QUERIES]
except Exception:
logger.warning("Sub-query generation failed, falling back to topic", exc_info=True)
return [topic]
async def _search_searxng(query: str) -> list[dict]:
"""Search SearXNG and return top results as [{url, title, snippet}]."""
url = Config.SEARXNG_URL.rstrip("/") + "/search"
params = {"q": query, "format": "json", "categories": "general"}
for attempt in range(3):
try:
async with httpx.AsyncClient(timeout=10.0) as client:
resp = await client.get(url, params=params)
if resp.status_code == 429:
retry_after = int(resp.headers.get("Retry-After", "5"))
wait = min(retry_after, 10) * (attempt + 1)
logger.warning(
"SearXNG 429 for query '%s' (attempt %d/3), waiting %ds",
query, attempt + 1, wait,
)
await asyncio.sleep(wait)
continue
resp.raise_for_status()
data = resp.json()
results = data.get("results", [])
out = []
for r in results[:RESULTS_PER_QUERY]:
out.append({
"url": r.get("url", ""),
"title": r.get("title", ""),
"snippet": r.get("content", ""),
})
return out
except httpx.HTTPStatusError:
logger.warning("SearXNG search failed for query '%s'", query, exc_info=True)
return []
except Exception:
logger.warning("SearXNG search failed for query '%s'", query, exc_info=True)
return []
logger.warning("SearXNG search gave up after 3 attempts for query '%s'", query)
return []
async def _synthesize_note(
topic: str,
sources: list[dict],
model: str,
) -> tuple[str, str]:
"""Synthesize a comprehensive markdown research document from fetched sources.
Returns (title, body_markdown).
Uses an extended context window so the output can be several thousand words.
"""
sources_text_parts = []
for i, s in enumerate(sources, 1):
content = (s.get("content") or s.get("snippet") or "")[:CHARS_PER_SOURCE]
sources_text_parts.append(
f"[Source {i}] {s['title']}\nURL: {s['url']}\nSearch query: {s['query']}\n\n{content}"
)
sources_block = "\n\n" + ("" * 60) + "\n\n".join(sources_text_parts)
messages = [
{
"role": "system",
"content": (
"You are a thorough researcher and writer. "
"Your task is to write an exhaustive, well-structured document on the given topic — "
"not a brief summary or intro paragraph.\n\n"
"Requirements:\n"
"- Write at least 2500 words of substantive content (excluding the Sources section)\n"
"- Choose sections (##) that make sense for the topic — let the subject matter determine the structure. "
"A technical topic might need implementation, configuration, and troubleshooting sections. "
"A comparison topic might need dedicated sections per subject being compared plus a summary. "
"A scientific topic might need background, mechanisms, research findings, and implications. "
"Use your judgment — minimum 6 major sections.\n"
"- Use ### for subsections where they add clarity\n"
"- Write in detailed prose paragraphs — do not reduce sections to bullet-point lists\n"
"- Include specific details, examples, data points, comparisons, and nuance from the sources\n"
"- Do not pad with vague generalities — every paragraph should say something concrete\n"
"- The first line must be the document title starting with '# '\n"
"- End with a '## Sources' section listing every source as a markdown hyperlink\n\n"
"The reader wants to finish this document with a thorough understanding of the topic, "
"not just an overview."
),
},
{
"role": "user",
"content": (
f"Write a comprehensive reference document on: {topic}\n\n"
f"Sources ({len(sources)} pages fetched):\n{sources_block}"
),
},
]
raw = await generate_completion(
messages,
model,
max_tokens=8192,
num_ctx=16384,
)
raw = raw.strip()
# Extract title from first # heading
lines = raw.splitlines()
title = f"Research: {topic}"
body_lines = lines
if lines and lines[0].startswith("# "):
title = lines[0][2:].strip()
body_lines = lines[1:]
body = "\n".join(body_lines).strip()
return title, body
+77
View File
@@ -19,6 +19,7 @@ from fabledassistant.services.caldav import (
update_event, update_event,
update_todo, update_todo,
) )
from fabledassistant.config import Config
from fabledassistant.services.notes import create_note, delete_note, get_note_by_title, list_notes, update_note from fabledassistant.services.notes import create_note, delete_note, get_note_by_title, list_notes, update_note
from fabledassistant.services.tag_suggestions import suggest_tags from fabledassistant.services.tag_suggestions import suggest_tags
@@ -663,11 +664,61 @@ _CALDAV_TOOLS = [
] ]
_SEARCH_TOOLS = [
{
"type": "function",
"function": {
"name": "search_web",
"description": (
"Search the web for quick information and answer the user's question from results. "
"Use for factual lookups, current events, version numbers, quick definitions, or any question "
"where a fast web answer is needed without creating a note. "
"Use research_topic instead when the user explicitly wants a comprehensive note or deep research."
),
"parameters": {
"type": "object",
"properties": {
"query": {"type": "string", "description": "The search query"}
},
"required": ["query"],
},
},
}
]
_RESEARCH_TOOLS = [
{
"type": "function",
"function": {
"name": "research_topic",
"description": (
"Research a topic by searching the web and compiling a note with cited sources. "
"Use for 'research X', 'look up X', 'find information about X', "
"'compile notes on X'. Takes 30120 seconds."
),
"parameters": {
"type": "object",
"properties": {
"topic": {
"type": "string",
"description": "The topic or question to research",
}
},
"required": ["topic"],
},
},
}
]
async def get_tools_for_user(user_id: int) -> list[dict]: async def get_tools_for_user(user_id: int) -> list[dict]:
"""Build the tool list for a user based on their configured integrations.""" """Build the tool list for a user based on their configured integrations."""
tools = list(_CORE_TOOLS) tools = list(_CORE_TOOLS)
if await is_caldav_configured(user_id): if await is_caldav_configured(user_id):
tools.extend(_CALDAV_TOOLS) tools.extend(_CALDAV_TOOLS)
if Config.searxng_enabled():
tools.extend(_SEARCH_TOOLS)
tools.extend(_RESEARCH_TOOLS)
logger.debug("User %d: %d tools available", user_id, len(tools)) logger.debug("User %d: %d tools available", user_id, len(tools))
return tools return tools
@@ -1128,6 +1179,32 @@ async def execute_tool(user_id: int, tool_name: str, arguments: dict) -> dict:
"data": {"id": note.id, "title": note.title}, "data": {"id": note.id, "title": note.title},
} }
elif tool_name == "search_web":
from fabledassistant.services.research import _search_searxng
query = arguments.get("query", "")
results = await _search_searxng(query)
if not results:
return {"success": False, "error": f"No results found for '{query}'"}
return {
"success": True,
"type": "web_search",
"data": {
"query": query,
"results": results,
"count": len(results),
},
}
elif tool_name == "research_topic":
# Research is always handled upstream in generation_task.py (round 0).
# This fallback exists in case it somehow reaches execute_tool.
topic = arguments.get("topic", "")
return {
"success": True,
"type": "research_pending",
"data": {"topic": topic},
}
else: else:
return {"success": False, "error": f"Unknown tool: {tool_name}"} return {"success": False, "error": f"Unknown tool: {tool_name}"}