Compare commits
18 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 7657f48f1e | |||
| 61e62a6904 | |||
| 9a252c8dde | |||
| 8db6b4d230 | |||
| d5e6a8f6da | |||
| 06cd3493fd | |||
| 6a8f0e9143 | |||
| 6f8c2201ed | |||
| ec49e24c8d | |||
| cac62c6caf | |||
| 619f069358 | |||
| ddab0db781 | |||
| 443b11f287 | |||
| 3c9e473823 | |||
| 3ac32dc3bc | |||
| 29ef17f4f3 | |||
| fc82f7a0cb | |||
| b743754ec2 |
@@ -0,0 +1,782 @@
|
||||
# Unified Lookup Tool & Wikipedia Integration — Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Replace `search_web` with a unified `lookup` tool (Wikipedia-first, SearXNG fallback) and add Wikipedia as a source in the research pipeline.
|
||||
|
||||
**Architecture:** New `wikipedia.py` service with `wiki_summary` and `wiki_search`. `lookup` tool in `web.py` replaces `search_web`. Research pipeline in `research.py` gains Wikipedia sources alongside SearXNG. All `search_web` references across the codebase are updated.
|
||||
|
||||
**Tech Stack:** Python 3.12, httpx, pytest, asyncio
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Wikipedia Service Module
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fabledassistant/services/wikipedia.py`
|
||||
- Create: `tests/test_wikipedia.py`
|
||||
|
||||
- [ ] **Step 1: Write failing tests for `wiki_summary`**
|
||||
|
||||
```python
|
||||
# tests/test_wikipedia.py
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, patch, MagicMock
|
||||
import httpx
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wiki_summary_returns_extract():
|
||||
from fabledassistant.services.wikipedia import wiki_summary
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {
|
||||
"type": "standard",
|
||||
"title": "Python (programming language)",
|
||||
"extract": "Python is a high-level programming language.",
|
||||
"content_urls": {
|
||||
"desktop": {"page": "https://en.wikipedia.org/wiki/Python_(programming_language)"}
|
||||
},
|
||||
}
|
||||
mock_response.raise_for_status = MagicMock()
|
||||
|
||||
with patch("fabledassistant.services.wikipedia.httpx.AsyncClient") as mock_client_cls:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||||
mock_client.get = AsyncMock(return_value=mock_response)
|
||||
mock_client_cls.return_value = mock_client
|
||||
|
||||
result = await wiki_summary("Python programming language")
|
||||
|
||||
assert result is not None
|
||||
assert result["title"] == "Python (programming language)"
|
||||
assert "high-level" in result["extract"]
|
||||
assert "wikipedia.org" in result["url"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wiki_summary_returns_none_on_404():
|
||||
from fabledassistant.services.wikipedia import wiki_summary
|
||||
|
||||
with patch("fabledassistant.services.wikipedia.httpx.AsyncClient") as mock_client_cls:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||||
mock_client.get = AsyncMock(side_effect=httpx.HTTPStatusError(
|
||||
"Not Found", request=MagicMock(), response=MagicMock(status_code=404)
|
||||
))
|
||||
mock_client_cls.return_value = mock_client
|
||||
|
||||
result = await wiki_summary("xyznonexistenttopic123")
|
||||
|
||||
assert result is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wiki_summary_returns_none_on_disambiguation():
|
||||
from fabledassistant.services.wikipedia import wiki_summary
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {
|
||||
"type": "disambiguation",
|
||||
"title": "Python",
|
||||
"extract": "Python may refer to...",
|
||||
}
|
||||
mock_response.raise_for_status = MagicMock()
|
||||
|
||||
with patch("fabledassistant.services.wikipedia.httpx.AsyncClient") as mock_client_cls:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||||
mock_client.get = AsyncMock(return_value=mock_response)
|
||||
mock_client_cls.return_value = mock_client
|
||||
|
||||
result = await wiki_summary("Python")
|
||||
|
||||
assert result is None
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run tests to verify they fail**
|
||||
|
||||
Run: `uv run pytest tests/test_wikipedia.py -v`
|
||||
Expected: FAIL with `ModuleNotFoundError: No module named 'fabledassistant.services.wikipedia'`
|
||||
|
||||
- [ ] **Step 3: Implement `wiki_summary`**
|
||||
|
||||
```python
|
||||
# src/fabledassistant/services/wikipedia.py
|
||||
"""Wikipedia API: lightweight topic lookups and article search."""
|
||||
|
||||
import logging
|
||||
from urllib.parse import quote as url_quote
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SUMMARY_URL = "https://en.wikipedia.org/api/rest_v1/page/summary"
|
||||
_SEARCH_URL = "https://en.wikipedia.org/w/api.php"
|
||||
_TIMEOUT = 5.0
|
||||
_USER_AGENT = "FabledAssistant/1.0 (https://fabledsword.com)"
|
||||
|
||||
|
||||
async def wiki_summary(query: str) -> dict | None:
|
||||
"""Look up a topic by title via the Wikipedia REST summary endpoint.
|
||||
|
||||
Returns {"title", "extract", "url"} on hit, None on miss.
|
||||
"""
|
||||
encoded = url_quote(query.replace(" ", "_"), safe="")
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=_TIMEOUT, headers={"User-Agent": _USER_AGENT}
|
||||
) as client:
|
||||
resp = await client.get(f"{_SUMMARY_URL}/{encoded}", follow_redirects=True)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
except Exception:
|
||||
logger.debug("Wikipedia summary lookup failed for %r", query, exc_info=True)
|
||||
return None
|
||||
|
||||
if data.get("type") == "disambiguation":
|
||||
return None
|
||||
|
||||
extract = data.get("extract", "").strip()
|
||||
if not extract:
|
||||
return None
|
||||
|
||||
url = (
|
||||
data.get("content_urls", {}).get("desktop", {}).get("page")
|
||||
or f"https://en.wikipedia.org/wiki/{encoded}"
|
||||
)
|
||||
return {"title": data.get("title", query), "extract": extract, "url": url}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run tests to verify they pass**
|
||||
|
||||
Run: `uv run pytest tests/test_wikipedia.py -v`
|
||||
Expected: 3 passed
|
||||
|
||||
- [ ] **Step 5: Write failing tests for `wiki_search`**
|
||||
|
||||
Add to `tests/test_wikipedia.py`:
|
||||
|
||||
```python
|
||||
@pytest.mark.asyncio
|
||||
async def test_wiki_search_returns_results():
|
||||
from fabledassistant.services.wikipedia import wiki_search
|
||||
|
||||
search_response = MagicMock()
|
||||
search_response.status_code = 200
|
||||
search_response.json.return_value = {
|
||||
"query": {
|
||||
"search": [
|
||||
{"title": "QUIC"},
|
||||
{"title": "HTTP/3"},
|
||||
]
|
||||
}
|
||||
}
|
||||
search_response.raise_for_status = MagicMock()
|
||||
|
||||
summary_response = MagicMock()
|
||||
summary_response.status_code = 200
|
||||
summary_response.json.return_value = {
|
||||
"type": "standard",
|
||||
"title": "QUIC",
|
||||
"extract": "QUIC is a transport layer protocol.",
|
||||
"content_urls": {"desktop": {"page": "https://en.wikipedia.org/wiki/QUIC"}},
|
||||
}
|
||||
summary_response.raise_for_status = MagicMock()
|
||||
|
||||
with patch("fabledassistant.services.wikipedia.httpx.AsyncClient") as mock_client_cls:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||||
mock_client.get = AsyncMock(side_effect=[search_response, summary_response, summary_response])
|
||||
mock_client_cls.return_value = mock_client
|
||||
|
||||
results = await wiki_search("QUIC protocol", limit=2)
|
||||
|
||||
assert len(results) >= 1
|
||||
assert results[0]["title"] == "QUIC"
|
||||
assert "transport" in results[0]["extract"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wiki_search_returns_empty_on_failure():
|
||||
from fabledassistant.services.wikipedia import wiki_search
|
||||
|
||||
with patch("fabledassistant.services.wikipedia.httpx.AsyncClient") as mock_client_cls:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||||
mock_client.get = AsyncMock(side_effect=httpx.ConnectError("connection failed"))
|
||||
mock_client_cls.return_value = mock_client
|
||||
|
||||
results = await wiki_search("anything")
|
||||
|
||||
assert results == []
|
||||
```
|
||||
|
||||
- [ ] **Step 6: Implement `wiki_search`**
|
||||
|
||||
Add to `src/fabledassistant/services/wikipedia.py`:
|
||||
|
||||
```python
|
||||
async def wiki_search(query: str, limit: int = 3) -> list[dict]:
|
||||
"""Search Wikipedia for articles matching a query.
|
||||
|
||||
Returns [{"title", "extract", "url"}, ...] (may be empty).
|
||||
"""
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=_TIMEOUT, headers={"User-Agent": _USER_AGENT}
|
||||
) as client:
|
||||
resp = await client.get(_SEARCH_URL, params={
|
||||
"action": "query",
|
||||
"list": "search",
|
||||
"srsearch": query,
|
||||
"srlimit": str(limit),
|
||||
"format": "json",
|
||||
})
|
||||
resp.raise_for_status()
|
||||
hits = resp.json().get("query", {}).get("search", [])
|
||||
if not hits:
|
||||
return []
|
||||
|
||||
results: list[dict] = []
|
||||
for hit in hits:
|
||||
title = hit.get("title", "")
|
||||
if not title:
|
||||
continue
|
||||
encoded = url_quote(title.replace(" ", "_"), safe="")
|
||||
try:
|
||||
summary_resp = await client.get(
|
||||
f"{_SUMMARY_URL}/{encoded}", follow_redirects=True,
|
||||
)
|
||||
summary_resp.raise_for_status()
|
||||
data = summary_resp.json()
|
||||
except Exception:
|
||||
continue
|
||||
if data.get("type") == "disambiguation":
|
||||
continue
|
||||
extract = data.get("extract", "").strip()
|
||||
if not extract:
|
||||
continue
|
||||
url = (
|
||||
data.get("content_urls", {}).get("desktop", {}).get("page")
|
||||
or f"https://en.wikipedia.org/wiki/{encoded}"
|
||||
)
|
||||
results.append({"title": data.get("title", title), "extract": extract, "url": url})
|
||||
return results
|
||||
except Exception:
|
||||
logger.debug("Wikipedia search failed for %r", query, exc_info=True)
|
||||
return []
|
||||
```
|
||||
|
||||
- [ ] **Step 7: Run all wikipedia tests**
|
||||
|
||||
Run: `uv run pytest tests/test_wikipedia.py -v`
|
||||
Expected: 5 passed
|
||||
|
||||
- [ ] **Step 8: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fabledassistant/services/wikipedia.py tests/test_wikipedia.py
|
||||
git commit -m "feat: add wikipedia service with summary lookup and search"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Lookup Tool (replaces search_web)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fabledassistant/services/tools/web.py`
|
||||
- Create: `tests/test_lookup_tool.py`
|
||||
|
||||
- [ ] **Step 1: Write failing tests for `lookup`**
|
||||
|
||||
```python
|
||||
# tests/test_lookup_tool.py
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, patch, MagicMock
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_lookup_wikipedia_hit():
|
||||
"""lookup returns wikipedia source when wiki_summary succeeds."""
|
||||
wiki_data = {
|
||||
"title": "QUIC",
|
||||
"extract": "QUIC is a transport layer protocol.",
|
||||
"url": "https://en.wikipedia.org/wiki/QUIC",
|
||||
}
|
||||
|
||||
with patch("fabledassistant.services.tools.web.wiki_summary", new_callable=AsyncMock, return_value=wiki_data):
|
||||
from fabledassistant.services.tools.web import lookup_tool
|
||||
result = await lookup_tool(user_id=1, arguments={"query": "QUIC"})
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["type"] == "lookup"
|
||||
assert result["source"] == "wikipedia"
|
||||
assert result["data"]["title"] == "QUIC"
|
||||
assert "transport" in result["data"]["extract"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_lookup_wikipedia_miss_searxng_fallback():
|
||||
"""lookup falls back to SearXNG + article fetch when Wikipedia misses."""
|
||||
searxng_results = [
|
||||
{"url": "https://example.com/quic", "title": "QUIC Explained", "snippet": "An overview..."},
|
||||
]
|
||||
|
||||
with patch("fabledassistant.services.tools.web.wiki_summary", new_callable=AsyncMock, return_value=None), \
|
||||
patch("fabledassistant.services.tools.web.Config") as mock_config, \
|
||||
patch("fabledassistant.services.tools.web._search_searxng", new_callable=AsyncMock, return_value=searxng_results), \
|
||||
patch("fabledassistant.services.tools.web._fetch_full_article", new_callable=AsyncMock, return_value="Full article about QUIC..."):
|
||||
mock_config.searxng_enabled.return_value = True
|
||||
from fabledassistant.services.tools.web import lookup_tool
|
||||
result = await lookup_tool(user_id=1, arguments={"query": "QUIC"})
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["type"] == "lookup"
|
||||
assert result["source"] == "web"
|
||||
assert result["data"]["results"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_lookup_wikipedia_miss_no_searxng():
|
||||
"""lookup returns no-results when Wikipedia misses and SearXNG is not configured."""
|
||||
with patch("fabledassistant.services.tools.web.wiki_summary", new_callable=AsyncMock, return_value=None), \
|
||||
patch("fabledassistant.services.tools.web.Config") as mock_config:
|
||||
mock_config.searxng_enabled.return_value = False
|
||||
from fabledassistant.services.tools.web import lookup_tool
|
||||
result = await lookup_tool(user_id=1, arguments={"query": "xyznonexistent"})
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["source"] == "none"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_lookup_always_available():
|
||||
"""lookup tool must appear in get_tools_for_user regardless of SearXNG config."""
|
||||
with patch("fabledassistant.services.tools._registry.is_caldav_configured", new_callable=AsyncMock, return_value=False), \
|
||||
patch("fabledassistant.services.settings.get_setting", new_callable=AsyncMock, return_value="false"):
|
||||
from fabledassistant.services.tools import get_tools_for_user
|
||||
tools = await get_tools_for_user(user_id=1)
|
||||
tool_names = {t["function"]["name"] for t in tools}
|
||||
assert "lookup" in tool_names
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run tests to verify they fail**
|
||||
|
||||
Run: `uv run pytest tests/test_lookup_tool.py -v`
|
||||
Expected: FAIL (no `lookup_tool` function)
|
||||
|
||||
- [ ] **Step 3: Replace `search_web` with `lookup` in `web.py`**
|
||||
|
||||
Replace the `search_web_tool` function (lines 12–36 of `src/fabledassistant/services/tools/web.py`) with:
|
||||
|
||||
```python
|
||||
@tool(
|
||||
name="lookup",
|
||||
description=(
|
||||
"Look up a topic, concept, or factual question. Returns a concise answer from "
|
||||
"Wikipedia or web sources. Use for definitions, explanations, 'what is X', "
|
||||
"'how does Y work', current events, or version numbers. No note is saved. "
|
||||
"For comprehensive written reports saved as notes, use research_topic instead."
|
||||
),
|
||||
parameters={
|
||||
"query": {"type": "string", "description": "The topic or question to look up"},
|
||||
},
|
||||
required=["query"],
|
||||
)
|
||||
async def lookup_tool(*, user_id, arguments, **_ctx):
|
||||
from fabledassistant.config import Config
|
||||
from fabledassistant.services.wikipedia import wiki_summary
|
||||
|
||||
query = arguments.get("query", "")
|
||||
|
||||
# 1. Try Wikipedia first
|
||||
wiki = await wiki_summary(query)
|
||||
if wiki:
|
||||
return {
|
||||
"success": True,
|
||||
"type": "lookup",
|
||||
"source": "wikipedia",
|
||||
"data": wiki,
|
||||
}
|
||||
|
||||
# 2. Fall back to SearXNG + article fetch
|
||||
if Config.searxng_enabled():
|
||||
from fabledassistant.services.research import _search_searxng
|
||||
from fabledassistant.services.rss import _fetch_full_article
|
||||
|
||||
results = await _search_searxng(query)
|
||||
if results:
|
||||
articles: list[dict] = []
|
||||
for r in results[:2]:
|
||||
url = r.get("url", "")
|
||||
if not url:
|
||||
continue
|
||||
content = await _fetch_full_article(url)
|
||||
articles.append({
|
||||
"url": url,
|
||||
"title": r.get("title", url),
|
||||
"snippet": r.get("snippet", ""),
|
||||
"content": (content or "")[:4000],
|
||||
})
|
||||
if articles:
|
||||
return {
|
||||
"success": True,
|
||||
"type": "lookup",
|
||||
"source": "web",
|
||||
"data": {"query": query, "results": articles},
|
||||
}
|
||||
|
||||
# 3. No sources available
|
||||
return {
|
||||
"success": True,
|
||||
"type": "lookup",
|
||||
"source": "none",
|
||||
"data": {
|
||||
"query": query,
|
||||
"message": "No results found. You can answer from your own knowledge.",
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run lookup tests to verify they pass**
|
||||
|
||||
Run: `uv run pytest tests/test_lookup_tool.py -v`
|
||||
Expected: 4 passed
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fabledassistant/services/tools/web.py tests/test_lookup_tool.py
|
||||
git commit -m "feat: replace search_web with unified lookup tool (Wikipedia + SearXNG fallback)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Update All `search_web` References
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fabledassistant/services/tools/web.py:45` (research_topic description)
|
||||
- Modify: `src/fabledassistant/services/tools/web.py:67` (search_images description)
|
||||
- Modify: `src/fabledassistant/services/tools/rss.py:75` (read_article description)
|
||||
- Modify: `src/fabledassistant/services/generation_task.py:133` (status label map)
|
||||
- Modify: `src/fabledassistant/services/llm.py:608` (action list)
|
||||
|
||||
- [ ] **Step 1: Update `research_topic` description**
|
||||
|
||||
In `src/fabledassistant/services/tools/web.py`, change the `research_topic` description from:
|
||||
|
||||
```python
|
||||
"For a quick factual answer without saving a note, use search_web."
|
||||
```
|
||||
|
||||
to:
|
||||
|
||||
```python
|
||||
"For a quick factual answer without saving a note, use lookup."
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Update `search_images` description**
|
||||
|
||||
In `src/fabledassistant/services/tools/web.py`, change the `search_images` description from:
|
||||
|
||||
```python
|
||||
description="Search and display images inline. Use ONLY when the user explicitly asks to see, show, or find an image or photo. Not for factual questions — use search_web for those.",
|
||||
```
|
||||
|
||||
to:
|
||||
|
||||
```python
|
||||
description="Search and display images inline. Use ONLY when the user explicitly asks to see, show, or find an image or photo. Not for factual questions — use lookup for those.",
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Update `read_article` description**
|
||||
|
||||
In `src/fabledassistant/services/tools/rss.py`, change:
|
||||
|
||||
```python
|
||||
"Do NOT use search_web for URLs — use this tool instead."
|
||||
```
|
||||
|
||||
to:
|
||||
|
||||
```python
|
||||
"Do NOT use lookup for URLs — use this tool instead."
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Update status label map in `generation_task.py`**
|
||||
|
||||
In `src/fabledassistant/services/generation_task.py`, line 133, change:
|
||||
|
||||
```python
|
||||
"search_web": "Searching the web",
|
||||
```
|
||||
|
||||
to:
|
||||
|
||||
```python
|
||||
"lookup": "Looking up information",
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Update action list in `llm.py`**
|
||||
|
||||
In `src/fabledassistant/services/llm.py`, line 608, change:
|
||||
|
||||
```python
|
||||
actions.extend(["search_web", "research_topic", "search_images"])
|
||||
```
|
||||
|
||||
to:
|
||||
|
||||
```python
|
||||
actions.extend(["lookup", "research_topic", "search_images"])
|
||||
```
|
||||
|
||||
- [ ] **Step 6: Run full test suite to check for regressions**
|
||||
|
||||
Run: `uv run pytest tests/ -v`
|
||||
Expected: All tests pass (no test references `search_web` by name in assertions)
|
||||
|
||||
- [ ] **Step 7: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fabledassistant/services/tools/web.py src/fabledassistant/services/tools/rss.py src/fabledassistant/services/generation_task.py src/fabledassistant/services/llm.py
|
||||
git commit -m "refactor: update all search_web references to lookup"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Add Wikipedia Sources to Research Pipeline
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fabledassistant/services/research.py`
|
||||
- Modify: `tests/test_research_pipeline.py`
|
||||
|
||||
- [ ] **Step 1: Write failing test for Wikipedia in research pipeline**
|
||||
|
||||
Add to `tests/test_research_pipeline.py`:
|
||||
|
||||
```python
|
||||
@pytest.mark.asyncio
|
||||
async def test_pipeline_includes_wikipedia_sources():
|
||||
"""run_research_pipeline should merge Wikipedia results into the source pool."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
wiki_results = [{"title": "Wiki Article", "extract": "Wikipedia content about the topic.", "url": "https://en.wikipedia.org/wiki/Topic"}]
|
||||
|
||||
outline = [
|
||||
{"title": "Section A", "focus": "Focus A"},
|
||||
{"title": "Section B", "focus": "Focus B"},
|
||||
]
|
||||
|
||||
note_id_counter = iter(range(30, 40))
|
||||
|
||||
def _make_note(user_id, title, body, tags, project_id=None, parent_id=None):
|
||||
n = MagicMock()
|
||||
n.id = next(note_id_counter)
|
||||
n.title = title
|
||||
return n
|
||||
|
||||
with patch("fabledassistant.services.research._generate_sub_queries", new_callable=AsyncMock, return_value=["q1"]), \
|
||||
patch("fabledassistant.services.research._search_searxng", new_callable=AsyncMock, return_value=[{"url": "http://x.com", "title": "X", "snippet": "s"}]), \
|
||||
patch("fabledassistant.services.research.wiki_search", new_callable=AsyncMock, return_value=wiki_results), \
|
||||
patch("fabledassistant.services.research.fetch_url_content", new_callable=AsyncMock, return_value="content"), \
|
||||
patch("fabledassistant.services.research._generate_outline", new_callable=AsyncMock, return_value=outline) as mock_outline, \
|
||||
patch("fabledassistant.services.research._synthesize_section", new_callable=AsyncMock, side_effect=lambda t, f, s, m: (t, f"Body for {t}")), \
|
||||
patch("fabledassistant.services.research._generate_executive_summary", new_callable=AsyncMock, return_value="Summary."), \
|
||||
patch("fabledassistant.services.research.create_note", new_callable=AsyncMock, side_effect=_make_note), \
|
||||
patch("fabledassistant.services.research.update_note", new_callable=AsyncMock):
|
||||
|
||||
from fabledassistant.services.research import run_research_pipeline
|
||||
await run_research_pipeline("test topic", user_id=1, model="test-model")
|
||||
|
||||
# The sources passed to _generate_outline should include the Wikipedia article
|
||||
sources_arg = mock_outline.call_args[0][1] # second positional arg
|
||||
source_urls = [s["url"] for s in sources_arg]
|
||||
assert "https://en.wikipedia.org/wiki/Topic" in source_urls
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `uv run pytest tests/test_research_pipeline.py::test_pipeline_includes_wikipedia_sources -v`
|
||||
Expected: FAIL (no `wiki_search` import in research.py)
|
||||
|
||||
- [ ] **Step 3: Add Wikipedia sources to the research pipeline**
|
||||
|
||||
In `src/fabledassistant/services/research.py`, add the import at the top (after existing imports):
|
||||
|
||||
```python
|
||||
from fabledassistant.services.wikipedia import wiki_search
|
||||
```
|
||||
|
||||
Then modify Step 2 (the parallel search section, around lines 208–246). Replace:
|
||||
|
||||
```python
|
||||
# Step 2: Search all queries in parallel (200 ms stagger to avoid hammering SearXNG)
|
||||
async def _search_with_stagger(i: int, query: str) -> tuple[str, list[dict]]:
|
||||
if i > 0:
|
||||
await asyncio.sleep(0.2 * i)
|
||||
_status(f"Searching: {query}...")
|
||||
results = await _search_searxng(query)
|
||||
logger.info("Research: query '%s' → %d results", query, len(results))
|
||||
return query, results
|
||||
|
||||
search_results = await asyncio.gather(
|
||||
*[_search_with_stagger(i, q) for i, q in enumerate(queries)]
|
||||
)
|
||||
|
||||
# Deduplicate URLs across all queries
|
||||
seen_urls: set[str] = set()
|
||||
url_tasks: list[tuple[str, dict, str]] = [] # (url, result_dict, query)
|
||||
for query, results in search_results:
|
||||
for result in results[:PAGES_PER_QUERY]:
|
||||
url = result.get("url", "")
|
||||
if url and url not in seen_urls:
|
||||
seen_urls.add(url)
|
||||
url_tasks.append((url, result, query))
|
||||
|
||||
# Fetch all unique URLs in parallel
|
||||
async def _fetch_source(url: str, result: dict, query: str) -> dict:
|
||||
title = result.get("title", url)
|
||||
_status(f"Reading: {title[:60]}...")
|
||||
content = await fetch_url_content(url)
|
||||
return {
|
||||
"url": url,
|
||||
"title": title,
|
||||
"query": query,
|
||||
"snippet": result.get("snippet", ""),
|
||||
"content": content,
|
||||
}
|
||||
|
||||
all_sources: list[dict] = list(await asyncio.gather(
|
||||
*[_fetch_source(url, result, query) for url, result, query in url_tasks]
|
||||
))
|
||||
```
|
||||
|
||||
with:
|
||||
|
||||
```python
|
||||
# Step 2: Search all queries in parallel (SearXNG + Wikipedia)
|
||||
async def _search_with_stagger(i: int, query: str) -> tuple[str, list[dict]]:
|
||||
if i > 0:
|
||||
await asyncio.sleep(0.2 * i)
|
||||
_status(f"Searching: {query}...")
|
||||
results = await _search_searxng(query)
|
||||
logger.info("Research: query '%s' → %d results", query, len(results))
|
||||
return query, results
|
||||
|
||||
async def _wiki_for_query(query: str) -> list[dict]:
|
||||
return await wiki_search(query, limit=1)
|
||||
|
||||
searxng_task = asyncio.gather(
|
||||
*[_search_with_stagger(i, q) for i, q in enumerate(queries)]
|
||||
)
|
||||
wiki_task = asyncio.gather(
|
||||
*[_wiki_for_query(q) for q in queries]
|
||||
)
|
||||
search_results, wiki_results = await asyncio.gather(searxng_task, wiki_task)
|
||||
|
||||
# Deduplicate URLs across all queries
|
||||
seen_urls: set[str] = set()
|
||||
url_tasks: list[tuple[str, dict, str]] = [] # (url, result_dict, query)
|
||||
wiki_sources: list[dict] = [] # Wikipedia articles (already have content)
|
||||
|
||||
for query, results in search_results:
|
||||
for result in results[:PAGES_PER_QUERY]:
|
||||
url = result.get("url", "")
|
||||
if url and url not in seen_urls:
|
||||
seen_urls.add(url)
|
||||
url_tasks.append((url, result, query))
|
||||
|
||||
# Add Wikipedia results (they already have content via extract)
|
||||
for query, wiki_hits in zip(queries, wiki_results):
|
||||
for hit in wiki_hits:
|
||||
url = hit.get("url", "")
|
||||
if url and url not in seen_urls:
|
||||
seen_urls.add(url)
|
||||
wiki_sources.append({
|
||||
"url": url,
|
||||
"title": hit["title"],
|
||||
"query": query,
|
||||
"snippet": hit["extract"][:200],
|
||||
"content": hit["extract"],
|
||||
})
|
||||
|
||||
# Fetch all unique SearXNG URLs in parallel
|
||||
async def _fetch_source(url: str, result: dict, query: str) -> dict:
|
||||
title = result.get("title", url)
|
||||
_status(f"Reading: {title[:60]}...")
|
||||
content = await fetch_url_content(url)
|
||||
return {
|
||||
"url": url,
|
||||
"title": title,
|
||||
"query": query,
|
||||
"snippet": result.get("snippet", ""),
|
||||
"content": content,
|
||||
}
|
||||
|
||||
fetched_sources: list[dict] = list(await asyncio.gather(
|
||||
*[_fetch_source(url, result, query) for url, result, query in url_tasks]
|
||||
))
|
||||
|
||||
all_sources = wiki_sources + fetched_sources
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run the new test to verify it passes**
|
||||
|
||||
Run: `uv run pytest tests/test_research_pipeline.py::test_pipeline_includes_wikipedia_sources -v`
|
||||
Expected: PASS
|
||||
|
||||
- [ ] **Step 5: Run the full research test suite for regressions**
|
||||
|
||||
Run: `uv run pytest tests/test_research_pipeline.py -v`
|
||||
Expected: All tests pass
|
||||
|
||||
- [ ] **Step 6: Run full test suite**
|
||||
|
||||
Run: `uv run pytest tests/ -v`
|
||||
Expected: All tests pass
|
||||
|
||||
- [ ] **Step 7: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fabledassistant/services/research.py tests/test_research_pipeline.py
|
||||
git commit -m "feat: add Wikipedia as research pipeline source alongside SearXNG"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Lint, Typecheck, Final Verification
|
||||
|
||||
**Files:**
|
||||
- All modified files
|
||||
|
||||
- [ ] **Step 1: Run ruff lint**
|
||||
|
||||
Run: `uv run ruff check src/fabledassistant/services/wikipedia.py src/fabledassistant/services/tools/web.py src/fabledassistant/services/research.py tests/test_wikipedia.py tests/test_lookup_tool.py`
|
||||
Expected: All checks passed (fix any issues if not)
|
||||
|
||||
- [ ] **Step 2: Run typecheck**
|
||||
|
||||
Run: `cd frontend && npx vue-tsc --noEmit` (frontend unchanged, but verify nothing broke)
|
||||
Expected: Clean
|
||||
|
||||
- [ ] **Step 3: Run full test suite one last time**
|
||||
|
||||
Run: `uv run pytest tests/ -v`
|
||||
Expected: All tests pass
|
||||
|
||||
- [ ] **Step 4: Commit any lint fixes if needed**
|
||||
|
||||
```bash
|
||||
git add -u
|
||||
git commit -m "fix: lint cleanup for lookup/wikipedia changes"
|
||||
```
|
||||
@@ -0,0 +1,119 @@
|
||||
# Unified Lookup Tool & Wikipedia Integration
|
||||
|
||||
## Goal
|
||||
|
||||
Replace the fragmented `search_web` tool with a single `lookup` tool that checks Wikipedia first and falls back to SearXNG web search. Add Wikipedia as an additional source in the research pipeline. Result: one lightweight tool for factual questions (always available, no config required), and richer research output.
|
||||
|
||||
## Architecture
|
||||
|
||||
Two changes to the search/knowledge stack:
|
||||
|
||||
1. **New `lookup` tool** replaces `search_web`. Tries Wikipedia REST API summary endpoint first (~200ms, reliable, no config). Falls back to SearXNG + trafilatura article fetch when Wikipedia misses and SearXNG is configured. Always available (no `requires` field).
|
||||
|
||||
2. **Wikipedia sources in research pipeline.** During sub-query execution, `wiki_search` runs alongside `_search_searxng`. Wikipedia articles merge into the source pool and get deduplicated by URL.
|
||||
|
||||
Shared Wikipedia logic lives in a new `wikipedia.py` service module.
|
||||
|
||||
## Components
|
||||
|
||||
### `src/fabledassistant/services/wikipedia.py` (new)
|
||||
|
||||
Two async functions:
|
||||
|
||||
**`wiki_summary(query: str) -> dict | None`**
|
||||
- Direct title lookup via `https://en.wikipedia.org/api/rest_v1/page/summary/{title}`
|
||||
- Returns `{"title": str, "extract": str, "url": str}` on hit
|
||||
- Returns `None` on 404, disambiguation pages (`"type": "disambiguation"`), network errors, or empty extracts
|
||||
- 5-second timeout
|
||||
- User-Agent: `"FabledAssistant/1.0 (https://fabledsword.com)"`
|
||||
|
||||
**`wiki_search(query: str, limit: int = 3) -> list[dict]`**
|
||||
- Search via `https://en.wikipedia.org/w/api.php?action=query&list=search&srsearch={query}&srlimit={limit}&format=json`
|
||||
- For each search result, fetch its summary via the summary endpoint to get the extract
|
||||
- Returns `[{"title": str, "extract": str, "url": str}, ...]`
|
||||
- Returns `[]` on any failure
|
||||
- Same timeout and User-Agent as above
|
||||
|
||||
### `src/fabledassistant/services/tools/web.py` (modified)
|
||||
|
||||
**Remove:** `search_web_tool`
|
||||
|
||||
**Add:** `lookup_tool`
|
||||
|
||||
```
|
||||
@tool(
|
||||
name="lookup",
|
||||
description="Look up a topic, concept, or factual question. Returns a concise
|
||||
answer from Wikipedia or web sources. Use for definitions,
|
||||
explanations, 'what is X', 'how does Y work'. For comprehensive
|
||||
written reports saved as notes, use research_topic instead.",
|
||||
parameters={
|
||||
"query": {"type": "string", "description": "The topic or question to look up"},
|
||||
},
|
||||
required=["query"],
|
||||
)
|
||||
```
|
||||
|
||||
No `requires` field — always available.
|
||||
|
||||
**Logic:**
|
||||
1. Call `wiki_summary(query)`
|
||||
2. If Wikipedia returns a result: return `{"success": True, "type": "lookup", "source": "wikipedia", "data": {"title": ..., "extract": ..., "url": ...}}`
|
||||
3. If Wikipedia misses and `Config.searxng_enabled()`:
|
||||
- Call `_search_searxng(query)` to get search results
|
||||
- Fetch top 1-2 result URLs via `_fetch_full_article` (from `rss.py`, trafilatura-based)
|
||||
- Return `{"success": True, "type": "lookup", "source": "web", "data": {"query": ..., "results": [...], "content": ...}}`
|
||||
4. If Wikipedia misses and no SearXNG: return `{"success": True, "type": "lookup", "source": "none", "data": {"query": ..., "message": "No results found. You can answer from your own knowledge."}}`
|
||||
|
||||
### `src/fabledassistant/services/research.py` (modified)
|
||||
|
||||
**In Step 2 (parallel search):**
|
||||
- For each sub-query, run `wiki_search(query, limit=1)` concurrently with `_search_searxng(query)`
|
||||
- Merge Wikipedia results into the per-query result list
|
||||
|
||||
**In Step 3 (deduplication):**
|
||||
- When deduplicating URLs, Wikipedia URLs (`wikipedia.org`) are checked against SearXNG results
|
||||
- If a Wikipedia article URL already appears in SearXNG results, skip the duplicate
|
||||
|
||||
**Wikipedia article content for synthesis:**
|
||||
- The `extract` from `wiki_search` is used as the source content (no additional fetch needed, unlike SearXNG URLs which require `fetch_url_content`)
|
||||
- This means Wikipedia sources are available immediately without an HTTP fetch step
|
||||
|
||||
## Error Handling
|
||||
|
||||
- All Wikipedia API failures (network, timeout, malformed JSON) return `None`/`[]` silently
|
||||
- `lookup` never raises — always returns a response the model can work with
|
||||
- In the research pipeline, Wikipedia is purely additive; its failure never degrades existing SearXNG-based research
|
||||
- Disambiguation pages are detected via `"type": "disambiguation"` in the summary response and treated as a miss
|
||||
|
||||
## Testing
|
||||
|
||||
### `tests/test_wikipedia.py` (new)
|
||||
|
||||
- `test_wiki_summary_returns_extract` — mock successful summary response, verify return shape
|
||||
- `test_wiki_summary_returns_none_on_404` — mock 404, verify `None`
|
||||
- `test_wiki_summary_returns_none_on_disambiguation` — mock disambiguation response, verify `None`
|
||||
- `test_wiki_search_returns_results` — mock search API + summary fetches, verify list
|
||||
- `test_wiki_search_returns_empty_on_failure` — mock network error, verify `[]`
|
||||
|
||||
### `tests/test_lookup_tool.py` (new)
|
||||
|
||||
- `test_lookup_wikipedia_hit` — mock `wiki_summary` returning data, verify tool returns wikipedia source
|
||||
- `test_lookup_wikipedia_miss_searxng_fallback` — mock `wiki_summary` returning None, SearXNG returning results + article fetch, verify web source
|
||||
- `test_lookup_wikipedia_miss_no_searxng` — mock both missing, verify graceful "no results" response
|
||||
- `test_lookup_always_available` — verify the tool appears in `get_tools_for_user` regardless of SearXNG config
|
||||
|
||||
### `tests/test_research_pipeline.py` (add to existing)
|
||||
|
||||
- `test_research_includes_wikipedia_sources` — mock `wiki_search` alongside SearXNG, verify Wikipedia results appear in source pool
|
||||
|
||||
All tests mock HTTP calls — no live API hits.
|
||||
|
||||
## What Doesn't Change
|
||||
|
||||
- `read_article` tool — stays as-is (explicit URL fetch, different purpose)
|
||||
- `research_topic` tool definition — stays as-is (same name, description, parameters)
|
||||
- `generation_task.py` research interception — stays as-is
|
||||
- `search_images` tool — stays as-is
|
||||
- `_search_searxng` and `_search_searxng_images` — stay as-is
|
||||
- `_fetch_full_article` in `rss.py` — stays as-is, reused by `lookup` for SearXNG fallback
|
||||
@@ -5,6 +5,7 @@ import { useTheme } from "@/composables/useTheme";
|
||||
import { useShortcuts } from "@/composables/useShortcuts";
|
||||
import { useAuthStore } from "@/stores/auth";
|
||||
import { useChatStore } from "@/stores/chat";
|
||||
import { useSettingsStore } from "@/stores/settings";
|
||||
import AppLogo from "@/components/AppLogo.vue";
|
||||
import NotificationBell from "@/components/NotificationBell.vue";
|
||||
|
||||
@@ -12,6 +13,7 @@ const { theme, toggleTheme } = useTheme();
|
||||
const { toggleShortcuts } = useShortcuts();
|
||||
const authStore = useAuthStore();
|
||||
const chatStore = useChatStore();
|
||||
const settingsStore = useSettingsStore();
|
||||
const router = useRouter();
|
||||
const route = useRoute();
|
||||
|
||||
@@ -78,7 +80,7 @@ router.afterEach(() => {
|
||||
<router-link to="/chat" :class="['nav-link', { 'router-link-active': isChatActive }]">Chat</router-link>
|
||||
<router-link to="/briefing" class="nav-link">Briefing</router-link>
|
||||
<router-link to="/calendar" class="nav-link">Calendar</router-link>
|
||||
<router-link to="/news" class="nav-link">News</router-link>
|
||||
<router-link v-if="settingsStore.rssEnabled" to="/news" class="nav-link">News</router-link>
|
||||
<router-link to="/projects" class="nav-link">Projects</router-link>
|
||||
</div>
|
||||
</div>
|
||||
@@ -128,7 +130,7 @@ router.afterEach(() => {
|
||||
<router-link to="/briefing" class="nav-link">Briefing</router-link>
|
||||
<router-link to="/calendar" class="nav-link">Calendar</router-link>
|
||||
<router-link to="/projects" class="nav-link">Projects</router-link>
|
||||
<router-link to="/news" class="nav-link">News</router-link>
|
||||
<router-link v-if="settingsStore.rssEnabled" to="/news" class="nav-link">News</router-link>
|
||||
<router-link to="/shared" class="nav-link">Shared</router-link>
|
||||
<div class="mobile-divider"></div>
|
||||
<router-link to="/settings" class="nav-link">Settings</router-link>
|
||||
|
||||
@@ -9,6 +9,8 @@ interface ForecastDay {
|
||||
precip_probability: number | null
|
||||
precip_mm: number | null
|
||||
windspeed_max: number
|
||||
precip_summary?: string
|
||||
precip_peak_hour?: string
|
||||
}
|
||||
|
||||
interface WeatherData {
|
||||
@@ -21,6 +23,7 @@ interface WeatherData {
|
||||
yesterday_high: number | null
|
||||
yesterday_low: number | null
|
||||
wind_unit?: string
|
||||
precip_summary?: string | null
|
||||
forecast: ForecastDay[]
|
||||
}
|
||||
|
||||
@@ -68,6 +71,11 @@ const fetchedAtLabel = computed(() => {
|
||||
return ''
|
||||
}
|
||||
})
|
||||
|
||||
function hasPrecip(day: ForecastDay): boolean {
|
||||
return (day.precip_probability != null && day.precip_probability > 0) ||
|
||||
(day.precip_mm != null && day.precip_mm > 0)
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
@@ -85,6 +93,9 @@ const fetchedAtLabel = computed(() => {
|
||||
Today: {{ weather.today_high }}° / {{ weather.today_low }}°
|
||||
<span v-if="tempDelta" class="weather-delta"> · {{ tempDelta }}</span>
|
||||
</div>
|
||||
<div v-if="weather.precip_summary" class="weather-precip-summary">
|
||||
💧 {{ weather.precip_summary }}
|
||||
</div>
|
||||
<table class="weather-forecast" v-if="weather.forecast.length">
|
||||
<thead>
|
||||
<tr>
|
||||
@@ -100,8 +111,14 @@ const fetchedAtLabel = computed(() => {
|
||||
<td class="forecast-day-name">{{ day.day }}</td>
|
||||
<td class="forecast-icon">{{ weatherIcon(day.condition) }}</td>
|
||||
<td class="forecast-temps">{{ day.high }}° / {{ day.low }}°</td>
|
||||
<td class="forecast-precip" :class="{ 'forecast-precip--dry': !(day.precip_probability != null && day.precip_probability > 0) && !(day.precip_mm != null && day.precip_mm > 0) }">
|
||||
<template v-if="day.precip_probability != null && day.precip_probability > 0">{{ day.precip_probability }}%</template>
|
||||
<td class="forecast-precip" :class="{ 'forecast-precip--dry': !hasPrecip(day) }">
|
||||
<template v-if="day.precip_summary">
|
||||
<span class="precip-detail" :title="day.precip_summary">
|
||||
{{ day.precip_probability }}%
|
||||
<span v-if="day.precip_peak_hour" class="precip-peak">{{ day.precip_peak_hour }}</span>
|
||||
</span>
|
||||
</template>
|
||||
<template v-else-if="day.precip_probability != null && day.precip_probability > 0">{{ day.precip_probability }}%</template>
|
||||
<template v-else-if="day.precip_mm != null && day.precip_mm > 0">{{ day.precip_mm.toFixed(1) }}mm</template>
|
||||
<template v-else>—</template>
|
||||
</td>
|
||||
@@ -168,7 +185,7 @@ const fetchedAtLabel = computed(() => {
|
||||
|
||||
.weather-today {
|
||||
color: var(--color-text-secondary);
|
||||
margin-bottom: 0.75rem;
|
||||
margin-bottom: 0.25rem;
|
||||
font-size: 0.85rem;
|
||||
}
|
||||
|
||||
@@ -177,6 +194,16 @@ const fetchedAtLabel = computed(() => {
|
||||
font-size: 0.82rem;
|
||||
}
|
||||
|
||||
.weather-precip-summary {
|
||||
color: var(--color-text-secondary);
|
||||
font-size: 0.83rem;
|
||||
margin-bottom: 0.75rem;
|
||||
padding: 0.35rem 0.5rem;
|
||||
background: color-mix(in srgb, var(--color-primary) 8%, transparent);
|
||||
border-radius: var(--radius-sm, 6px);
|
||||
border-left: 2px solid color-mix(in srgb, var(--color-primary) 40%, transparent);
|
||||
}
|
||||
|
||||
.weather-forecast {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
@@ -227,6 +254,18 @@ const fetchedAtLabel = computed(() => {
|
||||
opacity: 0.35;
|
||||
}
|
||||
|
||||
.precip-detail {
|
||||
display: inline-flex;
|
||||
align-items: baseline;
|
||||
gap: 0.3em;
|
||||
}
|
||||
|
||||
.precip-peak {
|
||||
font-size: 0.7rem;
|
||||
color: var(--color-text-muted);
|
||||
opacity: 0.8;
|
||||
}
|
||||
|
||||
.forecast-wind {
|
||||
text-align: right;
|
||||
color: var(--color-text-muted);
|
||||
|
||||
@@ -16,6 +16,10 @@ export const useSettingsStore = defineStore("settings", () => {
|
||||
() => settings.value.default_model || ""
|
||||
);
|
||||
|
||||
const rssEnabled = computed(
|
||||
() => settings.value.rss_enabled === "true"
|
||||
);
|
||||
|
||||
// Voice status — checked once on login, refreshable from Settings
|
||||
const voiceEnabled = ref(false);
|
||||
const voiceSttReady = ref(false);
|
||||
@@ -62,6 +66,7 @@ export const useSettingsStore = defineStore("settings", () => {
|
||||
loading,
|
||||
assistantName,
|
||||
defaultModel,
|
||||
rssEnabled,
|
||||
voiceEnabled,
|
||||
voiceSttReady,
|
||||
voiceTtsReady,
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
import { ref, computed, onMounted, onUnmounted, watch, nextTick } from 'vue'
|
||||
import { useBackgroundRefresh } from '@/composables/useBackgroundRefresh'
|
||||
import { useChatStore } from '@/stores/chat'
|
||||
import { useSettingsStore } from '@/stores/settings'
|
||||
import ChatPanel from '@/components/ChatPanel.vue'
|
||||
import WeatherCard from '@/components/WeatherCard.vue'
|
||||
import BriefingSetupWizard from '@/components/BriefingSetupWizard.vue'
|
||||
@@ -15,7 +16,9 @@ import {
|
||||
postRssReaction,
|
||||
deleteRssReaction,
|
||||
getNewsItems,
|
||||
listEvents,
|
||||
type BriefingConversation,
|
||||
type EventEntry,
|
||||
} from '@/api/client'
|
||||
import type { NewsItem } from '@/types/news'
|
||||
|
||||
@@ -33,6 +36,7 @@ interface WeatherData {
|
||||
}
|
||||
|
||||
const chatStore = useChatStore()
|
||||
const settingsStore = useSettingsStore()
|
||||
|
||||
// Setup wizard
|
||||
const showWizard = ref(false)
|
||||
@@ -90,6 +94,66 @@ async function loadWeather() {
|
||||
} catch { /* silent */ }
|
||||
}
|
||||
|
||||
const refreshingWeather = ref(false)
|
||||
async function refreshWeather() {
|
||||
refreshingWeather.value = true
|
||||
try {
|
||||
const data = await apiPost<{ locations: WeatherData[]; temp_unit: string }>('/api/briefing/weather/refresh', {})
|
||||
weatherData.value = data.locations ?? []
|
||||
tempUnit.value = data.temp_unit ?? 'C'
|
||||
} catch { /* silent */ }
|
||||
finally { refreshingWeather.value = false }
|
||||
}
|
||||
|
||||
// Upcoming events (right column, below weather)
|
||||
const upcomingEvents = ref<EventEntry[]>([])
|
||||
|
||||
interface GroupedDay {
|
||||
label: string
|
||||
dateKey: string
|
||||
events: EventEntry[]
|
||||
}
|
||||
|
||||
const groupedEvents = computed<GroupedDay[]>(() => {
|
||||
const groups = new Map<string, EventEntry[]>()
|
||||
const today = new Date()
|
||||
today.setHours(0, 0, 0, 0)
|
||||
|
||||
for (const ev of upcomingEvents.value) {
|
||||
const d = new Date(ev.start_dt)
|
||||
const key = d.toISOString().slice(0, 10)
|
||||
if (!groups.has(key)) groups.set(key, [])
|
||||
groups.get(key)!.push(ev)
|
||||
}
|
||||
|
||||
const result: GroupedDay[] = []
|
||||
for (const [key, events] of groups) {
|
||||
const d = new Date(key + 'T00:00:00')
|
||||
const diff = Math.round((d.getTime() - today.getTime()) / 86_400_000)
|
||||
let label: string
|
||||
if (diff === 0) label = 'Today'
|
||||
else if (diff === 1) label = 'Tomorrow'
|
||||
else label = d.toLocaleDateString(undefined, { weekday: 'short', month: 'short', day: 'numeric' })
|
||||
result.push({ label, dateKey: key, events })
|
||||
}
|
||||
return result
|
||||
})
|
||||
|
||||
function formatEventTime(ev: EventEntry): string {
|
||||
if (ev.all_day) return 'All day'
|
||||
const d = new Date(ev.start_dt)
|
||||
return d.toLocaleTimeString([], { hour: 'numeric', minute: '2-digit' })
|
||||
}
|
||||
|
||||
async function loadEvents() {
|
||||
try {
|
||||
const now = new Date()
|
||||
const end = new Date(now)
|
||||
end.setDate(end.getDate() + 14)
|
||||
upcomingEvents.value = await listEvents(now.toISOString(), end.toISOString())
|
||||
} catch { /* silent */ }
|
||||
}
|
||||
|
||||
// News panel (right column)
|
||||
const newsItems = ref<NewsItem[]>([])
|
||||
|
||||
@@ -113,6 +177,7 @@ async function loadAll() {
|
||||
loadWeather(),
|
||||
loadNews(),
|
||||
loadCurrentConditions(),
|
||||
loadEvents(),
|
||||
])
|
||||
conversations.value = convList
|
||||
if (today) {
|
||||
@@ -277,14 +342,23 @@ onMounted(async () => {
|
||||
<div class="briefing-right">
|
||||
<!-- Weather section (sticky) -->
|
||||
<div class="weather-section" v-if="weatherData.length">
|
||||
<div class="weather-tabs" v-if="weatherData.length > 1">
|
||||
<div class="weather-section-header">
|
||||
<div class="weather-tabs" v-if="weatherData.length > 1">
|
||||
<button
|
||||
v-for="(loc, i) in weatherData"
|
||||
:key="(loc as WeatherData).location"
|
||||
class="weather-tab"
|
||||
:class="{ active: selectedWeatherIdx === i }"
|
||||
@click="selectedWeatherIdx = i"
|
||||
>{{ (loc as WeatherData).location }}</button>
|
||||
</div>
|
||||
<button
|
||||
v-for="(loc, i) in weatherData"
|
||||
:key="(loc as WeatherData).location"
|
||||
class="weather-tab"
|
||||
:class="{ active: selectedWeatherIdx === i }"
|
||||
@click="selectedWeatherIdx = i"
|
||||
>{{ (loc as WeatherData).location }}</button>
|
||||
class="weather-refresh-btn"
|
||||
:class="{ spinning: refreshingWeather }"
|
||||
:disabled="refreshingWeather"
|
||||
@click="refreshWeather"
|
||||
title="Refresh weather"
|
||||
>↻</button>
|
||||
</div>
|
||||
<WeatherCard
|
||||
:weather="weatherData[selectedWeatherIdx]"
|
||||
@@ -292,8 +366,29 @@ onMounted(async () => {
|
||||
/>
|
||||
</div>
|
||||
|
||||
<!-- Upcoming events -->
|
||||
<div class="events-section" v-if="groupedEvents.length">
|
||||
<div class="panel-label-row">
|
||||
<div class="panel-label">Upcoming</div>
|
||||
<router-link to="/calendar" class="events-cal-link">Calendar →</router-link>
|
||||
</div>
|
||||
<div class="events-list">
|
||||
<div v-for="group in groupedEvents" :key="group.dateKey" class="events-day-group">
|
||||
<div class="events-day-label">{{ group.label }}</div>
|
||||
<div v-for="ev in group.events" :key="ev.id" class="event-row">
|
||||
<span class="event-dot" :style="ev.color ? { background: ev.color } : {}"></span>
|
||||
<span class="event-body">
|
||||
<span class="event-title">{{ ev.title }}</span>
|
||||
<span class="event-time">{{ formatEventTime(ev) }}</span>
|
||||
<span v-if="ev.location" class="event-loc">{{ ev.location }}</span>
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- News section (scrollable) -->
|
||||
<div class="news-section">
|
||||
<div v-if="settingsStore.rssEnabled" class="news-section">
|
||||
<div class="panel-label-row">
|
||||
<div class="panel-label">Today's News</div>
|
||||
<span v-if="newsItems.length" class="news-count">{{ newsItems.length }} items</span>
|
||||
@@ -462,10 +557,37 @@ onMounted(async () => {
|
||||
margin-bottom: 0;
|
||||
}
|
||||
|
||||
.weather-section-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
|
||||
.weather-refresh-btn {
|
||||
background: none;
|
||||
border: 1px solid var(--color-border);
|
||||
border-radius: 6px;
|
||||
color: var(--color-text-muted);
|
||||
font-size: 1rem;
|
||||
cursor: pointer;
|
||||
padding: 0.2rem 0.45rem;
|
||||
line-height: 1;
|
||||
transition: all 0.15s;
|
||||
}
|
||||
.weather-refresh-btn:hover {
|
||||
border-color: var(--color-primary);
|
||||
color: var(--color-primary);
|
||||
}
|
||||
.weather-refresh-btn:disabled { opacity: 0.5; cursor: not-allowed; }
|
||||
.weather-refresh-btn.spinning {
|
||||
animation: spin 0.8s linear infinite;
|
||||
}
|
||||
@keyframes spin { to { transform: rotate(360deg); } }
|
||||
|
||||
.weather-tabs {
|
||||
display: flex;
|
||||
gap: 0.25rem;
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
|
||||
.weather-tab {
|
||||
@@ -492,6 +614,79 @@ onMounted(async () => {
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
/* ─── Upcoming events ─────────────────────────────────────── */
|
||||
.events-section {
|
||||
padding: 0.75rem 1rem;
|
||||
border-top: 1px solid var(--color-border);
|
||||
}
|
||||
.events-cal-link {
|
||||
font-size: 0.75rem;
|
||||
color: var(--color-text-muted);
|
||||
text-decoration: none;
|
||||
}
|
||||
.events-cal-link:hover { color: var(--color-primary); }
|
||||
.events-list {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.6rem;
|
||||
}
|
||||
.events-day-group {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.2rem;
|
||||
}
|
||||
.events-day-label {
|
||||
font-size: 0.72rem;
|
||||
font-weight: 700;
|
||||
color: var(--color-text-muted);
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.03em;
|
||||
padding-bottom: 0.15rem;
|
||||
}
|
||||
.event-row {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
gap: 0.4rem;
|
||||
padding: 0.25rem 0.4rem;
|
||||
border-radius: 6px;
|
||||
}
|
||||
.event-row:hover {
|
||||
background: color-mix(in srgb, var(--color-primary) 6%, transparent);
|
||||
}
|
||||
.event-dot {
|
||||
width: 7px;
|
||||
height: 7px;
|
||||
border-radius: 50%;
|
||||
background: var(--color-primary);
|
||||
flex-shrink: 0;
|
||||
margin-top: 5px;
|
||||
}
|
||||
.event-body {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.05rem;
|
||||
min-width: 0;
|
||||
}
|
||||
.event-title {
|
||||
font-size: 0.82rem;
|
||||
font-weight: 600;
|
||||
color: var(--color-text);
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
.event-time {
|
||||
font-size: 0.72rem;
|
||||
color: var(--color-text-muted);
|
||||
}
|
||||
.event-loc {
|
||||
font-size: 0.7rem;
|
||||
color: var(--color-text-muted);
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
|
||||
.news-section {
|
||||
flex: 1;
|
||||
overflow-y: auto;
|
||||
|
||||
@@ -387,6 +387,14 @@ async function saveBriefingSettings() {
|
||||
}
|
||||
}
|
||||
|
||||
async function toggleRss() {
|
||||
try {
|
||||
await store.updateSettings({ rss_enabled: store.rssEnabled ? "false" : "true" });
|
||||
} catch {
|
||||
toastStore.show("Failed to update RSS setting", "error");
|
||||
}
|
||||
}
|
||||
|
||||
async function addFeed() {
|
||||
if (!newFeedUrl.value.trim() || addingFeed.value) return;
|
||||
addingFeed.value = true;
|
||||
@@ -2209,8 +2217,20 @@ function formatUserDate(iso: string): string {
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<!-- RSS Feeds -->
|
||||
<!-- RSS toggle -->
|
||||
<section class="settings-section full-width">
|
||||
<h2>RSS / News</h2>
|
||||
<div class="checkbox-field">
|
||||
<label>
|
||||
<input type="checkbox" :checked="store.rssEnabled" @change="toggleRss" />
|
||||
Enable RSS feeds
|
||||
</label>
|
||||
<p class="field-hint">Subscribe to RSS/Atom feeds and include news in your briefings.</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- RSS Feeds -->
|
||||
<section v-if="store.rssEnabled" class="settings-section full-width">
|
||||
<div class="briefing-feeds-header">
|
||||
<div>
|
||||
<h2>RSS Feeds</h2>
|
||||
@@ -2252,7 +2272,7 @@ function formatUserDate(iso: string): string {
|
||||
</section>
|
||||
|
||||
<!-- News Preferences -->
|
||||
<section class="settings-section full-width">
|
||||
<section v-if="store.rssEnabled" class="settings-section full-width">
|
||||
<h2>News Preferences</h2>
|
||||
<p class="section-desc">
|
||||
Tell the briefing what topics you care about. Topics are matched against
|
||||
|
||||
@@ -145,8 +145,11 @@ def create_app() -> Quart:
|
||||
"Content-Security-Policy",
|
||||
"default-src 'self'; "
|
||||
"script-src 'self' 'unsafe-inline' 'wasm-unsafe-eval'; "
|
||||
"style-src 'self' 'unsafe-inline'; img-src 'self' data: blob:; "
|
||||
"connect-src 'self'; font-src 'self' data:; object-src 'none'; "
|
||||
"style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; "
|
||||
"img-src 'self' data: blob:; "
|
||||
"connect-src 'self'; "
|
||||
"font-src 'self' data: https://fonts.gstatic.com; "
|
||||
"object-src 'none'; "
|
||||
"base-uri 'self'; worker-src 'self' blob:;"
|
||||
)
|
||||
|
||||
|
||||
@@ -54,11 +54,17 @@ async def put_config():
|
||||
return jsonify({"ok": True})
|
||||
|
||||
|
||||
async def _rss_enabled() -> bool:
|
||||
return (await get_setting(g.user.id, "rss_enabled", "false")).lower() == "true"
|
||||
|
||||
|
||||
# ── RSS Feeds ─────────────────────────────────────────────────────────────────
|
||||
|
||||
@briefing_bp.route("/feeds", methods=["GET"])
|
||||
@_REQUIRE
|
||||
async def list_feeds():
|
||||
if not await _rss_enabled():
|
||||
return jsonify([])
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(RssFeed).where(RssFeed.user_id == g.user.id).order_by(RssFeed.id)
|
||||
@@ -70,6 +76,8 @@ async def list_feeds():
|
||||
@briefing_bp.route("/feeds", methods=["POST"])
|
||||
@_REQUIRE
|
||||
async def add_feed():
|
||||
if not await _rss_enabled():
|
||||
return jsonify({"error": "RSS is disabled"}), 403
|
||||
data = await request.get_json()
|
||||
url = (data.get("url") or "").strip()
|
||||
if not url:
|
||||
@@ -120,6 +128,8 @@ async def delete_feed(feed_id: int):
|
||||
@briefing_bp.route("/feeds/refresh", methods=["POST"])
|
||||
@_REQUIRE
|
||||
async def refresh_feeds():
|
||||
if not await _rss_enabled():
|
||||
return jsonify({"feeds_refreshed": 0, "new_items": 0})
|
||||
results = await rss_svc.refresh_all_feeds(g.user.id)
|
||||
total_new = sum(results.values())
|
||||
return jsonify({"feeds_refreshed": len(results), "new_items": total_new})
|
||||
@@ -128,6 +138,8 @@ async def refresh_feeds():
|
||||
@briefing_bp.route("/feeds/recent", methods=["GET"])
|
||||
@_REQUIRE
|
||||
async def recent_items():
|
||||
if not await _rss_enabled():
|
||||
return jsonify({"items": []})
|
||||
limit = min(int(request.args.get("limit", 20)), 100)
|
||||
items = await rss_svc.get_recent_items(g.user.id, limit=limit)
|
||||
return jsonify({"items": items})
|
||||
@@ -155,6 +167,7 @@ async def get_weather():
|
||||
return jsonify({"locations": cards, "temp_unit": temp_unit})
|
||||
|
||||
|
||||
|
||||
@briefing_bp.route("/weather/current", methods=["GET"])
|
||||
@_REQUIRE
|
||||
async def get_current_weather():
|
||||
@@ -213,11 +226,14 @@ async def refresh_weather():
|
||||
raw = await get_setting(g.user.id, "briefing_config", "{}")
|
||||
try:
|
||||
config = json.loads(raw) if isinstance(raw, str) else {}
|
||||
temp_unit = config.get("temp_unit", "C")
|
||||
if temp_unit not in ("C", "F"):
|
||||
temp_unit = "C"
|
||||
except Exception:
|
||||
config = {}
|
||||
temp_unit = "C"
|
||||
|
||||
locations = config.get("locations", {})
|
||||
refreshed = []
|
||||
for key, loc in locations.items():
|
||||
if not loc.get("lat") or not loc.get("lon"):
|
||||
continue
|
||||
@@ -229,11 +245,15 @@ async def refresh_weather():
|
||||
lat=loc["lat"],
|
||||
lon=loc["lon"],
|
||||
)
|
||||
refreshed.append(key)
|
||||
except Exception:
|
||||
logger.warning("Failed to refresh weather for %s", key, exc_info=True)
|
||||
|
||||
return jsonify({"refreshed": refreshed})
|
||||
rows = await weather_svc.get_cached_weather_rows(g.user.id)
|
||||
cards = [
|
||||
card for row in rows
|
||||
if (card := weather_svc.parse_weather_card_data(row, temp_unit)) is not None
|
||||
]
|
||||
return jsonify({"locations": cards, "temp_unit": temp_unit})
|
||||
|
||||
|
||||
# ── Briefing Conversations ─────────────────────────────────────────────────────
|
||||
@@ -454,6 +474,9 @@ async def list_news():
|
||||
offset — pagination offset (default 0)
|
||||
feed_id — optional integer filter by feed
|
||||
"""
|
||||
if not await _rss_enabled():
|
||||
return jsonify({"items": [], "total": 0})
|
||||
|
||||
from sqlalchemy import text as _text
|
||||
|
||||
days = min(int(request.args.get("days", 2)), 90)
|
||||
|
||||
@@ -20,18 +20,26 @@ SLOT_NAMES = ("compilation", "morning", "midday", "afternoon")
|
||||
# ── External data gather ──────────────────────────────────────────────────────
|
||||
|
||||
async def _gather_external(user_id: int) -> dict:
|
||||
"""Collect RSS items and weather."""
|
||||
from fabledassistant.services.rss import get_recent_items
|
||||
"""Collect RSS items (when enabled) and weather."""
|
||||
from fabledassistant.services.weather import get_cached_weather
|
||||
|
||||
rss_items, weather = await asyncio.gather(
|
||||
get_recent_items(user_id, limit=20),
|
||||
get_cached_weather(user_id),
|
||||
return_exceptions=True,
|
||||
)
|
||||
rss_on = (await get_setting(user_id, "rss_enabled", "false")).lower() == "true"
|
||||
rss_items: list = []
|
||||
if rss_on:
|
||||
from fabledassistant.services.rss import get_recent_items
|
||||
try:
|
||||
rss_items = await get_recent_items(user_id, limit=20)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
weather = await get_cached_weather(user_id)
|
||||
except Exception:
|
||||
weather = []
|
||||
|
||||
return {
|
||||
"rss_items": rss_items if not isinstance(rss_items, Exception) else [],
|
||||
"weather": weather if not isinstance(weather, Exception) else [],
|
||||
"rss_items": rss_items,
|
||||
"weather": weather,
|
||||
}
|
||||
|
||||
|
||||
@@ -371,7 +379,11 @@ async def run_compilation(
|
||||
if item.get("id")
|
||||
]
|
||||
|
||||
weather_card = parse_weather_card_data(weather_rows[0], temp_unit) if weather_rows else None
|
||||
weather_cards = [
|
||||
card for row in weather_rows
|
||||
if (card := parse_weather_card_data(row, temp_unit)) is not None
|
||||
]
|
||||
weather_card = weather_cards[0] if weather_cards else None
|
||||
|
||||
briefing_text, agentic_messages = await run_agentic_briefing(
|
||||
user_id, slot, model, conv_id=None, rss_override=filtered_rss,
|
||||
|
||||
@@ -362,10 +362,12 @@ async def _run_slot_for_user(user_id: int, slot: str) -> None:
|
||||
# Refresh external data first
|
||||
try:
|
||||
import json
|
||||
from fabledassistant.services.rss import refresh_all_feeds
|
||||
config_raw = await get_setting(user_id, "briefing_config", "{}")
|
||||
config = json.loads(config_raw) if isinstance(config_raw, str) else {}
|
||||
await refresh_all_feeds(user_id)
|
||||
rss_on = (await get_setting(user_id, "rss_enabled", "false")).lower() == "true"
|
||||
if rss_on:
|
||||
from fabledassistant.services.rss import refresh_all_feeds
|
||||
await refresh_all_feeds(user_id)
|
||||
from fabledassistant.services import weather as wx
|
||||
for key, loc in config.get("locations", {}).items():
|
||||
if loc.get("lat") and loc.get("lon"):
|
||||
|
||||
@@ -130,7 +130,7 @@ _TOOL_LABELS: dict[str, str] = {
|
||||
"update_event": "Updating calendar event",
|
||||
"delete_event": "Removing calendar event",
|
||||
"list_calendars": "Listing calendars",
|
||||
"search_web": "Searching the web",
|
||||
"lookup": "Looking up information",
|
||||
"research_topic": "Researching topic",
|
||||
}
|
||||
|
||||
|
||||
@@ -604,8 +604,9 @@ async def build_context(
|
||||
"Always include the UTC offset in datetime strings (e.g. 2026-09-30T14:00:00+01:00)."
|
||||
)
|
||||
tool_lines.append("When the user says 'remind me' with a time before an event, use the reminder_minutes parameter.")
|
||||
actions.append("lookup")
|
||||
if Config.searxng_enabled():
|
||||
actions.extend(["search_web", "research_topic", "search_images"])
|
||||
actions.extend(["research_topic", "search_images"])
|
||||
tool_lines.append(f"Available actions: {', '.join(actions)}.")
|
||||
tool_lines.append(
|
||||
"For relative dates like 'Friday' or 'next week', resolve them to YYYY-MM-DD format. "
|
||||
@@ -810,6 +811,8 @@ async def build_context(
|
||||
"score": round(score, 2),
|
||||
})
|
||||
user_context_parts.append(
|
||||
"[The following are reference excerpts from the user's personal notes, "
|
||||
"not part of their message. Use them only if relevant to answering.]\n"
|
||||
"--- Relevant Notes ---\n"
|
||||
+ "\n\n".join(snippets)
|
||||
+ "\n--- End Relevant Notes ---"
|
||||
@@ -850,33 +853,6 @@ async def build_context(
|
||||
+ "\n--- End Included Notes ---"
|
||||
)
|
||||
|
||||
# Semantically relevant RSS news items
|
||||
try:
|
||||
from fabledassistant.services.embeddings import get_embedding, semantic_search_rss_items
|
||||
news_query_vec = await get_embedding(user_message)
|
||||
news_hits = await semantic_search_rss_items(user_id, news_query_vec)
|
||||
if news_hits:
|
||||
news_snippets = []
|
||||
for score, rss_item in news_hits:
|
||||
feed_title = getattr(rss_item, "feed_title", "") or ""
|
||||
excerpt = (rss_item.content or "")[:500].strip()
|
||||
news_snippets.append(
|
||||
f"[{feed_title or 'News'}] {rss_item.title} (relevance: {round(score * 100)}%)\n"
|
||||
+ (f"{excerpt}\n" if excerpt else "")
|
||||
+ f"URL: {rss_item.url}"
|
||||
)
|
||||
user_context_parts.append(
|
||||
"--- Recent News You've Seen ---\n"
|
||||
+ "\n\n".join(news_snippets)
|
||||
+ "\n--- End Recent News ---"
|
||||
)
|
||||
context_meta["rss_news"] = [
|
||||
{"id": item.id, "title": item.title, "score": round(score, 2)}
|
||||
for score, item in news_hits
|
||||
]
|
||||
except Exception:
|
||||
logger.debug("RSS semantic search skipped", exc_info=True)
|
||||
|
||||
# URL content fetched from links in the user message
|
||||
urls = _find_urls(user_message)
|
||||
for url in urls[:2]:
|
||||
|
||||
@@ -10,6 +10,7 @@ import httpx
|
||||
from fabledassistant.config import Config
|
||||
from fabledassistant.services.llm import fetch_url_content, generate_completion, stream_chat
|
||||
from fabledassistant.services.notes import create_note, update_note
|
||||
from fabledassistant.services.wikipedia import wiki_search
|
||||
from fabledassistant.models.note import Note
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -205,7 +206,7 @@ async def run_research_pipeline(
|
||||
queries = await _generate_sub_queries(topic, model)
|
||||
logger.info("Research: generated %d sub-queries for topic '%s'", len(queries), topic)
|
||||
|
||||
# Step 2: Search all queries in parallel (200 ms stagger to avoid hammering SearXNG)
|
||||
# Step 2: Search all queries in parallel (SearXNG + Wikipedia)
|
||||
async def _search_with_stagger(i: int, query: str) -> tuple[str, list[dict]]:
|
||||
if i > 0:
|
||||
await asyncio.sleep(0.2 * i)
|
||||
@@ -214,13 +215,22 @@ async def run_research_pipeline(
|
||||
logger.info("Research: query '%s' → %d results", query, len(results))
|
||||
return query, results
|
||||
|
||||
search_results = await asyncio.gather(
|
||||
async def _wiki_for_query(query: str) -> list[dict]:
|
||||
return await wiki_search(query, limit=1)
|
||||
|
||||
searxng_task = asyncio.gather(
|
||||
*[_search_with_stagger(i, q) for i, q in enumerate(queries)]
|
||||
)
|
||||
wiki_task = asyncio.gather(
|
||||
*[_wiki_for_query(q) for q in queries]
|
||||
)
|
||||
search_results, wiki_results = await asyncio.gather(searxng_task, wiki_task)
|
||||
|
||||
# Deduplicate URLs across all queries
|
||||
seen_urls: set[str] = set()
|
||||
url_tasks: list[tuple[str, dict, str]] = [] # (url, result_dict, query)
|
||||
wiki_sources: list[dict] = [] # Wikipedia articles (already have content)
|
||||
|
||||
for query, results in search_results:
|
||||
for result in results[:PAGES_PER_QUERY]:
|
||||
url = result.get("url", "")
|
||||
@@ -228,7 +238,21 @@ async def run_research_pipeline(
|
||||
seen_urls.add(url)
|
||||
url_tasks.append((url, result, query))
|
||||
|
||||
# Fetch all unique URLs in parallel
|
||||
# Add Wikipedia results (they already have content via extract)
|
||||
for query, wiki_hits in zip(queries, wiki_results):
|
||||
for hit in wiki_hits:
|
||||
url = hit.get("url", "")
|
||||
if url and url not in seen_urls:
|
||||
seen_urls.add(url)
|
||||
wiki_sources.append({
|
||||
"url": url,
|
||||
"title": hit["title"],
|
||||
"query": query,
|
||||
"snippet": hit["extract"][:200],
|
||||
"content": hit["extract"],
|
||||
})
|
||||
|
||||
# Fetch all unique SearXNG URLs in parallel
|
||||
async def _fetch_source(url: str, result: dict, query: str) -> dict:
|
||||
title = result.get("title", url)
|
||||
_status(f"Reading: {title[:60]}...")
|
||||
@@ -241,10 +265,12 @@ async def run_research_pipeline(
|
||||
"content": content,
|
||||
}
|
||||
|
||||
all_sources: list[dict] = list(await asyncio.gather(
|
||||
fetched_sources: list[dict] = list(await asyncio.gather(
|
||||
*[_fetch_source(url, result, query) for url, result, query in url_tasks]
|
||||
))
|
||||
|
||||
all_sources = wiki_sources + fetched_sources
|
||||
|
||||
if not all_sources:
|
||||
raise ValueError(f"No results found for '{topic}'")
|
||||
|
||||
|
||||
@@ -84,6 +84,9 @@ async def _check_requires(user_id: int, requires: str) -> bool:
|
||||
return await is_caldav_configured(user_id)
|
||||
if requires == "searxng":
|
||||
return Config.searxng_enabled()
|
||||
if requires == "rss":
|
||||
from fabledassistant.services.settings import get_setting
|
||||
return (await get_setting(user_id, "rss_enabled", "false")).lower() == "true"
|
||||
return True
|
||||
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ logger = logging.getLogger(__name__)
|
||||
},
|
||||
read_only=True,
|
||||
briefing=True,
|
||||
requires="rss",
|
||||
)
|
||||
async def get_rss_items_tool(*, user_id, arguments, **_ctx):
|
||||
from fabledassistant.services.rss import get_recent_items
|
||||
@@ -35,6 +36,7 @@ async def get_rss_items_tool(*, user_id, arguments, **_ctx):
|
||||
"category": {"type": "string", "description": "Optional category label (e.g. 'news', 'tech', 'reddit'). Omit if unsure."},
|
||||
},
|
||||
required=["url"],
|
||||
requires="rss",
|
||||
)
|
||||
async def add_rss_feed_tool(*, user_id, arguments, **_ctx):
|
||||
import asyncio as _asyncio
|
||||
@@ -70,7 +72,7 @@ async def add_rss_feed_tool(*, user_id, arguments, **_ctx):
|
||||
"Fetch and read the full text of a web page or article from a URL. "
|
||||
"Use when the user shares a URL and wants you to read it, "
|
||||
"or to get the full content of a linked page. "
|
||||
"Do NOT use search_web for URLs — use this tool instead."
|
||||
"Do NOT use lookup for URLs — use this tool instead."
|
||||
),
|
||||
parameters={
|
||||
"url": {"type": "string", "description": "The URL to fetch and read"},
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Web search, research, and image tools (require SearXNG)."""
|
||||
"""Web search, research, and image tools."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -10,29 +10,107 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@tool(
|
||||
name="search_web",
|
||||
name="lookup",
|
||||
description=(
|
||||
"Quick web lookup — returns search result snippets for factual questions, current events, "
|
||||
"definitions, or version numbers. No note is saved. Use research_topic when the user wants "
|
||||
"a comprehensive written report saved as a note."
|
||||
"Look up a topic, concept, or factual question. Returns a concise answer from "
|
||||
"Wikipedia or web sources. Use for definitions, explanations, 'what is X', "
|
||||
"'how does Y work', current events, or version numbers. No note is saved. "
|
||||
"For comprehensive written reports saved as notes, use research_topic instead."
|
||||
),
|
||||
parameters={
|
||||
"query": {"type": "string", "description": "The search query"},
|
||||
"query": {"type": "string", "description": "The topic or question to look up"},
|
||||
},
|
||||
required=["query"],
|
||||
requires="searxng",
|
||||
)
|
||||
async def search_web_tool(*, user_id, arguments, **_ctx):
|
||||
from fabledassistant.services.research import _search_searxng
|
||||
async def lookup_tool(*, user_id, arguments, **_ctx):
|
||||
import asyncio
|
||||
|
||||
query = arguments.get("query", "")
|
||||
results = await _search_searxng(query)
|
||||
if not results:
|
||||
return {"success": False, "error": f"No results found for '{query}'"}
|
||||
from fabledassistant.config import Config
|
||||
from fabledassistant.services.wikipedia import wiki_summary
|
||||
|
||||
query = arguments.get("query", "").strip()
|
||||
if not query:
|
||||
return {"success": False, "error": "query is required"}
|
||||
|
||||
searxng_enabled = Config.searxng_enabled()
|
||||
|
||||
async def _searxng_results() -> list[dict]:
|
||||
if not searxng_enabled:
|
||||
return []
|
||||
from fabledassistant.services.research import _search_searxng
|
||||
return await _search_searxng(query) or []
|
||||
|
||||
wiki, search_results = await asyncio.gather(
|
||||
wiki_summary(query),
|
||||
_searxng_results(),
|
||||
)
|
||||
|
||||
wiki_payload: dict | None = None
|
||||
if wiki:
|
||||
wiki_payload = {
|
||||
"title": wiki["title"],
|
||||
"extract": wiki["extract"],
|
||||
"url": wiki["url"],
|
||||
}
|
||||
thumb_url = wiki.get("thumbnail_url") or ""
|
||||
if thumb_url:
|
||||
from fabledassistant.services.images import fetch_and_store_image
|
||||
record = await fetch_and_store_image(
|
||||
url=thumb_url,
|
||||
title=wiki["title"],
|
||||
source_domain="en.wikipedia.org",
|
||||
)
|
||||
if record:
|
||||
wiki_payload["image"] = {
|
||||
"embed": f"![{wiki['title']}](/api/images/{record.id})",
|
||||
"citation": f"*Source: [Wikipedia]({wiki['url']})*",
|
||||
}
|
||||
|
||||
web_payload: list[dict] = []
|
||||
if search_results:
|
||||
# Sequential fetches: trafilatura/lxml is not safe to run concurrently
|
||||
# via run_in_executor — parallel calls can trip a libxml2 double-free.
|
||||
from fabledassistant.services.rss import _fetch_full_article
|
||||
|
||||
for r in search_results[:2]:
|
||||
url = r.get("url", "")
|
||||
if not url:
|
||||
continue
|
||||
try:
|
||||
content = await _fetch_full_article(url)
|
||||
except Exception:
|
||||
content = None
|
||||
web_payload.append({
|
||||
"url": url,
|
||||
"title": r.get("title", url),
|
||||
"snippet": r.get("snippet", ""),
|
||||
"content": (content or "")[:4000],
|
||||
})
|
||||
|
||||
if not wiki_payload and not web_payload:
|
||||
return {
|
||||
"success": True,
|
||||
"type": "lookup",
|
||||
"data": {
|
||||
"query": query,
|
||||
"message": "No results found. You can answer from your own knowledge.",
|
||||
},
|
||||
}
|
||||
|
||||
data: dict = {
|
||||
"query": query,
|
||||
"wikipedia": wiki_payload,
|
||||
"web": web_payload,
|
||||
}
|
||||
if wiki_payload and wiki_payload.get("image"):
|
||||
data["image_instructions"] = (
|
||||
"If an image is relevant to your reply, embed it by writing the wikipedia.image.embed "
|
||||
"field verbatim, then the citation field on the next line. Otherwise omit it."
|
||||
)
|
||||
return {
|
||||
"success": True,
|
||||
"type": "web_search",
|
||||
"data": {"query": query, "results": results, "count": len(results)},
|
||||
"type": "lookup",
|
||||
"data": data,
|
||||
}
|
||||
|
||||
|
||||
@@ -42,7 +120,7 @@ async def search_web_tool(*, user_id, arguments, **_ctx):
|
||||
"Deep web research — searches multiple sources, synthesizes findings, and saves the result "
|
||||
"as a structured note with sections and citations. Use when the user says 'research', "
|
||||
"'look into', or wants a comprehensive write-up. Takes 30–120 seconds. "
|
||||
"For a quick factual answer without saving a note, use search_web."
|
||||
"For a quick factual answer without saving a note, use lookup."
|
||||
),
|
||||
parameters={
|
||||
"topic": {"type": "string", "description": "The topic or question to research"},
|
||||
@@ -64,7 +142,7 @@ async def research_topic_tool(*, user_id, arguments, **_ctx):
|
||||
|
||||
@tool(
|
||||
name="search_images",
|
||||
description="Search and display images inline. Use ONLY when the user explicitly asks to see, show, or find an image or photo. Not for factual questions — use search_web for those.",
|
||||
description="Search and display images inline. Use ONLY when the user explicitly asks to see, show, or find an image or photo. Not for factual questions — use lookup for those.",
|
||||
parameters={
|
||||
"query": {"type": "string", "description": "The image search query"},
|
||||
},
|
||||
|
||||
@@ -17,6 +17,7 @@ OPEN_METEO_DAILY = (
|
||||
"temperature_2m_max,temperature_2m_min,precipitation_sum,"
|
||||
"precipitation_probability_max,weathercode,windspeed_10m_max"
|
||||
)
|
||||
OPEN_METEO_HOURLY = "precipitation_probability"
|
||||
|
||||
# WMO weather code → description (subset; covers the most common codes)
|
||||
_WMO_CODES: dict[int, str] = {
|
||||
@@ -93,6 +94,55 @@ def detect_changes(old_days: list[dict], new_days: list[dict]) -> list[str]:
|
||||
return changes
|
||||
|
||||
|
||||
def _summarize_precip(hourly_probs: list[tuple[int, int]], threshold: int = 30) -> str | None:
|
||||
"""Build a human-readable precipitation summary from (hour, probability) pairs.
|
||||
|
||||
Returns None when no significant precipitation is expected.
|
||||
"""
|
||||
wet_hours = [(h, p) for h, p in hourly_probs if p >= threshold]
|
||||
if not wet_hours:
|
||||
return None
|
||||
|
||||
peak_hour, peak_prob = max(wet_hours, key=lambda x: x[1])
|
||||
daytime_hours = [h for h, _ in hourly_probs if 6 <= h <= 22]
|
||||
if not daytime_hours:
|
||||
return None
|
||||
|
||||
wet_daytime = [h for h, p in hourly_probs if 6 <= h <= 22 and p >= threshold]
|
||||
if len(wet_daytime) >= 10:
|
||||
return f"Rain likely all day (up to {peak_prob}%)"
|
||||
|
||||
if not wet_daytime:
|
||||
return None
|
||||
|
||||
def _fmt_hour(h: int) -> str:
|
||||
if h == 0 or h == 24:
|
||||
return "12 AM"
|
||||
if h == 12:
|
||||
return "12 PM"
|
||||
return f"{h} AM" if h < 12 else f"{h - 12} PM"
|
||||
|
||||
start = wet_daytime[0]
|
||||
end = wet_daytime[-1]
|
||||
if start == end:
|
||||
return f"{peak_prob}% chance around {_fmt_hour(start)}"
|
||||
return f"Rain likely {_fmt_hour(start)}–{_fmt_hour(end + 1)} (up to {peak_prob}%)"
|
||||
|
||||
|
||||
def _extract_hourly_precip_for_date(raw: dict, date_str: str) -> list[tuple[int, int]]:
|
||||
"""Extract (hour, probability) pairs for a specific date from cached forecast JSON."""
|
||||
hourly = raw.get("hourly", {})
|
||||
times = hourly.get("precipitation_probability", [])
|
||||
time_labels = hourly.get("time", [])
|
||||
pairs: list[tuple[int, int]] = []
|
||||
prefix = date_str + "T"
|
||||
for i, t in enumerate(time_labels):
|
||||
if t.startswith(prefix) and i < len(times) and times[i] is not None:
|
||||
hour = int(t[11:13])
|
||||
pairs.append((hour, times[i]))
|
||||
return pairs
|
||||
|
||||
|
||||
def parse_weather_card_data(
|
||||
cache_row,
|
||||
temp_unit: str = "C",
|
||||
@@ -138,6 +188,30 @@ def parse_weather_card_data(
|
||||
|
||||
wind_unit = "mph" if imperial else "km/h"
|
||||
|
||||
today_hourly = _extract_hourly_precip_for_date(raw, today_str)
|
||||
today_precip_summary = _summarize_precip(today_hourly)
|
||||
|
||||
def _forecast_day(d: dict) -> dict:
|
||||
entry: dict = {
|
||||
"day": day_label(d["date"]),
|
||||
"condition": d["description"],
|
||||
"high": to_temp(d["temp_max"]),
|
||||
"low": to_temp(d["temp_min"]),
|
||||
"precip_probability": d["precip_probability"],
|
||||
"precip_mm": d["precip_mm"],
|
||||
"windspeed_max": to_wind(d["windspeed_max"]),
|
||||
}
|
||||
hourly = _extract_hourly_precip_for_date(raw, d["date"])
|
||||
summary = _summarize_precip(hourly)
|
||||
if summary:
|
||||
entry["precip_summary"] = summary
|
||||
if hourly:
|
||||
peak = max(hourly, key=lambda x: x[1])
|
||||
if peak[1] >= 30:
|
||||
h = peak[0]
|
||||
entry["precip_peak_hour"] = f"{h} AM" if h < 12 else ("12 PM" if h == 12 else f"{h - 12} PM")
|
||||
return entry
|
||||
|
||||
return {
|
||||
"location": getattr(cache_row, "location_label", ""),
|
||||
"fetched_at": cache_row.fetched_at.isoformat(),
|
||||
@@ -148,18 +222,8 @@ def parse_weather_card_data(
|
||||
"yesterday_high": to_temp(yesterday_day["temp_max"]) if yesterday_day else None,
|
||||
"yesterday_low": to_temp(yesterday_day["temp_min"]) if yesterday_day else None,
|
||||
"wind_unit": wind_unit,
|
||||
"forecast": [
|
||||
{
|
||||
"day": day_label(d["date"]),
|
||||
"condition": d["description"],
|
||||
"high": to_temp(d["temp_max"]),
|
||||
"low": to_temp(d["temp_min"]),
|
||||
"precip_probability": d["precip_probability"],
|
||||
"precip_mm": d["precip_mm"],
|
||||
"windspeed_max": to_wind(d["windspeed_max"]),
|
||||
}
|
||||
for d in future_days
|
||||
],
|
||||
"precip_summary": today_precip_summary,
|
||||
"forecast": [_forecast_day(d) for d in future_days],
|
||||
}
|
||||
|
||||
|
||||
@@ -259,12 +323,13 @@ async def fetch_hourly_precip(lat: float, lon: float) -> dict[str, int]:
|
||||
|
||||
|
||||
async def _fetch_open_meteo(lat: float, lon: float) -> dict:
|
||||
"""Fetch 7-day forecast from Open-Meteo with current conditions and yesterday's data."""
|
||||
"""Fetch 7-day forecast from Open-Meteo with current conditions, hourly precip, and yesterday's data."""
|
||||
async with httpx.AsyncClient(timeout=15.0) as client:
|
||||
resp = await client.get(OPEN_METEO_URL, params={
|
||||
"latitude": lat,
|
||||
"longitude": lon,
|
||||
"daily": OPEN_METEO_DAILY,
|
||||
"hourly": OPEN_METEO_HOURLY,
|
||||
"current_weather": "true",
|
||||
"past_days": 1,
|
||||
"timezone": "auto",
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Wikipedia API: lightweight topic lookups and article search."""
|
||||
import logging
|
||||
from urllib.parse import quote as url_quote
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SUMMARY_URL = "https://en.wikipedia.org/api/rest_v1/page/summary"
|
||||
_SEARCH_URL = "https://en.wikipedia.org/w/api.php"
|
||||
_TIMEOUT = 5.0
|
||||
_USER_AGENT = "FabledAssistant/1.0 (https://fabledsword.com)"
|
||||
|
||||
|
||||
def _extract_thumbnail(data: dict) -> str:
|
||||
"""Prefer the 320px thumbnail (kinder to cache/bandwidth) over originalimage."""
|
||||
thumb = data.get("thumbnail", {}).get("source", "")
|
||||
if thumb:
|
||||
return thumb
|
||||
return data.get("originalimage", {}).get("source", "")
|
||||
|
||||
|
||||
async def wiki_summary(query: str) -> dict | None:
|
||||
"""Fetch a Wikipedia summary for a given title/query.
|
||||
|
||||
Does a direct title lookup via the REST v1 summary endpoint.
|
||||
Returns a dict with title, extract, url, and thumbnail_url on success;
|
||||
thumbnail_url is an empty string when the article has no image.
|
||||
Returns None on any failure.
|
||||
"""
|
||||
title = url_quote(query.replace(" ", "_"))
|
||||
url = f"{_SUMMARY_URL}/{title}"
|
||||
headers = {"User-Agent": _USER_AGENT}
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.get(url, headers=headers, timeout=_TIMEOUT, follow_redirects=True)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if data.get("type") == "disambiguation":
|
||||
logger.debug("wiki_summary: disambiguation page for %r", query)
|
||||
return None
|
||||
extract = data.get("extract", "").strip()
|
||||
if not extract:
|
||||
logger.debug("wiki_summary: empty extract for %r", query)
|
||||
return None
|
||||
return {
|
||||
"title": data.get("title", query),
|
||||
"extract": extract,
|
||||
"url": data.get("content_urls", {}).get("desktop", {}).get("page", ""),
|
||||
"thumbnail_url": _extract_thumbnail(data),
|
||||
}
|
||||
except Exception as exc:
|
||||
logger.debug("wiki_summary failed for %r: %s", query, exc)
|
||||
return None
|
||||
|
||||
|
||||
async def wiki_search(query: str, limit: int = 3) -> list[dict]:
|
||||
"""Search Wikipedia and return summaries for the top results.
|
||||
|
||||
Uses the MediaWiki search API then fetches summaries for each hit.
|
||||
Skips disambiguation pages and empty extracts.
|
||||
Returns a list of dicts with title, extract, and url.
|
||||
"""
|
||||
headers = {"User-Agent": _USER_AGENT}
|
||||
params = {
|
||||
"action": "query",
|
||||
"list": "search",
|
||||
"srsearch": query,
|
||||
"srlimit": str(limit),
|
||||
"format": "json",
|
||||
}
|
||||
results: list[dict] = []
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.get(
|
||||
_SEARCH_URL, params=params, headers=headers, timeout=_TIMEOUT
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
hits = data.get("query", {}).get("search", [])
|
||||
for hit in hits:
|
||||
title = hit.get("title", "")
|
||||
encoded = url_quote(title.replace(" ", "_"))
|
||||
summary_url = f"{_SUMMARY_URL}/{encoded}"
|
||||
try:
|
||||
sr = await client.get(
|
||||
summary_url, headers=headers, timeout=_TIMEOUT, follow_redirects=True
|
||||
)
|
||||
sr.raise_for_status()
|
||||
sdata = sr.json()
|
||||
if sdata.get("type") == "disambiguation":
|
||||
logger.debug("wiki_search: skipping disambiguation %r", title)
|
||||
continue
|
||||
extract = sdata.get("extract", "").strip()
|
||||
if not extract:
|
||||
logger.debug("wiki_search: empty extract for %r", title)
|
||||
continue
|
||||
results.append(
|
||||
{
|
||||
"title": sdata.get("title", title),
|
||||
"extract": extract,
|
||||
"url": sdata.get("content_urls", {})
|
||||
.get("desktop", {})
|
||||
.get("page", ""),
|
||||
"thumbnail_url": _extract_thumbnail(sdata),
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("wiki_search: summary fetch failed for %r: %s", title, exc)
|
||||
except Exception as exc:
|
||||
logger.debug("wiki_search failed for %r: %s", query, exc)
|
||||
return []
|
||||
return results
|
||||
@@ -127,8 +127,10 @@ async def test_update_event_fires_caldav_push():
|
||||
@pytest.mark.asyncio
|
||||
async def test_tools_calendar_always_available():
|
||||
"""Calendar tools must appear in get_tools_for_user even without CalDAV."""
|
||||
with patch("fabledassistant.services.tools._registry.is_caldav_configured", new_callable=AsyncMock) as mock_configured:
|
||||
with patch("fabledassistant.services.tools._registry.is_caldav_configured", new_callable=AsyncMock) as mock_configured, \
|
||||
patch("fabledassistant.services.settings.get_setting", new_callable=AsyncMock) as mock_setting:
|
||||
mock_configured.return_value = False
|
||||
mock_setting.return_value = "false"
|
||||
from fabledassistant.services.tools import get_tools_for_user
|
||||
tools = await get_tools_for_user(user_id=1)
|
||||
tool_names = {t["function"]["name"] for t in tools}
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from fabledassistant.services.tools.web import lookup_tool
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_lookup_wikipedia_hit():
|
||||
wiki_data = {
|
||||
"title": "QUIC",
|
||||
"extract": "QUIC is a transport layer protocol.",
|
||||
"url": "https://en.wikipedia.org/wiki/QUIC",
|
||||
"thumbnail_url": "",
|
||||
}
|
||||
|
||||
with patch("fabledassistant.services.wikipedia.wiki_summary", new_callable=AsyncMock, return_value=wiki_data), \
|
||||
patch("fabledassistant.config.Config") as mock_config:
|
||||
mock_config.searxng_enabled.return_value = False
|
||||
result = await lookup_tool(user_id=1, arguments={"query": "QUIC"})
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["type"] == "lookup"
|
||||
assert result["data"]["wikipedia"]["title"] == "QUIC"
|
||||
assert result["data"]["web"] == []
|
||||
assert "image" not in result["data"]["wikipedia"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_lookup_wikipedia_miss_searxng_fallback():
|
||||
searxng_results = [
|
||||
{"url": "https://example.com/quic", "title": "QUIC Explained", "snippet": "An overview..."},
|
||||
]
|
||||
|
||||
with patch("fabledassistant.services.wikipedia.wiki_summary", new_callable=AsyncMock, return_value=None), \
|
||||
patch("fabledassistant.config.Config") as mock_config, \
|
||||
patch("fabledassistant.services.research._search_searxng", new_callable=AsyncMock, return_value=searxng_results), \
|
||||
patch("fabledassistant.services.rss._fetch_full_article", new_callable=AsyncMock, return_value="Full article about QUIC..."):
|
||||
mock_config.searxng_enabled.return_value = True
|
||||
result = await lookup_tool(user_id=1, arguments={"query": "QUIC"})
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["data"]["wikipedia"] is None
|
||||
assert len(result["data"]["web"]) == 1
|
||||
assert result["data"]["web"][0]["url"] == "https://example.com/quic"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_lookup_parallel_both_sources():
|
||||
wiki_data = {
|
||||
"title": "President of the United States",
|
||||
"extract": "The president is the head of state.",
|
||||
"url": "https://en.wikipedia.org/wiki/President_of_the_United_States",
|
||||
"thumbnail_url": "",
|
||||
}
|
||||
searxng_results = [
|
||||
{"url": "https://whitehouse.gov", "title": "The White House", "snippet": "Current admin..."},
|
||||
]
|
||||
|
||||
with patch("fabledassistant.services.wikipedia.wiki_summary", new_callable=AsyncMock, return_value=wiki_data), \
|
||||
patch("fabledassistant.config.Config") as mock_config, \
|
||||
patch("fabledassistant.services.research._search_searxng", new_callable=AsyncMock, return_value=searxng_results), \
|
||||
patch("fabledassistant.services.rss._fetch_full_article", new_callable=AsyncMock, return_value="Current president is..."):
|
||||
mock_config.searxng_enabled.return_value = True
|
||||
result = await lookup_tool(user_id=1, arguments={"query": "president of the united states"})
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["data"]["wikipedia"]["title"] == "President of the United States"
|
||||
assert len(result["data"]["web"]) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_lookup_wikipedia_thumbnail_cached():
|
||||
wiki_data = {
|
||||
"title": "Hedgehog",
|
||||
"extract": "Hedgehogs are spiny mammals.",
|
||||
"url": "https://en.wikipedia.org/wiki/Hedgehog",
|
||||
"thumbnail_url": "https://upload.wikimedia.org/.../Hedgehog.jpg",
|
||||
}
|
||||
mock_record = MagicMock()
|
||||
mock_record.id = 42
|
||||
|
||||
with patch("fabledassistant.services.wikipedia.wiki_summary", new_callable=AsyncMock, return_value=wiki_data), \
|
||||
patch("fabledassistant.config.Config") as mock_config, \
|
||||
patch("fabledassistant.services.images.fetch_and_store_image", new_callable=AsyncMock, return_value=mock_record):
|
||||
mock_config.searxng_enabled.return_value = False
|
||||
result = await lookup_tool(user_id=1, arguments={"query": "hedgehog"})
|
||||
|
||||
assert result["data"]["wikipedia"]["image"]["embed"] == ""
|
||||
assert "Wikipedia" in result["data"]["wikipedia"]["image"]["citation"]
|
||||
assert "image_instructions" in result["data"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_lookup_wikipedia_miss_no_searxng():
|
||||
with patch("fabledassistant.services.wikipedia.wiki_summary", new_callable=AsyncMock, return_value=None), \
|
||||
patch("fabledassistant.config.Config") as mock_config:
|
||||
mock_config.searxng_enabled.return_value = False
|
||||
result = await lookup_tool(user_id=1, arguments={"query": "xyznonexistent"})
|
||||
|
||||
assert result["success"] is True
|
||||
assert "message" in result["data"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_lookup_always_available():
|
||||
with patch("fabledassistant.services.tools._registry.is_caldav_configured", new_callable=AsyncMock, return_value=False), \
|
||||
patch("fabledassistant.services.settings.get_setting", new_callable=AsyncMock, return_value="false"):
|
||||
from fabledassistant.services.tools import get_tools_for_user
|
||||
tools = await get_tools_for_user(user_id=1)
|
||||
tool_names = {t["function"]["name"] for t in tools}
|
||||
assert "lookup" in tool_names
|
||||
@@ -123,8 +123,6 @@ async def test_pipeline_creates_section_notes_and_index():
|
||||
n.title = title
|
||||
return n
|
||||
|
||||
mock_update = AsyncMock()
|
||||
|
||||
with patch("fabledassistant.services.research._generate_sub_queries", new_callable=AsyncMock, return_value=["q1"]), \
|
||||
patch("fabledassistant.services.research._search_searxng", new_callable=AsyncMock, return_value=[{"url": "http://x.com", "title": "X", "snippet": "s"}]), \
|
||||
patch("fabledassistant.services.research.fetch_url_content", new_callable=AsyncMock, return_value="content"), \
|
||||
@@ -192,3 +190,42 @@ async def test_pipeline_falls_back_when_all_sections_fail():
|
||||
result = await run_research_pipeline("test topic", user_id=1, model="test-model")
|
||||
|
||||
assert result.id == 77
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pipeline_includes_wikipedia_sources():
|
||||
"""run_research_pipeline should merge Wikipedia results into the source pool."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
wiki_results = [{"title": "Wiki Article", "extract": "Wikipedia content about the topic.", "url": "https://en.wikipedia.org/wiki/Topic"}]
|
||||
|
||||
outline = [
|
||||
{"title": "Section A", "focus": "Focus A"},
|
||||
{"title": "Section B", "focus": "Focus B"},
|
||||
]
|
||||
|
||||
note_id_counter = iter(range(30, 40))
|
||||
|
||||
def _make_note(user_id, title, body, tags, project_id=None, parent_id=None):
|
||||
n = MagicMock()
|
||||
n.id = next(note_id_counter)
|
||||
n.title = title
|
||||
return n
|
||||
|
||||
with patch("fabledassistant.services.research._generate_sub_queries", new_callable=AsyncMock, return_value=["q1"]), \
|
||||
patch("fabledassistant.services.research._search_searxng", new_callable=AsyncMock, return_value=[{"url": "http://x.com", "title": "X", "snippet": "s"}]), \
|
||||
patch("fabledassistant.services.research.wiki_search", new_callable=AsyncMock, return_value=wiki_results), \
|
||||
patch("fabledassistant.services.research.fetch_url_content", new_callable=AsyncMock, return_value="content"), \
|
||||
patch("fabledassistant.services.research._generate_outline", new_callable=AsyncMock, return_value=outline) as mock_outline, \
|
||||
patch("fabledassistant.services.research._synthesize_section", new_callable=AsyncMock, side_effect=lambda t, f, s, m: (t, f"Body for {t}")), \
|
||||
patch("fabledassistant.services.research._generate_executive_summary", new_callable=AsyncMock, return_value="Summary."), \
|
||||
patch("fabledassistant.services.research.create_note", new_callable=AsyncMock, side_effect=_make_note), \
|
||||
patch("fabledassistant.services.research.update_note", new_callable=AsyncMock):
|
||||
|
||||
from fabledassistant.services.research import run_research_pipeline
|
||||
await run_research_pipeline("test topic", user_id=1, model="test-model")
|
||||
|
||||
# The sources passed to _generate_outline should include the Wikipedia article
|
||||
sources_arg = mock_outline.call_args[0][1] # second positional arg
|
||||
source_urls = [s["url"] for s in sources_arg]
|
||||
assert "https://en.wikipedia.org/wiki/Topic" in source_urls
|
||||
|
||||
@@ -59,7 +59,6 @@ def test_detect_forecast_changes_rain_added():
|
||||
def test_parse_weather_card_data_returns_none_when_stale():
|
||||
"""Should return None when cache is older than 24 hours."""
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from unittest.mock import MagicMock
|
||||
from fabledassistant.services.weather import parse_weather_card_data
|
||||
|
||||
cache = MagicMock()
|
||||
@@ -71,7 +70,6 @@ def test_parse_weather_card_data_returns_none_when_stale():
|
||||
def test_parse_weather_card_data_returns_card_schema():
|
||||
"""Should return the correct metadata.weather schema for fresh cache."""
|
||||
from datetime import datetime, timezone, timedelta, date
|
||||
from unittest.mock import MagicMock
|
||||
from fabledassistant.services.weather import parse_weather_card_data
|
||||
|
||||
today = date.today().isoformat()
|
||||
@@ -103,3 +101,66 @@ def test_parse_weather_card_data_returns_card_schema():
|
||||
assert card["yesterday_low"] == 9
|
||||
assert len(card["forecast"]) >= 1
|
||||
assert card["location"] == "Berlin, DE"
|
||||
assert card["precip_summary"] is None
|
||||
|
||||
|
||||
def test_summarize_precip_dry():
|
||||
from fabledassistant.services.weather import _summarize_precip
|
||||
assert _summarize_precip([(h, 10) for h in range(24)]) is None
|
||||
|
||||
|
||||
def test_summarize_precip_all_day():
|
||||
from fabledassistant.services.weather import _summarize_precip
|
||||
result = _summarize_precip([(h, 60) for h in range(24)])
|
||||
assert result is not None
|
||||
assert "all day" in result.lower()
|
||||
|
||||
|
||||
def test_summarize_precip_window():
|
||||
from fabledassistant.services.weather import _summarize_precip
|
||||
hourly = [(h, 5) for h in range(24)]
|
||||
for h in range(14, 18):
|
||||
hourly[h] = (h, 65)
|
||||
result = _summarize_precip(hourly)
|
||||
assert result is not None
|
||||
assert "2 PM" in result
|
||||
assert "65%" in result
|
||||
|
||||
|
||||
def test_parse_weather_card_includes_hourly_precip_summary():
|
||||
from datetime import datetime, timezone, timedelta, date
|
||||
from fabledassistant.services.weather import parse_weather_card_data
|
||||
|
||||
today = date.today().isoformat()
|
||||
yesterday = (date.today() - timedelta(days=1)).isoformat()
|
||||
tomorrow = (date.today() + timedelta(days=1)).isoformat()
|
||||
|
||||
hourly_times = [f"{today}T{h:02d}:00" for h in range(24)]
|
||||
hourly_probs = [5] * 24
|
||||
for h in range(14, 18):
|
||||
hourly_probs[h] = 70
|
||||
|
||||
cache = MagicMock()
|
||||
cache.fetched_at = datetime.now(timezone.utc)
|
||||
cache.location_label = "Berlin, DE"
|
||||
cache.forecast_json = {
|
||||
"current_weather": {"temperature": 12.0, "weathercode": 61},
|
||||
"daily": {
|
||||
"time": [yesterday, today, tomorrow],
|
||||
"temperature_2m_max": [14.0, 16.0, 18.0],
|
||||
"temperature_2m_min": [9.0, 8.0, 10.0],
|
||||
"precipitation_sum": [0.0, 5.0, 1.5],
|
||||
"precipitation_probability_max": [0, 70, 30],
|
||||
"weathercode": [1, 61, 61],
|
||||
"windspeed_10m_max": [10.0, 12.0, 8.0],
|
||||
},
|
||||
"hourly": {
|
||||
"time": hourly_times,
|
||||
"precipitation_probability": hourly_probs,
|
||||
},
|
||||
}
|
||||
|
||||
card = parse_weather_card_data(cache)
|
||||
assert card is not None
|
||||
assert card["precip_summary"] is not None
|
||||
assert "2 PM" in card["precip_summary"]
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
"""Tests for the Wikipedia service module."""
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from fabledassistant.services.wikipedia import wiki_summary, wiki_search
|
||||
|
||||
|
||||
def _make_mock_response(json_data: dict, status_code: int = 200) -> MagicMock:
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status_code = status_code
|
||||
mock_resp.json.return_value = json_data
|
||||
mock_resp.raise_for_status = MagicMock()
|
||||
return mock_resp
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wiki_summary_returns_extract():
|
||||
"""Successful lookup returns a dict with title, extract, and url."""
|
||||
payload = {
|
||||
"type": "standard",
|
||||
"title": "Python (programming language)",
|
||||
"extract": "Python is a high-level programming language.",
|
||||
"content_urls": {"desktop": {"page": "https://en.wikipedia.org/wiki/Python_(programming_language)"}},
|
||||
}
|
||||
mock_resp = _make_mock_response(payload)
|
||||
|
||||
with patch("fabledassistant.services.wikipedia.httpx.AsyncClient") as mock_client_cls:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||||
mock_client.get = AsyncMock(return_value=mock_resp)
|
||||
mock_client_cls.return_value = mock_client
|
||||
|
||||
result = await wiki_summary("Python programming language")
|
||||
|
||||
assert result is not None
|
||||
assert result["title"] == "Python (programming language)"
|
||||
assert "Python" in result["extract"]
|
||||
assert result["url"].startswith("https://")
|
||||
assert result["thumbnail_url"] == ""
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wiki_summary_includes_thumbnail():
|
||||
"""Thumbnail URL is extracted when the article has an image."""
|
||||
payload = {
|
||||
"type": "standard",
|
||||
"title": "Hedgehog",
|
||||
"extract": "Hedgehogs are spiny mammals.",
|
||||
"content_urls": {"desktop": {"page": "https://en.wikipedia.org/wiki/Hedgehog"}},
|
||||
"thumbnail": {"source": "https://upload.wikimedia.org/foo-320px.jpg"},
|
||||
"originalimage": {"source": "https://upload.wikimedia.org/foo.jpg"},
|
||||
}
|
||||
mock_resp = _make_mock_response(payload)
|
||||
|
||||
with patch("fabledassistant.services.wikipedia.httpx.AsyncClient") as mock_client_cls:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||||
mock_client.get = AsyncMock(return_value=mock_resp)
|
||||
mock_client_cls.return_value = mock_client
|
||||
|
||||
result = await wiki_summary("Hedgehog")
|
||||
|
||||
assert result is not None
|
||||
assert result["thumbnail_url"] == "https://upload.wikimedia.org/foo-320px.jpg"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wiki_summary_returns_none_on_404():
|
||||
"""HTTP error (e.g. 404) causes wiki_summary to return None."""
|
||||
import httpx
|
||||
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.raise_for_status.side_effect = httpx.HTTPStatusError(
|
||||
"404", request=MagicMock(), response=MagicMock()
|
||||
)
|
||||
|
||||
with patch("fabledassistant.services.wikipedia.httpx.AsyncClient") as mock_client_cls:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||||
mock_client.get = AsyncMock(return_value=mock_resp)
|
||||
mock_client_cls.return_value = mock_client
|
||||
|
||||
result = await wiki_summary("NonExistentPageXYZ123")
|
||||
|
||||
assert result is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wiki_summary_returns_none_on_disambiguation():
|
||||
"""Disambiguation pages cause wiki_summary to return None."""
|
||||
payload = {
|
||||
"type": "disambiguation",
|
||||
"title": "Mercury",
|
||||
"extract": "Mercury may refer to:",
|
||||
"content_urls": {"desktop": {"page": "https://en.wikipedia.org/wiki/Mercury"}},
|
||||
}
|
||||
mock_resp = _make_mock_response(payload)
|
||||
|
||||
with patch("fabledassistant.services.wikipedia.httpx.AsyncClient") as mock_client_cls:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||||
mock_client.get = AsyncMock(return_value=mock_resp)
|
||||
mock_client_cls.return_value = mock_client
|
||||
|
||||
result = await wiki_summary("Mercury")
|
||||
|
||||
assert result is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wiki_search_returns_results():
|
||||
"""wiki_search returns a list of result dicts for a successful query."""
|
||||
search_payload = {
|
||||
"query": {
|
||||
"search": [
|
||||
{"title": "Quantum mechanics"},
|
||||
{"title": "Quantum field theory"},
|
||||
]
|
||||
}
|
||||
}
|
||||
summary_payloads = [
|
||||
{
|
||||
"type": "standard",
|
||||
"title": "Quantum mechanics",
|
||||
"extract": "Quantum mechanics is a fundamental theory in physics.",
|
||||
"content_urls": {"desktop": {"page": "https://en.wikipedia.org/wiki/Quantum_mechanics"}},
|
||||
},
|
||||
{
|
||||
"type": "standard",
|
||||
"title": "Quantum field theory",
|
||||
"extract": "Quantum field theory is a framework in theoretical physics.",
|
||||
"content_urls": {"desktop": {"page": "https://en.wikipedia.org/wiki/Quantum_field_theory"}},
|
||||
},
|
||||
]
|
||||
|
||||
search_resp = _make_mock_response(search_payload)
|
||||
summary_resp_1 = _make_mock_response(summary_payloads[0])
|
||||
summary_resp_2 = _make_mock_response(summary_payloads[1])
|
||||
|
||||
with patch("fabledassistant.services.wikipedia.httpx.AsyncClient") as mock_client_cls:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||||
mock_client.get = AsyncMock(side_effect=[search_resp, summary_resp_1, summary_resp_2])
|
||||
mock_client_cls.return_value = mock_client
|
||||
|
||||
results = await wiki_search("quantum physics", limit=2)
|
||||
|
||||
assert len(results) == 2
|
||||
assert results[0]["title"] == "Quantum mechanics"
|
||||
assert "quantum" in results[0]["extract"].lower()
|
||||
assert results[1]["title"] == "Quantum field theory"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wiki_search_returns_empty_on_failure():
|
||||
"""wiki_search returns an empty list when the network request fails."""
|
||||
import httpx
|
||||
|
||||
with patch("fabledassistant.services.wikipedia.httpx.AsyncClient") as mock_client_cls:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
||||
mock_client.__aexit__ = AsyncMock(return_value=False)
|
||||
mock_client.get = AsyncMock(side_effect=httpx.ConnectError("connection refused"))
|
||||
mock_client_cls.return_value = mock_client
|
||||
|
||||
results = await wiki_search("anything")
|
||||
|
||||
assert results == []
|
||||
Reference in New Issue
Block a user