diff --git a/docs/superpowers/plans/2026-04-17-unified-lookup-wikipedia.md b/docs/superpowers/plans/2026-04-17-unified-lookup-wikipedia.md new file mode 100644 index 0000000..c1fc2cb --- /dev/null +++ b/docs/superpowers/plans/2026-04-17-unified-lookup-wikipedia.md @@ -0,0 +1,782 @@ +# Unified Lookup Tool & Wikipedia Integration — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace `search_web` with a unified `lookup` tool (Wikipedia-first, SearXNG fallback) and add Wikipedia as a source in the research pipeline. + +**Architecture:** New `wikipedia.py` service with `wiki_summary` and `wiki_search`. `lookup` tool in `web.py` replaces `search_web`. Research pipeline in `research.py` gains Wikipedia sources alongside SearXNG. All `search_web` references across the codebase are updated. + +**Tech Stack:** Python 3.12, httpx, pytest, asyncio + +--- + +### Task 1: Wikipedia Service Module + +**Files:** +- Create: `src/fabledassistant/services/wikipedia.py` +- Create: `tests/test_wikipedia.py` + +- [ ] **Step 1: Write failing tests for `wiki_summary`** + +```python +# tests/test_wikipedia.py +import pytest +from unittest.mock import AsyncMock, patch, MagicMock +import httpx + + +@pytest.mark.asyncio +async def test_wiki_summary_returns_extract(): + from fabledassistant.services.wikipedia import wiki_summary + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "type": "standard", + "title": "Python (programming language)", + "extract": "Python is a high-level programming language.", + "content_urls": { + "desktop": {"page": "https://en.wikipedia.org/wiki/Python_(programming_language)"} + }, + } + mock_response.raise_for_status = MagicMock() + + with patch("fabledassistant.services.wikipedia.httpx.AsyncClient") as mock_client_cls: + mock_client = AsyncMock() + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + mock_client.get = AsyncMock(return_value=mock_response) + mock_client_cls.return_value = mock_client + + result = await wiki_summary("Python programming language") + + assert result is not None + assert result["title"] == "Python (programming language)" + assert "high-level" in result["extract"] + assert "wikipedia.org" in result["url"] + + +@pytest.mark.asyncio +async def test_wiki_summary_returns_none_on_404(): + from fabledassistant.services.wikipedia import wiki_summary + + with patch("fabledassistant.services.wikipedia.httpx.AsyncClient") as mock_client_cls: + mock_client = AsyncMock() + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + mock_client.get = AsyncMock(side_effect=httpx.HTTPStatusError( + "Not Found", request=MagicMock(), response=MagicMock(status_code=404) + )) + mock_client_cls.return_value = mock_client + + result = await wiki_summary("xyznonexistenttopic123") + + assert result is None + + +@pytest.mark.asyncio +async def test_wiki_summary_returns_none_on_disambiguation(): + from fabledassistant.services.wikipedia import wiki_summary + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "type": "disambiguation", + "title": "Python", + "extract": "Python may refer to...", + } + mock_response.raise_for_status = MagicMock() + + with patch("fabledassistant.services.wikipedia.httpx.AsyncClient") as mock_client_cls: + mock_client = AsyncMock() + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + mock_client.get = AsyncMock(return_value=mock_response) + mock_client_cls.return_value = mock_client + + result = await wiki_summary("Python") + + assert result is None +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `uv run pytest tests/test_wikipedia.py -v` +Expected: FAIL with `ModuleNotFoundError: No module named 'fabledassistant.services.wikipedia'` + +- [ ] **Step 3: Implement `wiki_summary`** + +```python +# src/fabledassistant/services/wikipedia.py +"""Wikipedia API: lightweight topic lookups and article search.""" + +import logging +from urllib.parse import quote as url_quote + +import httpx + +logger = logging.getLogger(__name__) + +_SUMMARY_URL = "https://en.wikipedia.org/api/rest_v1/page/summary" +_SEARCH_URL = "https://en.wikipedia.org/w/api.php" +_TIMEOUT = 5.0 +_USER_AGENT = "FabledAssistant/1.0 (https://fabledsword.com)" + + +async def wiki_summary(query: str) -> dict | None: + """Look up a topic by title via the Wikipedia REST summary endpoint. + + Returns {"title", "extract", "url"} on hit, None on miss. + """ + encoded = url_quote(query.replace(" ", "_"), safe="") + try: + async with httpx.AsyncClient( + timeout=_TIMEOUT, headers={"User-Agent": _USER_AGENT} + ) as client: + resp = await client.get(f"{_SUMMARY_URL}/{encoded}", follow_redirects=True) + resp.raise_for_status() + data = resp.json() + except Exception: + logger.debug("Wikipedia summary lookup failed for %r", query, exc_info=True) + return None + + if data.get("type") == "disambiguation": + return None + + extract = data.get("extract", "").strip() + if not extract: + return None + + url = ( + data.get("content_urls", {}).get("desktop", {}).get("page") + or f"https://en.wikipedia.org/wiki/{encoded}" + ) + return {"title": data.get("title", query), "extract": extract, "url": url} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `uv run pytest tests/test_wikipedia.py -v` +Expected: 3 passed + +- [ ] **Step 5: Write failing tests for `wiki_search`** + +Add to `tests/test_wikipedia.py`: + +```python +@pytest.mark.asyncio +async def test_wiki_search_returns_results(): + from fabledassistant.services.wikipedia import wiki_search + + search_response = MagicMock() + search_response.status_code = 200 + search_response.json.return_value = { + "query": { + "search": [ + {"title": "QUIC"}, + {"title": "HTTP/3"}, + ] + } + } + search_response.raise_for_status = MagicMock() + + summary_response = MagicMock() + summary_response.status_code = 200 + summary_response.json.return_value = { + "type": "standard", + "title": "QUIC", + "extract": "QUIC is a transport layer protocol.", + "content_urls": {"desktop": {"page": "https://en.wikipedia.org/wiki/QUIC"}}, + } + summary_response.raise_for_status = MagicMock() + + with patch("fabledassistant.services.wikipedia.httpx.AsyncClient") as mock_client_cls: + mock_client = AsyncMock() + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + mock_client.get = AsyncMock(side_effect=[search_response, summary_response, summary_response]) + mock_client_cls.return_value = mock_client + + results = await wiki_search("QUIC protocol", limit=2) + + assert len(results) >= 1 + assert results[0]["title"] == "QUIC" + assert "transport" in results[0]["extract"] + + +@pytest.mark.asyncio +async def test_wiki_search_returns_empty_on_failure(): + from fabledassistant.services.wikipedia import wiki_search + + with patch("fabledassistant.services.wikipedia.httpx.AsyncClient") as mock_client_cls: + mock_client = AsyncMock() + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + mock_client.get = AsyncMock(side_effect=httpx.ConnectError("connection failed")) + mock_client_cls.return_value = mock_client + + results = await wiki_search("anything") + + assert results == [] +``` + +- [ ] **Step 6: Implement `wiki_search`** + +Add to `src/fabledassistant/services/wikipedia.py`: + +```python +async def wiki_search(query: str, limit: int = 3) -> list[dict]: + """Search Wikipedia for articles matching a query. + + Returns [{"title", "extract", "url"}, ...] (may be empty). + """ + try: + async with httpx.AsyncClient( + timeout=_TIMEOUT, headers={"User-Agent": _USER_AGENT} + ) as client: + resp = await client.get(_SEARCH_URL, params={ + "action": "query", + "list": "search", + "srsearch": query, + "srlimit": str(limit), + "format": "json", + }) + resp.raise_for_status() + hits = resp.json().get("query", {}).get("search", []) + if not hits: + return [] + + results: list[dict] = [] + for hit in hits: + title = hit.get("title", "") + if not title: + continue + encoded = url_quote(title.replace(" ", "_"), safe="") + try: + summary_resp = await client.get( + f"{_SUMMARY_URL}/{encoded}", follow_redirects=True, + ) + summary_resp.raise_for_status() + data = summary_resp.json() + except Exception: + continue + if data.get("type") == "disambiguation": + continue + extract = data.get("extract", "").strip() + if not extract: + continue + url = ( + data.get("content_urls", {}).get("desktop", {}).get("page") + or f"https://en.wikipedia.org/wiki/{encoded}" + ) + results.append({"title": data.get("title", title), "extract": extract, "url": url}) + return results + except Exception: + logger.debug("Wikipedia search failed for %r", query, exc_info=True) + return [] +``` + +- [ ] **Step 7: Run all wikipedia tests** + +Run: `uv run pytest tests/test_wikipedia.py -v` +Expected: 5 passed + +- [ ] **Step 8: Commit** + +```bash +git add src/fabledassistant/services/wikipedia.py tests/test_wikipedia.py +git commit -m "feat: add wikipedia service with summary lookup and search" +``` + +--- + +### Task 2: Lookup Tool (replaces search_web) + +**Files:** +- Modify: `src/fabledassistant/services/tools/web.py` +- Create: `tests/test_lookup_tool.py` + +- [ ] **Step 1: Write failing tests for `lookup`** + +```python +# tests/test_lookup_tool.py +import pytest +from unittest.mock import AsyncMock, patch, MagicMock + + +@pytest.mark.asyncio +async def test_lookup_wikipedia_hit(): + """lookup returns wikipedia source when wiki_summary succeeds.""" + wiki_data = { + "title": "QUIC", + "extract": "QUIC is a transport layer protocol.", + "url": "https://en.wikipedia.org/wiki/QUIC", + } + + with patch("fabledassistant.services.tools.web.wiki_summary", new_callable=AsyncMock, return_value=wiki_data): + from fabledassistant.services.tools.web import lookup_tool + result = await lookup_tool(user_id=1, arguments={"query": "QUIC"}) + + assert result["success"] is True + assert result["type"] == "lookup" + assert result["source"] == "wikipedia" + assert result["data"]["title"] == "QUIC" + assert "transport" in result["data"]["extract"] + + +@pytest.mark.asyncio +async def test_lookup_wikipedia_miss_searxng_fallback(): + """lookup falls back to SearXNG + article fetch when Wikipedia misses.""" + searxng_results = [ + {"url": "https://example.com/quic", "title": "QUIC Explained", "snippet": "An overview..."}, + ] + + with patch("fabledassistant.services.tools.web.wiki_summary", new_callable=AsyncMock, return_value=None), \ + patch("fabledassistant.services.tools.web.Config") as mock_config, \ + patch("fabledassistant.services.tools.web._search_searxng", new_callable=AsyncMock, return_value=searxng_results), \ + patch("fabledassistant.services.tools.web._fetch_full_article", new_callable=AsyncMock, return_value="Full article about QUIC..."): + mock_config.searxng_enabled.return_value = True + from fabledassistant.services.tools.web import lookup_tool + result = await lookup_tool(user_id=1, arguments={"query": "QUIC"}) + + assert result["success"] is True + assert result["type"] == "lookup" + assert result["source"] == "web" + assert result["data"]["results"] + + +@pytest.mark.asyncio +async def test_lookup_wikipedia_miss_no_searxng(): + """lookup returns no-results when Wikipedia misses and SearXNG is not configured.""" + with patch("fabledassistant.services.tools.web.wiki_summary", new_callable=AsyncMock, return_value=None), \ + patch("fabledassistant.services.tools.web.Config") as mock_config: + mock_config.searxng_enabled.return_value = False + from fabledassistant.services.tools.web import lookup_tool + result = await lookup_tool(user_id=1, arguments={"query": "xyznonexistent"}) + + assert result["success"] is True + assert result["source"] == "none" + + +@pytest.mark.asyncio +async def test_lookup_always_available(): + """lookup tool must appear in get_tools_for_user regardless of SearXNG config.""" + with patch("fabledassistant.services.tools._registry.is_caldav_configured", new_callable=AsyncMock, return_value=False), \ + patch("fabledassistant.services.settings.get_setting", new_callable=AsyncMock, return_value="false"): + from fabledassistant.services.tools import get_tools_for_user + tools = await get_tools_for_user(user_id=1) + tool_names = {t["function"]["name"] for t in tools} + assert "lookup" in tool_names +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `uv run pytest tests/test_lookup_tool.py -v` +Expected: FAIL (no `lookup_tool` function) + +- [ ] **Step 3: Replace `search_web` with `lookup` in `web.py`** + +Replace the `search_web_tool` function (lines 12–36 of `src/fabledassistant/services/tools/web.py`) with: + +```python +@tool( + name="lookup", + description=( + "Look up a topic, concept, or factual question. Returns a concise answer from " + "Wikipedia or web sources. Use for definitions, explanations, 'what is X', " + "'how does Y work', current events, or version numbers. No note is saved. " + "For comprehensive written reports saved as notes, use research_topic instead." + ), + parameters={ + "query": {"type": "string", "description": "The topic or question to look up"}, + }, + required=["query"], +) +async def lookup_tool(*, user_id, arguments, **_ctx): + from fabledassistant.config import Config + from fabledassistant.services.wikipedia import wiki_summary + + query = arguments.get("query", "") + + # 1. Try Wikipedia first + wiki = await wiki_summary(query) + if wiki: + return { + "success": True, + "type": "lookup", + "source": "wikipedia", + "data": wiki, + } + + # 2. Fall back to SearXNG + article fetch + if Config.searxng_enabled(): + from fabledassistant.services.research import _search_searxng + from fabledassistant.services.rss import _fetch_full_article + + results = await _search_searxng(query) + if results: + articles: list[dict] = [] + for r in results[:2]: + url = r.get("url", "") + if not url: + continue + content = await _fetch_full_article(url) + articles.append({ + "url": url, + "title": r.get("title", url), + "snippet": r.get("snippet", ""), + "content": (content or "")[:4000], + }) + if articles: + return { + "success": True, + "type": "lookup", + "source": "web", + "data": {"query": query, "results": articles}, + } + + # 3. No sources available + return { + "success": True, + "type": "lookup", + "source": "none", + "data": { + "query": query, + "message": "No results found. You can answer from your own knowledge.", + }, + } +``` + +- [ ] **Step 4: Run lookup tests to verify they pass** + +Run: `uv run pytest tests/test_lookup_tool.py -v` +Expected: 4 passed + +- [ ] **Step 5: Commit** + +```bash +git add src/fabledassistant/services/tools/web.py tests/test_lookup_tool.py +git commit -m "feat: replace search_web with unified lookup tool (Wikipedia + SearXNG fallback)" +``` + +--- + +### Task 3: Update All `search_web` References + +**Files:** +- Modify: `src/fabledassistant/services/tools/web.py:45` (research_topic description) +- Modify: `src/fabledassistant/services/tools/web.py:67` (search_images description) +- Modify: `src/fabledassistant/services/tools/rss.py:75` (read_article description) +- Modify: `src/fabledassistant/services/generation_task.py:133` (status label map) +- Modify: `src/fabledassistant/services/llm.py:608` (action list) + +- [ ] **Step 1: Update `research_topic` description** + +In `src/fabledassistant/services/tools/web.py`, change the `research_topic` description from: + +```python + "For a quick factual answer without saving a note, use search_web." +``` + +to: + +```python + "For a quick factual answer without saving a note, use lookup." +``` + +- [ ] **Step 2: Update `search_images` description** + +In `src/fabledassistant/services/tools/web.py`, change the `search_images` description from: + +```python + description="Search and display images inline. Use ONLY when the user explicitly asks to see, show, or find an image or photo. Not for factual questions — use search_web for those.", +``` + +to: + +```python + description="Search and display images inline. Use ONLY when the user explicitly asks to see, show, or find an image or photo. Not for factual questions — use lookup for those.", +``` + +- [ ] **Step 3: Update `read_article` description** + +In `src/fabledassistant/services/tools/rss.py`, change: + +```python + "Do NOT use search_web for URLs — use this tool instead." +``` + +to: + +```python + "Do NOT use lookup for URLs — use this tool instead." +``` + +- [ ] **Step 4: Update status label map in `generation_task.py`** + +In `src/fabledassistant/services/generation_task.py`, line 133, change: + +```python + "search_web": "Searching the web", +``` + +to: + +```python + "lookup": "Looking up information", +``` + +- [ ] **Step 5: Update action list in `llm.py`** + +In `src/fabledassistant/services/llm.py`, line 608, change: + +```python + actions.extend(["search_web", "research_topic", "search_images"]) +``` + +to: + +```python + actions.extend(["lookup", "research_topic", "search_images"]) +``` + +- [ ] **Step 6: Run full test suite to check for regressions** + +Run: `uv run pytest tests/ -v` +Expected: All tests pass (no test references `search_web` by name in assertions) + +- [ ] **Step 7: Commit** + +```bash +git add src/fabledassistant/services/tools/web.py src/fabledassistant/services/tools/rss.py src/fabledassistant/services/generation_task.py src/fabledassistant/services/llm.py +git commit -m "refactor: update all search_web references to lookup" +``` + +--- + +### Task 4: Add Wikipedia Sources to Research Pipeline + +**Files:** +- Modify: `src/fabledassistant/services/research.py` +- Modify: `tests/test_research_pipeline.py` + +- [ ] **Step 1: Write failing test for Wikipedia in research pipeline** + +Add to `tests/test_research_pipeline.py`: + +```python +@pytest.mark.asyncio +async def test_pipeline_includes_wikipedia_sources(): + """run_research_pipeline should merge Wikipedia results into the source pool.""" + from unittest.mock import MagicMock + + wiki_results = [{"title": "Wiki Article", "extract": "Wikipedia content about the topic.", "url": "https://en.wikipedia.org/wiki/Topic"}] + + outline = [ + {"title": "Section A", "focus": "Focus A"}, + {"title": "Section B", "focus": "Focus B"}, + ] + + note_id_counter = iter(range(30, 40)) + + def _make_note(user_id, title, body, tags, project_id=None, parent_id=None): + n = MagicMock() + n.id = next(note_id_counter) + n.title = title + return n + + with patch("fabledassistant.services.research._generate_sub_queries", new_callable=AsyncMock, return_value=["q1"]), \ + patch("fabledassistant.services.research._search_searxng", new_callable=AsyncMock, return_value=[{"url": "http://x.com", "title": "X", "snippet": "s"}]), \ + patch("fabledassistant.services.research.wiki_search", new_callable=AsyncMock, return_value=wiki_results), \ + patch("fabledassistant.services.research.fetch_url_content", new_callable=AsyncMock, return_value="content"), \ + patch("fabledassistant.services.research._generate_outline", new_callable=AsyncMock, return_value=outline) as mock_outline, \ + patch("fabledassistant.services.research._synthesize_section", new_callable=AsyncMock, side_effect=lambda t, f, s, m: (t, f"Body for {t}")), \ + patch("fabledassistant.services.research._generate_executive_summary", new_callable=AsyncMock, return_value="Summary."), \ + patch("fabledassistant.services.research.create_note", new_callable=AsyncMock, side_effect=_make_note), \ + patch("fabledassistant.services.research.update_note", new_callable=AsyncMock): + + from fabledassistant.services.research import run_research_pipeline + await run_research_pipeline("test topic", user_id=1, model="test-model") + + # The sources passed to _generate_outline should include the Wikipedia article + sources_arg = mock_outline.call_args[0][1] # second positional arg + source_urls = [s["url"] for s in sources_arg] + assert "https://en.wikipedia.org/wiki/Topic" in source_urls +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/test_research_pipeline.py::test_pipeline_includes_wikipedia_sources -v` +Expected: FAIL (no `wiki_search` import in research.py) + +- [ ] **Step 3: Add Wikipedia sources to the research pipeline** + +In `src/fabledassistant/services/research.py`, add the import at the top (after existing imports): + +```python +from fabledassistant.services.wikipedia import wiki_search +``` + +Then modify Step 2 (the parallel search section, around lines 208–246). Replace: + +```python + # Step 2: Search all queries in parallel (200 ms stagger to avoid hammering SearXNG) + async def _search_with_stagger(i: int, query: str) -> tuple[str, list[dict]]: + if i > 0: + await asyncio.sleep(0.2 * i) + _status(f"Searching: {query}...") + results = await _search_searxng(query) + logger.info("Research: query '%s' → %d results", query, len(results)) + return query, results + + search_results = await asyncio.gather( + *[_search_with_stagger(i, q) for i, q in enumerate(queries)] + ) + + # Deduplicate URLs across all queries + seen_urls: set[str] = set() + url_tasks: list[tuple[str, dict, str]] = [] # (url, result_dict, query) + for query, results in search_results: + for result in results[:PAGES_PER_QUERY]: + url = result.get("url", "") + if url and url not in seen_urls: + seen_urls.add(url) + url_tasks.append((url, result, query)) + + # Fetch all unique URLs in parallel + async def _fetch_source(url: str, result: dict, query: str) -> dict: + title = result.get("title", url) + _status(f"Reading: {title[:60]}...") + content = await fetch_url_content(url) + return { + "url": url, + "title": title, + "query": query, + "snippet": result.get("snippet", ""), + "content": content, + } + + all_sources: list[dict] = list(await asyncio.gather( + *[_fetch_source(url, result, query) for url, result, query in url_tasks] + )) +``` + +with: + +```python + # Step 2: Search all queries in parallel (SearXNG + Wikipedia) + async def _search_with_stagger(i: int, query: str) -> tuple[str, list[dict]]: + if i > 0: + await asyncio.sleep(0.2 * i) + _status(f"Searching: {query}...") + results = await _search_searxng(query) + logger.info("Research: query '%s' → %d results", query, len(results)) + return query, results + + async def _wiki_for_query(query: str) -> list[dict]: + return await wiki_search(query, limit=1) + + searxng_task = asyncio.gather( + *[_search_with_stagger(i, q) for i, q in enumerate(queries)] + ) + wiki_task = asyncio.gather( + *[_wiki_for_query(q) for q in queries] + ) + search_results, wiki_results = await asyncio.gather(searxng_task, wiki_task) + + # Deduplicate URLs across all queries + seen_urls: set[str] = set() + url_tasks: list[tuple[str, dict, str]] = [] # (url, result_dict, query) + wiki_sources: list[dict] = [] # Wikipedia articles (already have content) + + for query, results in search_results: + for result in results[:PAGES_PER_QUERY]: + url = result.get("url", "") + if url and url not in seen_urls: + seen_urls.add(url) + url_tasks.append((url, result, query)) + + # Add Wikipedia results (they already have content via extract) + for query, wiki_hits in zip(queries, wiki_results): + for hit in wiki_hits: + url = hit.get("url", "") + if url and url not in seen_urls: + seen_urls.add(url) + wiki_sources.append({ + "url": url, + "title": hit["title"], + "query": query, + "snippet": hit["extract"][:200], + "content": hit["extract"], + }) + + # Fetch all unique SearXNG URLs in parallel + async def _fetch_source(url: str, result: dict, query: str) -> dict: + title = result.get("title", url) + _status(f"Reading: {title[:60]}...") + content = await fetch_url_content(url) + return { + "url": url, + "title": title, + "query": query, + "snippet": result.get("snippet", ""), + "content": content, + } + + fetched_sources: list[dict] = list(await asyncio.gather( + *[_fetch_source(url, result, query) for url, result, query in url_tasks] + )) + + all_sources = wiki_sources + fetched_sources +``` + +- [ ] **Step 4: Run the new test to verify it passes** + +Run: `uv run pytest tests/test_research_pipeline.py::test_pipeline_includes_wikipedia_sources -v` +Expected: PASS + +- [ ] **Step 5: Run the full research test suite for regressions** + +Run: `uv run pytest tests/test_research_pipeline.py -v` +Expected: All tests pass + +- [ ] **Step 6: Run full test suite** + +Run: `uv run pytest tests/ -v` +Expected: All tests pass + +- [ ] **Step 7: Commit** + +```bash +git add src/fabledassistant/services/research.py tests/test_research_pipeline.py +git commit -m "feat: add Wikipedia as research pipeline source alongside SearXNG" +``` + +--- + +### Task 5: Lint, Typecheck, Final Verification + +**Files:** +- All modified files + +- [ ] **Step 1: Run ruff lint** + +Run: `uv run ruff check src/fabledassistant/services/wikipedia.py src/fabledassistant/services/tools/web.py src/fabledassistant/services/research.py tests/test_wikipedia.py tests/test_lookup_tool.py` +Expected: All checks passed (fix any issues if not) + +- [ ] **Step 2: Run typecheck** + +Run: `cd frontend && npx vue-tsc --noEmit` (frontend unchanged, but verify nothing broke) +Expected: Clean + +- [ ] **Step 3: Run full test suite one last time** + +Run: `uv run pytest tests/ -v` +Expected: All tests pass + +- [ ] **Step 4: Commit any lint fixes if needed** + +```bash +git add -u +git commit -m "fix: lint cleanup for lookup/wikipedia changes" +``` diff --git a/docs/superpowers/specs/2026-04-17-unified-lookup-wikipedia-design.md b/docs/superpowers/specs/2026-04-17-unified-lookup-wikipedia-design.md new file mode 100644 index 0000000..1b68d10 --- /dev/null +++ b/docs/superpowers/specs/2026-04-17-unified-lookup-wikipedia-design.md @@ -0,0 +1,119 @@ +# Unified Lookup Tool & Wikipedia Integration + +## Goal + +Replace the fragmented `search_web` tool with a single `lookup` tool that checks Wikipedia first and falls back to SearXNG web search. Add Wikipedia as an additional source in the research pipeline. Result: one lightweight tool for factual questions (always available, no config required), and richer research output. + +## Architecture + +Two changes to the search/knowledge stack: + +1. **New `lookup` tool** replaces `search_web`. Tries Wikipedia REST API summary endpoint first (~200ms, reliable, no config). Falls back to SearXNG + trafilatura article fetch when Wikipedia misses and SearXNG is configured. Always available (no `requires` field). + +2. **Wikipedia sources in research pipeline.** During sub-query execution, `wiki_search` runs alongside `_search_searxng`. Wikipedia articles merge into the source pool and get deduplicated by URL. + +Shared Wikipedia logic lives in a new `wikipedia.py` service module. + +## Components + +### `src/fabledassistant/services/wikipedia.py` (new) + +Two async functions: + +**`wiki_summary(query: str) -> dict | None`** +- Direct title lookup via `https://en.wikipedia.org/api/rest_v1/page/summary/{title}` +- Returns `{"title": str, "extract": str, "url": str}` on hit +- Returns `None` on 404, disambiguation pages (`"type": "disambiguation"`), network errors, or empty extracts +- 5-second timeout +- User-Agent: `"FabledAssistant/1.0 (https://fabledsword.com)"` + +**`wiki_search(query: str, limit: int = 3) -> list[dict]`** +- Search via `https://en.wikipedia.org/w/api.php?action=query&list=search&srsearch={query}&srlimit={limit}&format=json` +- For each search result, fetch its summary via the summary endpoint to get the extract +- Returns `[{"title": str, "extract": str, "url": str}, ...]` +- Returns `[]` on any failure +- Same timeout and User-Agent as above + +### `src/fabledassistant/services/tools/web.py` (modified) + +**Remove:** `search_web_tool` + +**Add:** `lookup_tool` + +``` +@tool( + name="lookup", + description="Look up a topic, concept, or factual question. Returns a concise + answer from Wikipedia or web sources. Use for definitions, + explanations, 'what is X', 'how does Y work'. For comprehensive + written reports saved as notes, use research_topic instead.", + parameters={ + "query": {"type": "string", "description": "The topic or question to look up"}, + }, + required=["query"], +) +``` + +No `requires` field — always available. + +**Logic:** +1. Call `wiki_summary(query)` +2. If Wikipedia returns a result: return `{"success": True, "type": "lookup", "source": "wikipedia", "data": {"title": ..., "extract": ..., "url": ...}}` +3. If Wikipedia misses and `Config.searxng_enabled()`: + - Call `_search_searxng(query)` to get search results + - Fetch top 1-2 result URLs via `_fetch_full_article` (from `rss.py`, trafilatura-based) + - Return `{"success": True, "type": "lookup", "source": "web", "data": {"query": ..., "results": [...], "content": ...}}` +4. If Wikipedia misses and no SearXNG: return `{"success": True, "type": "lookup", "source": "none", "data": {"query": ..., "message": "No results found. You can answer from your own knowledge."}}` + +### `src/fabledassistant/services/research.py` (modified) + +**In Step 2 (parallel search):** +- For each sub-query, run `wiki_search(query, limit=1)` concurrently with `_search_searxng(query)` +- Merge Wikipedia results into the per-query result list + +**In Step 3 (deduplication):** +- When deduplicating URLs, Wikipedia URLs (`wikipedia.org`) are checked against SearXNG results +- If a Wikipedia article URL already appears in SearXNG results, skip the duplicate + +**Wikipedia article content for synthesis:** +- The `extract` from `wiki_search` is used as the source content (no additional fetch needed, unlike SearXNG URLs which require `fetch_url_content`) +- This means Wikipedia sources are available immediately without an HTTP fetch step + +## Error Handling + +- All Wikipedia API failures (network, timeout, malformed JSON) return `None`/`[]` silently +- `lookup` never raises — always returns a response the model can work with +- In the research pipeline, Wikipedia is purely additive; its failure never degrades existing SearXNG-based research +- Disambiguation pages are detected via `"type": "disambiguation"` in the summary response and treated as a miss + +## Testing + +### `tests/test_wikipedia.py` (new) + +- `test_wiki_summary_returns_extract` — mock successful summary response, verify return shape +- `test_wiki_summary_returns_none_on_404` — mock 404, verify `None` +- `test_wiki_summary_returns_none_on_disambiguation` — mock disambiguation response, verify `None` +- `test_wiki_search_returns_results` — mock search API + summary fetches, verify list +- `test_wiki_search_returns_empty_on_failure` — mock network error, verify `[]` + +### `tests/test_lookup_tool.py` (new) + +- `test_lookup_wikipedia_hit` — mock `wiki_summary` returning data, verify tool returns wikipedia source +- `test_lookup_wikipedia_miss_searxng_fallback` — mock `wiki_summary` returning None, SearXNG returning results + article fetch, verify web source +- `test_lookup_wikipedia_miss_no_searxng` — mock both missing, verify graceful "no results" response +- `test_lookup_always_available` — verify the tool appears in `get_tools_for_user` regardless of SearXNG config + +### `tests/test_research_pipeline.py` (add to existing) + +- `test_research_includes_wikipedia_sources` — mock `wiki_search` alongside SearXNG, verify Wikipedia results appear in source pool + +All tests mock HTTP calls — no live API hits. + +## What Doesn't Change + +- `read_article` tool — stays as-is (explicit URL fetch, different purpose) +- `research_topic` tool definition — stays as-is (same name, description, parameters) +- `generation_task.py` research interception — stays as-is +- `search_images` tool — stays as-is +- `_search_searxng` and `_search_searxng_images` — stay as-is +- `_fetch_full_article` in `rss.py` — stays as-is, reused by `lookup` for SearXNG fallback diff --git a/frontend/src/components/AppHeader.vue b/frontend/src/components/AppHeader.vue index 4be5a93..d366490 100644 --- a/frontend/src/components/AppHeader.vue +++ b/frontend/src/components/AppHeader.vue @@ -5,6 +5,7 @@ import { useTheme } from "@/composables/useTheme"; import { useShortcuts } from "@/composables/useShortcuts"; import { useAuthStore } from "@/stores/auth"; import { useChatStore } from "@/stores/chat"; +import { useSettingsStore } from "@/stores/settings"; import AppLogo from "@/components/AppLogo.vue"; import NotificationBell from "@/components/NotificationBell.vue"; @@ -12,6 +13,7 @@ const { theme, toggleTheme } = useTheme(); const { toggleShortcuts } = useShortcuts(); const authStore = useAuthStore(); const chatStore = useChatStore(); +const settingsStore = useSettingsStore(); const router = useRouter(); const route = useRoute(); @@ -78,7 +80,7 @@ router.afterEach(() => { Chat Briefing Calendar - News + News Projects @@ -128,7 +130,7 @@ router.afterEach(() => { Briefing Calendar Projects - News + News Shared
Settings diff --git a/frontend/src/components/WeatherCard.vue b/frontend/src/components/WeatherCard.vue index 1fbb247..dac6b57 100644 --- a/frontend/src/components/WeatherCard.vue +++ b/frontend/src/components/WeatherCard.vue @@ -9,6 +9,8 @@ interface ForecastDay { precip_probability: number | null precip_mm: number | null windspeed_max: number + precip_summary?: string + precip_peak_hour?: string } interface WeatherData { @@ -21,6 +23,7 @@ interface WeatherData { yesterday_high: number | null yesterday_low: number | null wind_unit?: string + precip_summary?: string | null forecast: ForecastDay[] } @@ -68,6 +71,11 @@ const fetchedAtLabel = computed(() => { return '' } }) + +function hasPrecip(day: ForecastDay): boolean { + return (day.precip_probability != null && day.precip_probability > 0) || + (day.precip_mm != null && day.precip_mm > 0) +}