feat: full article fetching with trafilatura + html2text cleanup

- Add trafilatura + html2text to dependencies
- Replace custom HTMLStripper with html2text for RSS feed content
- Fetch full article text via httpx + trafilatura after each new item is stored;
  falls back to RSS-provided content if fetch/extraction fails
- Raise CONTENT_MAX_CHARS from 2000 to 50000 (TEXT column, no migration needed)
- Re-embed items with full article content once enrichment completes
- Startup backfill enriches existing items with short content (<1000 chars)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

pyproject.toml

@@ -19,6 +19,8 @@ dependencies = [
     "caldav>=1.3",
     "icalendar>=5.0",
     "feedparser>=6.0",
+    "html2text>=2024.2",
+    "trafilatura>=1.12",
     "APScheduler>=3.10,<4.0",
     "pywebpush>=2.0",
 ]

src/fabledassistant/app.py

@@ -266,6 +266,11 @@ def create_app() -> Quart:
                 await backfill_rss_item_embeddings()
             except Exception:
                 logger.warning("RSS embedding backfill failed", exc_info=True)
+            try:
+                from fabledassistant.services.embeddings import backfill_rss_article_content
+                await backfill_rss_article_content()
+            except Exception:
+                logger.warning("RSS article content backfill failed", exc_info=True)
 
         asyncio.create_task(_delayed_backfill())
 

src/fabledassistant/services/embeddings.py

@@ -297,3 +297,59 @@ async def backfill_rss_item_embeddings() -> None:
         await asyncio.sleep(0.05)
 
     logger.info("RSS embedding backfill complete: %d/%d items embedded", success, len(items_to_embed))
+
+
+async def backfill_rss_article_content() -> None:
+    """Fetch full article text for RSS items that only have short feed-provided content.
+
+    An item is considered unenriched if its content is shorter than 1000 chars —
+    typical of feed summaries/teasers rather than full articles.
+    Runs at startup after the embedding backfill.
+    """
+    from fabledassistant.services.rss import _fetch_full_article, CONTENT_MAX_CHARS
+    from fabledassistant.models.rss_feed import RssFeed
+
+    SHORT_THRESHOLD = 1000
+
+    try:
+        async with async_session() as session:
+            feed_result = await session.execute(select(RssFeed.id, RssFeed.user_id))
+            feed_user_map = {fid: uid for fid, uid in feed_result.fetchall()}
+
+            item_result = await session.execute(
+                select(RssItem.id, RssItem.feed_id, RssItem.url, RssItem.title, RssItem.content)
+                .where(RssItem.url != "")
+            )
+            candidates = [
+                row for row in item_result.fetchall()
+                if len(row[4] or "") < SHORT_THRESHOLD
+            ]
+    except Exception:
+        logger.warning("Article content backfill: failed to query items", exc_info=True)
+        return
+
+    if not candidates:
+        logger.info("Article content backfill: no unenriched items found")
+        return
+
+    logger.info("Article content backfill: enriching %d items", len(candidates))
+    enriched = 0
+    for item_id, feed_id, url, title, _ in candidates:
+        user_id = feed_user_map.get(feed_id)
+        if user_id is None:
+            continue
+        full_text = await _fetch_full_article(url)
+        if full_text and len(full_text) > SHORT_THRESHOLD:
+            try:
+                async with async_session() as session:
+                    item = await session.get(RssItem, item_id)
+                    if item:
+                        item.content = full_text[:CONTENT_MAX_CHARS]
+                        await session.commit()
+                await upsert_rss_item_embedding(item_id, user_id, title or "", full_text[:CONTENT_MAX_CHARS])
+                enriched += 1
+            except Exception:
+                logger.debug("Failed to store enriched content for item %d", item_id, exc_info=True)
+        await asyncio.sleep(0.5)
+
+    logger.info("Article content backfill complete: %d/%d items enriched", enriched, len(candidates))

src/fabledassistant/services/rss.py

@@ -2,11 +2,11 @@
 
 import asyncio
 import logging
-import re
 from datetime import datetime, timezone
-from html.parser import HTMLParser
 
 import feedparser
+import html2text
+import httpx
 from sqlalchemy import select, text
 
 from fabledassistant.models import async_session
@@ -14,60 +14,61 @@ from fabledassistant.models.rss_feed import RssFeed, RssItem
 
 logger = logging.getLogger(__name__)
 
-CONTENT_MAX_CHARS = 2000
+# Safety cap on stored content — effectively unlimited for typical articles
+CONTENT_MAX_CHARS = 50_000
 # Keep only items from the last N days to avoid unbounded growth
 ITEM_MAX_AGE_DAYS = 90
 
-
+_h2t = html2text.HTML2Text()
-class _HTMLStripper(HTMLParser):
+_h2t.ignore_links = True
-    """Minimal HTML-to-text converter using stdlib only."""
+_h2t.ignore_images = True
-
+_h2t.ignore_emphasis = False
-    # Tags that should produce a line break in the output
+_h2t.body_width = 0  # No line-wrapping
-    BLOCK_TAGS = frozenset({
-        "p", "br", "li", "h1", "h2", "h3", "h4", "h5", "h6",
-        "div", "section", "article", "blockquote", "tr",
-    })
-    # Tags whose inner text should be skipped entirely
-    SKIP_TAGS = frozenset({"script", "style", "noscript"})
-
-    def __init__(self) -> None:
-        super().__init__(convert_charrefs=True)
-        self._parts: list[str] = []
-        self._skip_depth = 0
-
-    def handle_starttag(self, tag: str, attrs: list) -> None:
-        if tag in self.SKIP_TAGS:
-            self._skip_depth += 1
-        elif tag in self.BLOCK_TAGS and self._skip_depth == 0:
-            self._parts.append("\n")
-
-    def handle_endtag(self, tag: str) -> None:
-        if tag in self.SKIP_TAGS and self._skip_depth > 0:
-            self._skip_depth -= 1
-
-    def handle_data(self, data: str) -> None:
-        if self._skip_depth == 0:
-            self._parts.append(data)
-
-    def get_text(self) -> str:
-        raw = "".join(self._parts)
-        # Collapse runs of whitespace/blank lines to single spaces / newlines
-        raw = re.sub(r"[ \t]+", " ", raw)
-        raw = re.sub(r"\n{3,}", "\n\n", raw)
-        return raw.strip()
 
 
-def _strip_html(html: str) -> str:
+def _html_to_text(html: str) -> str:
-    """Strip HTML tags and return clean plain text."""
+    """Convert HTML to clean plain text via html2text."""
-    if not html or not ("<" in html and ">" in html):
+    if not html or "<" not in html:
-        return html  # Already plain text — skip parsing
+        return html
-    stripper = _HTMLStripper()
     try:
-        stripper.feed(html)
+        return _h2t.handle(html).strip()
-        return stripper.get_text()
     except Exception:
-        # Fallback: crude tag removal
+        return html
-        return re.sub(r"<[^>]+>", " ", html).strip()
+
+
+async def _fetch_full_article(url: str) -> str | None:
+    """Fetch a URL and extract its main article text via trafilatura.
+
+    Returns clean plain text, or None if extraction fails or yields nothing useful.
+    Runs trafilatura in a thread executor since it does synchronous HTML parsing.
+    """
+    try:
+        async with httpx.AsyncClient(timeout=15.0, follow_redirects=True, headers={
+            "User-Agent": "Mozilla/5.0 (compatible; FabledAssistant/1.0; +https://fabledsword.com)",
+        }) as client:
+            resp = await client.get(url)
+            resp.raise_for_status()
+            raw_html = resp.text
+    except Exception:
+        logger.debug("Failed to fetch article URL %s", url)
+        return None
+
+    loop = asyncio.get_event_loop()
+    try:
+        import trafilatura
+        text = await loop.run_in_executor(
+            None,
+            lambda: trafilatura.extract(
+                raw_html,
+                include_comments=False,
+                include_tables=True,
+                favor_recall=True,
+            ),
+        )
+        return text or None
+    except Exception:
+        logger.debug("trafilatura extraction failed for %s", url, exc_info=True)
+        return None
 
 
 def extract_item(entry) -> dict:
@@ -79,7 +80,7 @@ def extract_item(entry) -> dict:
         content = raw_content[0].value
     else:
         content = entry.get("summary", "")
-    content = _strip_html(content)[:CONTENT_MAX_CHARS]
+    content = _html_to_text(content)[:CONTENT_MAX_CHARS]
 
     pub = None
     if entry.published_parsed:
@@ -186,16 +187,55 @@ async def fetch_and_cache_feed(feed_id: int, url: str) -> int:
         from fabledassistant.services.rss_classifier import classify_and_store
         asyncio.create_task(classify_and_store(unclassified_ids, feed_user_id))
 
-    # Fire-and-forget embedding for new items
+    # Collect (id, url) for newly inserted items to enrich with full article text
-    if new_item_data and feed_user_id is not None:
+    new_items_for_enrichment: list[tuple[int, str]] = []
+    if new_count > 0:
+        async with async_session() as session:
+            result = await session.execute(
+                select(RssItem.id, RssItem.url).where(
+                    RssItem.feed_id == feed_id,
+                    RssItem.url != "",
+                ).order_by(RssItem.fetched_at.desc()).limit(new_count)
+            )
+            new_items_for_enrichment = list(result.fetchall())
+
+    # Fire-and-forget: fetch full article text, then re-embed with richer content
+    if new_items_for_enrichment and feed_user_id is not None:
         from fabledassistant.services.embeddings import upsert_rss_item_embedding
 
-        async def _embed_new_items() -> None:
+        async def _enrich_and_embed() -> None:
+            for item_id, article_url in new_items_for_enrichment:
+                full_text = await _fetch_full_article(article_url)
+                if full_text and len(full_text) > 200:
+                    async with async_session() as session:
+                        item = await session.get(RssItem, item_id)
+                        if item:
+                            item.content = full_text[:CONTENT_MAX_CHARS]
+                            await session.commit()
+                            await upsert_rss_item_embedding(
+                                item_id, feed_user_id, item.title or "", item.content
+                            )
+                else:
+                    # Enrich failed — still embed with RSS-provided content
+                    async with async_session() as session:
+                        item = await session.get(RssItem, item_id)
+                    if item:
+                        await upsert_rss_item_embedding(
+                            item_id, feed_user_id, item.title or "", item.content or ""
+                        )
+                await asyncio.sleep(0.5)  # Polite pacing between article fetches
+
+        asyncio.create_task(_enrich_and_embed())
+    elif new_item_data and feed_user_id is not None:
+        # No URLs to enrich — embed with RSS content only
+        from fabledassistant.services.embeddings import upsert_rss_item_embedding
+
+        async def _embed_only() -> None:
             for item_id, title, content in new_item_data:
                 await upsert_rss_item_embedding(item_id, feed_user_id, title, content)
                 await asyncio.sleep(0.05)
 
-        asyncio.create_task(_embed_new_items())
+        asyncio.create_task(_embed_only())
 
     return new_count