From e613485474f23ac1702e4ea62c10ff8f8dd654d6 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 30 Mar 2026 16:33:27 -0400 Subject: [PATCH] feat: full article fetching with trafilatura + html2text cleanup - Add trafilatura + html2text to dependencies - Replace custom HTMLStripper with html2text for RSS feed content - Fetch full article text via httpx + trafilatura after each new item is stored; falls back to RSS-provided content if fetch/extraction fails - Raise CONTENT_MAX_CHARS from 2000 to 50000 (TEXT column, no migration needed) - Re-embed items with full article content once enrichment completes - Startup backfill enriches existing items with short content (<1000 chars) Co-Authored-By: Claude Sonnet 4.6 --- pyproject.toml | 2 + src/fabledassistant/app.py | 5 + src/fabledassistant/services/embeddings.py | 56 ++++++++ src/fabledassistant/services/rss.py | 148 +++++++++++++-------- 4 files changed, 157 insertions(+), 54 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f097880..c530669 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,8 @@ dependencies = [ "caldav>=1.3", "icalendar>=5.0", "feedparser>=6.0", + "html2text>=2024.2", + "trafilatura>=1.12", "APScheduler>=3.10,<4.0", "pywebpush>=2.0", ] diff --git a/src/fabledassistant/app.py b/src/fabledassistant/app.py index 79bf9f3..13875c3 100644 --- a/src/fabledassistant/app.py +++ b/src/fabledassistant/app.py @@ -266,6 +266,11 @@ def create_app() -> Quart: await backfill_rss_item_embeddings() except Exception: logger.warning("RSS embedding backfill failed", exc_info=True) + try: + from fabledassistant.services.embeddings import backfill_rss_article_content + await backfill_rss_article_content() + except Exception: + logger.warning("RSS article content backfill failed", exc_info=True) asyncio.create_task(_delayed_backfill()) diff --git a/src/fabledassistant/services/embeddings.py b/src/fabledassistant/services/embeddings.py index 66d6a90..4c2e297 100644 --- a/src/fabledassistant/services/embeddings.py +++ b/src/fabledassistant/services/embeddings.py @@ -297,3 +297,59 @@ async def backfill_rss_item_embeddings() -> None: await asyncio.sleep(0.05) logger.info("RSS embedding backfill complete: %d/%d items embedded", success, len(items_to_embed)) + + +async def backfill_rss_article_content() -> None: + """Fetch full article text for RSS items that only have short feed-provided content. + + An item is considered unenriched if its content is shorter than 1000 chars — + typical of feed summaries/teasers rather than full articles. + Runs at startup after the embedding backfill. + """ + from fabledassistant.services.rss import _fetch_full_article, CONTENT_MAX_CHARS + from fabledassistant.models.rss_feed import RssFeed + + SHORT_THRESHOLD = 1000 + + try: + async with async_session() as session: + feed_result = await session.execute(select(RssFeed.id, RssFeed.user_id)) + feed_user_map = {fid: uid for fid, uid in feed_result.fetchall()} + + item_result = await session.execute( + select(RssItem.id, RssItem.feed_id, RssItem.url, RssItem.title, RssItem.content) + .where(RssItem.url != "") + ) + candidates = [ + row for row in item_result.fetchall() + if len(row[4] or "") < SHORT_THRESHOLD + ] + except Exception: + logger.warning("Article content backfill: failed to query items", exc_info=True) + return + + if not candidates: + logger.info("Article content backfill: no unenriched items found") + return + + logger.info("Article content backfill: enriching %d items", len(candidates)) + enriched = 0 + for item_id, feed_id, url, title, _ in candidates: + user_id = feed_user_map.get(feed_id) + if user_id is None: + continue + full_text = await _fetch_full_article(url) + if full_text and len(full_text) > SHORT_THRESHOLD: + try: + async with async_session() as session: + item = await session.get(RssItem, item_id) + if item: + item.content = full_text[:CONTENT_MAX_CHARS] + await session.commit() + await upsert_rss_item_embedding(item_id, user_id, title or "", full_text[:CONTENT_MAX_CHARS]) + enriched += 1 + except Exception: + logger.debug("Failed to store enriched content for item %d", item_id, exc_info=True) + await asyncio.sleep(0.5) + + logger.info("Article content backfill complete: %d/%d items enriched", enriched, len(candidates)) diff --git a/src/fabledassistant/services/rss.py b/src/fabledassistant/services/rss.py index 8674d6f..606159f 100644 --- a/src/fabledassistant/services/rss.py +++ b/src/fabledassistant/services/rss.py @@ -2,11 +2,11 @@ import asyncio import logging -import re from datetime import datetime, timezone -from html.parser import HTMLParser import feedparser +import html2text +import httpx from sqlalchemy import select, text from fabledassistant.models import async_session @@ -14,60 +14,61 @@ from fabledassistant.models.rss_feed import RssFeed, RssItem logger = logging.getLogger(__name__) -CONTENT_MAX_CHARS = 2000 +# Safety cap on stored content — effectively unlimited for typical articles +CONTENT_MAX_CHARS = 50_000 # Keep only items from the last N days to avoid unbounded growth ITEM_MAX_AGE_DAYS = 90 - -class _HTMLStripper(HTMLParser): - """Minimal HTML-to-text converter using stdlib only.""" - - # Tags that should produce a line break in the output - BLOCK_TAGS = frozenset({ - "p", "br", "li", "h1", "h2", "h3", "h4", "h5", "h6", - "div", "section", "article", "blockquote", "tr", - }) - # Tags whose inner text should be skipped entirely - SKIP_TAGS = frozenset({"script", "style", "noscript"}) - - def __init__(self) -> None: - super().__init__(convert_charrefs=True) - self._parts: list[str] = [] - self._skip_depth = 0 - - def handle_starttag(self, tag: str, attrs: list) -> None: - if tag in self.SKIP_TAGS: - self._skip_depth += 1 - elif tag in self.BLOCK_TAGS and self._skip_depth == 0: - self._parts.append("\n") - - def handle_endtag(self, tag: str) -> None: - if tag in self.SKIP_TAGS and self._skip_depth > 0: - self._skip_depth -= 1 - - def handle_data(self, data: str) -> None: - if self._skip_depth == 0: - self._parts.append(data) - - def get_text(self) -> str: - raw = "".join(self._parts) - # Collapse runs of whitespace/blank lines to single spaces / newlines - raw = re.sub(r"[ \t]+", " ", raw) - raw = re.sub(r"\n{3,}", "\n\n", raw) - return raw.strip() +_h2t = html2text.HTML2Text() +_h2t.ignore_links = True +_h2t.ignore_images = True +_h2t.ignore_emphasis = False +_h2t.body_width = 0 # No line-wrapping -def _strip_html(html: str) -> str: - """Strip HTML tags and return clean plain text.""" - if not html or not ("<" in html and ">" in html): - return html # Already plain text — skip parsing - stripper = _HTMLStripper() +def _html_to_text(html: str) -> str: + """Convert HTML to clean plain text via html2text.""" + if not html or "<" not in html: + return html try: - stripper.feed(html) - return stripper.get_text() + return _h2t.handle(html).strip() except Exception: - # Fallback: crude tag removal - return re.sub(r"<[^>]+>", " ", html).strip() + return html + + +async def _fetch_full_article(url: str) -> str | None: + """Fetch a URL and extract its main article text via trafilatura. + + Returns clean plain text, or None if extraction fails or yields nothing useful. + Runs trafilatura in a thread executor since it does synchronous HTML parsing. + """ + try: + async with httpx.AsyncClient(timeout=15.0, follow_redirects=True, headers={ + "User-Agent": "Mozilla/5.0 (compatible; FabledAssistant/1.0; +https://fabledsword.com)", + }) as client: + resp = await client.get(url) + resp.raise_for_status() + raw_html = resp.text + except Exception: + logger.debug("Failed to fetch article URL %s", url) + return None + + loop = asyncio.get_event_loop() + try: + import trafilatura + text = await loop.run_in_executor( + None, + lambda: trafilatura.extract( + raw_html, + include_comments=False, + include_tables=True, + favor_recall=True, + ), + ) + return text or None + except Exception: + logger.debug("trafilatura extraction failed for %s", url, exc_info=True) + return None def extract_item(entry) -> dict: @@ -79,7 +80,7 @@ def extract_item(entry) -> dict: content = raw_content[0].value else: content = entry.get("summary", "") - content = _strip_html(content)[:CONTENT_MAX_CHARS] + content = _html_to_text(content)[:CONTENT_MAX_CHARS] pub = None if entry.published_parsed: @@ -186,16 +187,55 @@ async def fetch_and_cache_feed(feed_id: int, url: str) -> int: from fabledassistant.services.rss_classifier import classify_and_store asyncio.create_task(classify_and_store(unclassified_ids, feed_user_id)) - # Fire-and-forget embedding for new items - if new_item_data and feed_user_id is not None: + # Collect (id, url) for newly inserted items to enrich with full article text + new_items_for_enrichment: list[tuple[int, str]] = [] + if new_count > 0: + async with async_session() as session: + result = await session.execute( + select(RssItem.id, RssItem.url).where( + RssItem.feed_id == feed_id, + RssItem.url != "", + ).order_by(RssItem.fetched_at.desc()).limit(new_count) + ) + new_items_for_enrichment = list(result.fetchall()) + + # Fire-and-forget: fetch full article text, then re-embed with richer content + if new_items_for_enrichment and feed_user_id is not None: from fabledassistant.services.embeddings import upsert_rss_item_embedding - async def _embed_new_items() -> None: + async def _enrich_and_embed() -> None: + for item_id, article_url in new_items_for_enrichment: + full_text = await _fetch_full_article(article_url) + if full_text and len(full_text) > 200: + async with async_session() as session: + item = await session.get(RssItem, item_id) + if item: + item.content = full_text[:CONTENT_MAX_CHARS] + await session.commit() + await upsert_rss_item_embedding( + item_id, feed_user_id, item.title or "", item.content + ) + else: + # Enrich failed — still embed with RSS-provided content + async with async_session() as session: + item = await session.get(RssItem, item_id) + if item: + await upsert_rss_item_embedding( + item_id, feed_user_id, item.title or "", item.content or "" + ) + await asyncio.sleep(0.5) # Polite pacing between article fetches + + asyncio.create_task(_enrich_and_embed()) + elif new_item_data and feed_user_id is not None: + # No URLs to enrich — embed with RSS content only + from fabledassistant.services.embeddings import upsert_rss_item_embedding + + async def _embed_only() -> None: for item_id, title, content in new_item_data: await upsert_rss_item_embedding(item_id, feed_user_id, title, content) await asyncio.sleep(0.05) - asyncio.create_task(_embed_new_items()) + asyncio.create_task(_embed_only()) return new_count