feat: full article fetching with trafilatura + html2text cleanup

- Add trafilatura + html2text to dependencies
- Replace custom HTMLStripper with html2text for RSS feed content
- Fetch full article text via httpx + trafilatura after each new item is stored;
  falls back to RSS-provided content if fetch/extraction fails
- Raise CONTENT_MAX_CHARS from 2000 to 50000 (TEXT column, no migration needed)
- Re-embed items with full article content once enrichment completes
- Startup backfill enriches existing items with short content (<1000 chars)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-30 16:33:27 -04:00
parent 0b05b03987
commit e613485474
4 changed files with 157 additions and 54 deletions
@@ -297,3 +297,59 @@ async def backfill_rss_item_embeddings() -> None:
await asyncio.sleep(0.05)
logger.info("RSS embedding backfill complete: %d/%d items embedded", success, len(items_to_embed))
async def backfill_rss_article_content() -> None:
"""Fetch full article text for RSS items that only have short feed-provided content.
An item is considered unenriched if its content is shorter than 1000 chars —
typical of feed summaries/teasers rather than full articles.
Runs at startup after the embedding backfill.
"""
from fabledassistant.services.rss import _fetch_full_article, CONTENT_MAX_CHARS
from fabledassistant.models.rss_feed import RssFeed
SHORT_THRESHOLD = 1000
try:
async with async_session() as session:
feed_result = await session.execute(select(RssFeed.id, RssFeed.user_id))
feed_user_map = {fid: uid for fid, uid in feed_result.fetchall()}
item_result = await session.execute(
select(RssItem.id, RssItem.feed_id, RssItem.url, RssItem.title, RssItem.content)
.where(RssItem.url != "")
)
candidates = [
row for row in item_result.fetchall()
if len(row[4] or "") < SHORT_THRESHOLD
]
except Exception:
logger.warning("Article content backfill: failed to query items", exc_info=True)
return
if not candidates:
logger.info("Article content backfill: no unenriched items found")
return
logger.info("Article content backfill: enriching %d items", len(candidates))
enriched = 0
for item_id, feed_id, url, title, _ in candidates:
user_id = feed_user_map.get(feed_id)
if user_id is None:
continue
full_text = await _fetch_full_article(url)
if full_text and len(full_text) > SHORT_THRESHOLD:
try:
async with async_session() as session:
item = await session.get(RssItem, item_id)
if item:
item.content = full_text[:CONTENT_MAX_CHARS]
await session.commit()
await upsert_rss_item_embedding(item_id, user_id, title or "", full_text[:CONTENT_MAX_CHARS])
enriched += 1
except Exception:
logger.debug("Failed to store enriched content for item %d", item_id, exc_info=True)
await asyncio.sleep(0.5)
logger.info("Article content backfill complete: %d/%d items enriched", enriched, len(candidates))