feat: full article fetching with trafilatura + html2text cleanup
- Add trafilatura + html2text to dependencies - Replace custom HTMLStripper with html2text for RSS feed content - Fetch full article text via httpx + trafilatura after each new item is stored; falls back to RSS-provided content if fetch/extraction fails - Raise CONTENT_MAX_CHARS from 2000 to 50000 (TEXT column, no migration needed) - Re-embed items with full article content once enrichment completes - Startup backfill enriches existing items with short content (<1000 chars) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -297,3 +297,59 @@ async def backfill_rss_item_embeddings() -> None:
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
logger.info("RSS embedding backfill complete: %d/%d items embedded", success, len(items_to_embed))
|
||||
|
||||
|
||||
async def backfill_rss_article_content() -> None:
|
||||
"""Fetch full article text for RSS items that only have short feed-provided content.
|
||||
|
||||
An item is considered unenriched if its content is shorter than 1000 chars —
|
||||
typical of feed summaries/teasers rather than full articles.
|
||||
Runs at startup after the embedding backfill.
|
||||
"""
|
||||
from fabledassistant.services.rss import _fetch_full_article, CONTENT_MAX_CHARS
|
||||
from fabledassistant.models.rss_feed import RssFeed
|
||||
|
||||
SHORT_THRESHOLD = 1000
|
||||
|
||||
try:
|
||||
async with async_session() as session:
|
||||
feed_result = await session.execute(select(RssFeed.id, RssFeed.user_id))
|
||||
feed_user_map = {fid: uid for fid, uid in feed_result.fetchall()}
|
||||
|
||||
item_result = await session.execute(
|
||||
select(RssItem.id, RssItem.feed_id, RssItem.url, RssItem.title, RssItem.content)
|
||||
.where(RssItem.url != "")
|
||||
)
|
||||
candidates = [
|
||||
row for row in item_result.fetchall()
|
||||
if len(row[4] or "") < SHORT_THRESHOLD
|
||||
]
|
||||
except Exception:
|
||||
logger.warning("Article content backfill: failed to query items", exc_info=True)
|
||||
return
|
||||
|
||||
if not candidates:
|
||||
logger.info("Article content backfill: no unenriched items found")
|
||||
return
|
||||
|
||||
logger.info("Article content backfill: enriching %d items", len(candidates))
|
||||
enriched = 0
|
||||
for item_id, feed_id, url, title, _ in candidates:
|
||||
user_id = feed_user_map.get(feed_id)
|
||||
if user_id is None:
|
||||
continue
|
||||
full_text = await _fetch_full_article(url)
|
||||
if full_text and len(full_text) > SHORT_THRESHOLD:
|
||||
try:
|
||||
async with async_session() as session:
|
||||
item = await session.get(RssItem, item_id)
|
||||
if item:
|
||||
item.content = full_text[:CONTENT_MAX_CHARS]
|
||||
await session.commit()
|
||||
await upsert_rss_item_embedding(item_id, user_id, title or "", full_text[:CONTENT_MAX_CHARS])
|
||||
enriched += 1
|
||||
except Exception:
|
||||
logger.debug("Failed to store enriched content for item %d", item_id, exc_info=True)
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
logger.info("Article content backfill complete: %d/%d items enriched", enriched, len(candidates))
|
||||
|
||||
Reference in New Issue
Block a user