"""RSS feed service: fetch, parse with feedparser, and cache items to DB.""" import asyncio import logging from datetime import datetime, timezone import feedparser from sqlalchemy import select, text from fabledassistant.models import async_session from fabledassistant.models.rss_feed import RssFeed, RssItem logger = logging.getLogger(__name__) CONTENT_MAX_CHARS = 2000 # Keep only items from the last N days to avoid unbounded growth ITEM_MAX_AGE_DAYS = 90 def extract_item(entry) -> dict: """Extract a clean item dict from a feedparser entry object.""" # Prefer full content over summary (feedparser uses a list of Content objects) content = "" raw_content = getattr(entry, "content", None) if isinstance(raw_content, list) and raw_content: content = raw_content[0].value else: content = entry.get("summary", "") content = content[:CONTENT_MAX_CHARS] pub = None if entry.published_parsed: try: pub = datetime(*entry.published_parsed[:6], tzinfo=timezone.utc) except Exception: pass return { "guid": entry.get("id", entry.get("link", "")), "title": entry.get("title", ""), "url": entry.get("link", ""), "content": content, "published_at": pub, } async def _parse_feed(url: str) -> feedparser.FeedParserDict: """Run feedparser in a thread executor so we don't block the event loop.""" loop = asyncio.get_event_loop() return await loop.run_in_executor(None, feedparser.parse, url) async def fetch_and_cache_feed(feed_id: int, url: str) -> int: """ Fetch a feed URL, parse it, and upsert new items into rss_items. Returns the number of new items stored. """ scheme = url.split("://")[0].lower() if "://" in url else "" if scheme not in ("http", "https"): logger.warning("Blocked RSS fetch with non-http(s) scheme: %s", url[:80]) return 0 try: parsed = await _parse_feed(url) except Exception: logger.warning("Failed to fetch RSS feed %s", url, exc_info=True) return 0 if parsed.bozo and not parsed.entries: logger.warning("Malformed RSS feed %s: %s", url, parsed.bozo_exception) return 0 new_count = 0 feed_user_id: int | None = None async with async_session() as session: for entry in parsed.entries: item_data = extract_item(entry) if not item_data["guid"]: continue # Check if already stored existing = await session.execute( select(RssItem).where( RssItem.feed_id == feed_id, RssItem.guid == item_data["guid"], ) ) if existing.scalars().first() is not None: continue item = RssItem( feed_id=feed_id, **item_data, ) session.add(item) new_count += 1 # Update last_fetched_at on the feed feed_row = await session.get(RssFeed, feed_id) if feed_row: feed_row.last_fetched_at = datetime.now(timezone.utc) feed_user_id = feed_row.user_id # Auto-populate title from feed metadata if blank if not feed_row.title and parsed.feed.get("title"): feed_row.title = parsed.feed.title[:200] await session.commit() # Collect IDs of unclassified items after commit. # We query classified_at IS NULL (not just the items inserted above) because # classification is best-effort and may have failed on previous fetches. # Re-queuing all unclassified items for this feed on each fetch is intentional: # it provides automatic retry without a separate retry loop. The classifier # only writes to items it successfully classifies, so already-classified items # are not re-processed (they have classified_at set). unclassified_ids: list[int] = [] if new_count > 0: result = await session.execute( select(RssItem.id).where( RssItem.feed_id == feed_id, RssItem.classified_at.is_(None), ) ) unclassified_ids = list(result.scalars().all()) # Prune old items to keep DB tidy await _prune_old_items(feed_id) # Fire-and-forget classification for unclassified items if unclassified_ids and feed_user_id is not None: from fabledassistant.services.rss_classifier import classify_and_store asyncio.create_task(classify_and_store(unclassified_ids, feed_user_id)) return new_count async def _prune_old_items(feed_id: int) -> None: """Delete items older than ITEM_MAX_AGE_DAYS from a feed.""" async with async_session() as session: await session.execute( text(""" DELETE FROM rss_items WHERE feed_id = :feed_id AND published_at < NOW() - INTERVAL '90 days' """).bindparams(feed_id=feed_id) ) await session.commit() async def get_recent_items(user_id: int, limit: int = 20) -> list[dict]: """Return the most recent RSS items across all of a user's feeds.""" async with async_session() as session: result = await session.execute( select(RssItem, RssFeed.title.label("feed_title")) .join(RssFeed, RssItem.feed_id == RssFeed.id) .where(RssFeed.user_id == user_id) .order_by(RssItem.published_at.desc().nullslast()) .limit(limit) ) rows = result.all() return [ {**item.to_dict(), "feed_title": feed_title} for item, feed_title in rows ] async def refresh_all_feeds(user_id: int) -> dict[int, int]: """Fetch all feeds for a user. Returns {feed_id: new_items_count}.""" async with async_session() as session: result = await session.execute( select(RssFeed).where(RssFeed.user_id == user_id) ) feeds = list(result.scalars().all()) results = {} for feed in feeds: count = await fetch_and_cache_feed(feed.id, feed.url) results[feed.id] = count return results