feat: full article fetching with trafilatura + html2text cleanup
- Add trafilatura + html2text to dependencies - Replace custom HTMLStripper with html2text for RSS feed content - Fetch full article text via httpx + trafilatura after each new item is stored; falls back to RSS-provided content if fetch/extraction fails - Raise CONTENT_MAX_CHARS from 2000 to 50000 (TEXT column, no migration needed) - Re-embed items with full article content once enrichment completes - Startup backfill enriches existing items with short content (<1000 chars) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -2,11 +2,11 @@
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from html.parser import HTMLParser
|
||||
|
||||
import feedparser
|
||||
import html2text
|
||||
import httpx
|
||||
from sqlalchemy import select, text
|
||||
|
||||
from fabledassistant.models import async_session
|
||||
@@ -14,60 +14,61 @@ from fabledassistant.models.rss_feed import RssFeed, RssItem
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CONTENT_MAX_CHARS = 2000
|
||||
# Safety cap on stored content — effectively unlimited for typical articles
|
||||
CONTENT_MAX_CHARS = 50_000
|
||||
# Keep only items from the last N days to avoid unbounded growth
|
||||
ITEM_MAX_AGE_DAYS = 90
|
||||
|
||||
|
||||
class _HTMLStripper(HTMLParser):
|
||||
"""Minimal HTML-to-text converter using stdlib only."""
|
||||
|
||||
# Tags that should produce a line break in the output
|
||||
BLOCK_TAGS = frozenset({
|
||||
"p", "br", "li", "h1", "h2", "h3", "h4", "h5", "h6",
|
||||
"div", "section", "article", "blockquote", "tr",
|
||||
})
|
||||
# Tags whose inner text should be skipped entirely
|
||||
SKIP_TAGS = frozenset({"script", "style", "noscript"})
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self._parts: list[str] = []
|
||||
self._skip_depth = 0
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list) -> None:
|
||||
if tag in self.SKIP_TAGS:
|
||||
self._skip_depth += 1
|
||||
elif tag in self.BLOCK_TAGS and self._skip_depth == 0:
|
||||
self._parts.append("\n")
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
if tag in self.SKIP_TAGS and self._skip_depth > 0:
|
||||
self._skip_depth -= 1
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self._skip_depth == 0:
|
||||
self._parts.append(data)
|
||||
|
||||
def get_text(self) -> str:
|
||||
raw = "".join(self._parts)
|
||||
# Collapse runs of whitespace/blank lines to single spaces / newlines
|
||||
raw = re.sub(r"[ \t]+", " ", raw)
|
||||
raw = re.sub(r"\n{3,}", "\n\n", raw)
|
||||
return raw.strip()
|
||||
_h2t = html2text.HTML2Text()
|
||||
_h2t.ignore_links = True
|
||||
_h2t.ignore_images = True
|
||||
_h2t.ignore_emphasis = False
|
||||
_h2t.body_width = 0 # No line-wrapping
|
||||
|
||||
|
||||
def _strip_html(html: str) -> str:
|
||||
"""Strip HTML tags and return clean plain text."""
|
||||
if not html or not ("<" in html and ">" in html):
|
||||
return html # Already plain text — skip parsing
|
||||
stripper = _HTMLStripper()
|
||||
def _html_to_text(html: str) -> str:
|
||||
"""Convert HTML to clean plain text via html2text."""
|
||||
if not html or "<" not in html:
|
||||
return html
|
||||
try:
|
||||
stripper.feed(html)
|
||||
return stripper.get_text()
|
||||
return _h2t.handle(html).strip()
|
||||
except Exception:
|
||||
# Fallback: crude tag removal
|
||||
return re.sub(r"<[^>]+>", " ", html).strip()
|
||||
return html
|
||||
|
||||
|
||||
async def _fetch_full_article(url: str) -> str | None:
|
||||
"""Fetch a URL and extract its main article text via trafilatura.
|
||||
|
||||
Returns clean plain text, or None if extraction fails or yields nothing useful.
|
||||
Runs trafilatura in a thread executor since it does synchronous HTML parsing.
|
||||
"""
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=15.0, follow_redirects=True, headers={
|
||||
"User-Agent": "Mozilla/5.0 (compatible; FabledAssistant/1.0; +https://fabledsword.com)",
|
||||
}) as client:
|
||||
resp = await client.get(url)
|
||||
resp.raise_for_status()
|
||||
raw_html = resp.text
|
||||
except Exception:
|
||||
logger.debug("Failed to fetch article URL %s", url)
|
||||
return None
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
try:
|
||||
import trafilatura
|
||||
text = await loop.run_in_executor(
|
||||
None,
|
||||
lambda: trafilatura.extract(
|
||||
raw_html,
|
||||
include_comments=False,
|
||||
include_tables=True,
|
||||
favor_recall=True,
|
||||
),
|
||||
)
|
||||
return text or None
|
||||
except Exception:
|
||||
logger.debug("trafilatura extraction failed for %s", url, exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
def extract_item(entry) -> dict:
|
||||
@@ -79,7 +80,7 @@ def extract_item(entry) -> dict:
|
||||
content = raw_content[0].value
|
||||
else:
|
||||
content = entry.get("summary", "")
|
||||
content = _strip_html(content)[:CONTENT_MAX_CHARS]
|
||||
content = _html_to_text(content)[:CONTENT_MAX_CHARS]
|
||||
|
||||
pub = None
|
||||
if entry.published_parsed:
|
||||
@@ -186,16 +187,55 @@ async def fetch_and_cache_feed(feed_id: int, url: str) -> int:
|
||||
from fabledassistant.services.rss_classifier import classify_and_store
|
||||
asyncio.create_task(classify_and_store(unclassified_ids, feed_user_id))
|
||||
|
||||
# Fire-and-forget embedding for new items
|
||||
if new_item_data and feed_user_id is not None:
|
||||
# Collect (id, url) for newly inserted items to enrich with full article text
|
||||
new_items_for_enrichment: list[tuple[int, str]] = []
|
||||
if new_count > 0:
|
||||
async with async_session() as session:
|
||||
result = await session.execute(
|
||||
select(RssItem.id, RssItem.url).where(
|
||||
RssItem.feed_id == feed_id,
|
||||
RssItem.url != "",
|
||||
).order_by(RssItem.fetched_at.desc()).limit(new_count)
|
||||
)
|
||||
new_items_for_enrichment = list(result.fetchall())
|
||||
|
||||
# Fire-and-forget: fetch full article text, then re-embed with richer content
|
||||
if new_items_for_enrichment and feed_user_id is not None:
|
||||
from fabledassistant.services.embeddings import upsert_rss_item_embedding
|
||||
|
||||
async def _embed_new_items() -> None:
|
||||
async def _enrich_and_embed() -> None:
|
||||
for item_id, article_url in new_items_for_enrichment:
|
||||
full_text = await _fetch_full_article(article_url)
|
||||
if full_text and len(full_text) > 200:
|
||||
async with async_session() as session:
|
||||
item = await session.get(RssItem, item_id)
|
||||
if item:
|
||||
item.content = full_text[:CONTENT_MAX_CHARS]
|
||||
await session.commit()
|
||||
await upsert_rss_item_embedding(
|
||||
item_id, feed_user_id, item.title or "", item.content
|
||||
)
|
||||
else:
|
||||
# Enrich failed — still embed with RSS-provided content
|
||||
async with async_session() as session:
|
||||
item = await session.get(RssItem, item_id)
|
||||
if item:
|
||||
await upsert_rss_item_embedding(
|
||||
item_id, feed_user_id, item.title or "", item.content or ""
|
||||
)
|
||||
await asyncio.sleep(0.5) # Polite pacing between article fetches
|
||||
|
||||
asyncio.create_task(_enrich_and_embed())
|
||||
elif new_item_data and feed_user_id is not None:
|
||||
# No URLs to enrich — embed with RSS content only
|
||||
from fabledassistant.services.embeddings import upsert_rss_item_embedding
|
||||
|
||||
async def _embed_only() -> None:
|
||||
for item_id, title, content in new_item_data:
|
||||
await upsert_rss_item_embedding(item_id, feed_user_id, title, content)
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
asyncio.create_task(_embed_new_items())
|
||||
asyncio.create_task(_embed_only())
|
||||
|
||||
return new_count
|
||||
|
||||
|
||||
Reference in New Issue
Block a user