fix: strip HTML from RSS item content during ingestion

feedparser returns HTML in content/summary fields for many feeds.
Raw tags were being stored in the DB and passed to the LLM/embeddings.
Added a stdlib HTMLParser-based stripper in extract_item() — block elements
become newlines, script/style content is dropped, plain text passes through.
No new dependencies required.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-30 16:15:49 -04:00
parent a773c11aa0
commit 0b05b03987
+54 -1
View File
@@ -2,7 +2,9 @@
import asyncio
import logging
import re
from datetime import datetime, timezone
from html.parser import HTMLParser
import feedparser
from sqlalchemy import select, text
@@ -17,6 +19,57 @@ CONTENT_MAX_CHARS = 2000
ITEM_MAX_AGE_DAYS = 90
class _HTMLStripper(HTMLParser):
"""Minimal HTML-to-text converter using stdlib only."""
# Tags that should produce a line break in the output
BLOCK_TAGS = frozenset({
"p", "br", "li", "h1", "h2", "h3", "h4", "h5", "h6",
"div", "section", "article", "blockquote", "tr",
})
# Tags whose inner text should be skipped entirely
SKIP_TAGS = frozenset({"script", "style", "noscript"})
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self._parts: list[str] = []
self._skip_depth = 0
def handle_starttag(self, tag: str, attrs: list) -> None:
if tag in self.SKIP_TAGS:
self._skip_depth += 1
elif tag in self.BLOCK_TAGS and self._skip_depth == 0:
self._parts.append("\n")
def handle_endtag(self, tag: str) -> None:
if tag in self.SKIP_TAGS and self._skip_depth > 0:
self._skip_depth -= 1
def handle_data(self, data: str) -> None:
if self._skip_depth == 0:
self._parts.append(data)
def get_text(self) -> str:
raw = "".join(self._parts)
# Collapse runs of whitespace/blank lines to single spaces / newlines
raw = re.sub(r"[ \t]+", " ", raw)
raw = re.sub(r"\n{3,}", "\n\n", raw)
return raw.strip()
def _strip_html(html: str) -> str:
"""Strip HTML tags and return clean plain text."""
if not html or not ("<" in html and ">" in html):
return html # Already plain text — skip parsing
stripper = _HTMLStripper()
try:
stripper.feed(html)
return stripper.get_text()
except Exception:
# Fallback: crude tag removal
return re.sub(r"<[^>]+>", " ", html).strip()
def extract_item(entry) -> dict:
"""Extract a clean item dict from a feedparser entry object."""
# Prefer full content over summary (feedparser uses a list of Content objects)
@@ -26,7 +79,7 @@ def extract_item(entry) -> dict:
content = raw_content[0].value
else:
content = entry.get("summary", "")
content = content[:CONTENT_MAX_CHARS]
content = _strip_html(content)[:CONTENT_MAX_CHARS]
pub = None
if entry.published_parsed: