fix: strip HTML from RSS item content during ingestion

feedparser returns HTML in content/summary fields for many feeds.
Raw tags were being stored in the DB and passed to the LLM/embeddings.
Added a stdlib HTMLParser-based stripper in extract_item() — block elements
become newlines, script/style content is dropped, plain text passes through.
No new dependencies required.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-30 16:15:49 -04:00
parent a773c11aa0
commit 0b05b03987
+54 -1
View File
@@ -2,7 +2,9 @@
import asyncio import asyncio
import logging import logging
import re
from datetime import datetime, timezone from datetime import datetime, timezone
from html.parser import HTMLParser
import feedparser import feedparser
from sqlalchemy import select, text from sqlalchemy import select, text
@@ -17,6 +19,57 @@ CONTENT_MAX_CHARS = 2000
ITEM_MAX_AGE_DAYS = 90 ITEM_MAX_AGE_DAYS = 90
class _HTMLStripper(HTMLParser):
"""Minimal HTML-to-text converter using stdlib only."""
# Tags that should produce a line break in the output
BLOCK_TAGS = frozenset({
"p", "br", "li", "h1", "h2", "h3", "h4", "h5", "h6",
"div", "section", "article", "blockquote", "tr",
})
# Tags whose inner text should be skipped entirely
SKIP_TAGS = frozenset({"script", "style", "noscript"})
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self._parts: list[str] = []
self._skip_depth = 0
def handle_starttag(self, tag: str, attrs: list) -> None:
if tag in self.SKIP_TAGS:
self._skip_depth += 1
elif tag in self.BLOCK_TAGS and self._skip_depth == 0:
self._parts.append("\n")
def handle_endtag(self, tag: str) -> None:
if tag in self.SKIP_TAGS and self._skip_depth > 0:
self._skip_depth -= 1
def handle_data(self, data: str) -> None:
if self._skip_depth == 0:
self._parts.append(data)
def get_text(self) -> str:
raw = "".join(self._parts)
# Collapse runs of whitespace/blank lines to single spaces / newlines
raw = re.sub(r"[ \t]+", " ", raw)
raw = re.sub(r"\n{3,}", "\n\n", raw)
return raw.strip()
def _strip_html(html: str) -> str:
"""Strip HTML tags and return clean plain text."""
if not html or not ("<" in html and ">" in html):
return html # Already plain text — skip parsing
stripper = _HTMLStripper()
try:
stripper.feed(html)
return stripper.get_text()
except Exception:
# Fallback: crude tag removal
return re.sub(r"<[^>]+>", " ", html).strip()
def extract_item(entry) -> dict: def extract_item(entry) -> dict:
"""Extract a clean item dict from a feedparser entry object.""" """Extract a clean item dict from a feedparser entry object."""
# Prefer full content over summary (feedparser uses a list of Content objects) # Prefer full content over summary (feedparser uses a list of Content objects)
@@ -26,7 +79,7 @@ def extract_item(entry) -> dict:
content = raw_content[0].value content = raw_content[0].value
else: else:
content = entry.get("summary", "") content = entry.get("summary", "")
content = content[:CONTENT_MAX_CHARS] content = _strip_html(content)[:CONTENT_MAX_CHARS]
pub = None pub = None
if entry.published_parsed: if entry.published_parsed: