fix: strip HTML from RSS item content during ingestion
feedparser returns HTML in content/summary fields for many feeds. Raw tags were being stored in the DB and passed to the LLM/embeddings. Added a stdlib HTMLParser-based stripper in extract_item() — block elements become newlines, script/style content is dropped, plain text passes through. No new dependencies required. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -2,7 +2,9 @@
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from html.parser import HTMLParser
|
||||
|
||||
import feedparser
|
||||
from sqlalchemy import select, text
|
||||
@@ -17,6 +19,57 @@ CONTENT_MAX_CHARS = 2000
|
||||
ITEM_MAX_AGE_DAYS = 90
|
||||
|
||||
|
||||
class _HTMLStripper(HTMLParser):
|
||||
"""Minimal HTML-to-text converter using stdlib only."""
|
||||
|
||||
# Tags that should produce a line break in the output
|
||||
BLOCK_TAGS = frozenset({
|
||||
"p", "br", "li", "h1", "h2", "h3", "h4", "h5", "h6",
|
||||
"div", "section", "article", "blockquote", "tr",
|
||||
})
|
||||
# Tags whose inner text should be skipped entirely
|
||||
SKIP_TAGS = frozenset({"script", "style", "noscript"})
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self._parts: list[str] = []
|
||||
self._skip_depth = 0
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list) -> None:
|
||||
if tag in self.SKIP_TAGS:
|
||||
self._skip_depth += 1
|
||||
elif tag in self.BLOCK_TAGS and self._skip_depth == 0:
|
||||
self._parts.append("\n")
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
if tag in self.SKIP_TAGS and self._skip_depth > 0:
|
||||
self._skip_depth -= 1
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self._skip_depth == 0:
|
||||
self._parts.append(data)
|
||||
|
||||
def get_text(self) -> str:
|
||||
raw = "".join(self._parts)
|
||||
# Collapse runs of whitespace/blank lines to single spaces / newlines
|
||||
raw = re.sub(r"[ \t]+", " ", raw)
|
||||
raw = re.sub(r"\n{3,}", "\n\n", raw)
|
||||
return raw.strip()
|
||||
|
||||
|
||||
def _strip_html(html: str) -> str:
|
||||
"""Strip HTML tags and return clean plain text."""
|
||||
if not html or not ("<" in html and ">" in html):
|
||||
return html # Already plain text — skip parsing
|
||||
stripper = _HTMLStripper()
|
||||
try:
|
||||
stripper.feed(html)
|
||||
return stripper.get_text()
|
||||
except Exception:
|
||||
# Fallback: crude tag removal
|
||||
return re.sub(r"<[^>]+>", " ", html).strip()
|
||||
|
||||
|
||||
def extract_item(entry) -> dict:
|
||||
"""Extract a clean item dict from a feedparser entry object."""
|
||||
# Prefer full content over summary (feedparser uses a list of Content objects)
|
||||
@@ -26,7 +79,7 @@ def extract_item(entry) -> dict:
|
||||
content = raw_content[0].value
|
||||
else:
|
||||
content = entry.get("summary", "")
|
||||
content = content[:CONTENT_MAX_CHARS]
|
||||
content = _strip_html(content)[:CONTENT_MAX_CHARS]
|
||||
|
||||
pub = None
|
||||
if entry.published_parsed:
|
||||
|
||||
Reference in New Issue
Block a user