fix: strip HTML from RSS item content during ingestion
feedparser returns HTML in content/summary fields for many feeds. Raw tags were being stored in the DB and passed to the LLM/embeddings. Added a stdlib HTMLParser-based stripper in extract_item() — block elements become newlines, script/style content is dropped, plain text passes through. No new dependencies required. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -2,7 +2,9 @@
|
|||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
|
import re
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
from html.parser import HTMLParser
|
||||||
|
|
||||||
import feedparser
|
import feedparser
|
||||||
from sqlalchemy import select, text
|
from sqlalchemy import select, text
|
||||||
@@ -17,6 +19,57 @@ CONTENT_MAX_CHARS = 2000
|
|||||||
ITEM_MAX_AGE_DAYS = 90
|
ITEM_MAX_AGE_DAYS = 90
|
||||||
|
|
||||||
|
|
||||||
|
class _HTMLStripper(HTMLParser):
|
||||||
|
"""Minimal HTML-to-text converter using stdlib only."""
|
||||||
|
|
||||||
|
# Tags that should produce a line break in the output
|
||||||
|
BLOCK_TAGS = frozenset({
|
||||||
|
"p", "br", "li", "h1", "h2", "h3", "h4", "h5", "h6",
|
||||||
|
"div", "section", "article", "blockquote", "tr",
|
||||||
|
})
|
||||||
|
# Tags whose inner text should be skipped entirely
|
||||||
|
SKIP_TAGS = frozenset({"script", "style", "noscript"})
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
super().__init__(convert_charrefs=True)
|
||||||
|
self._parts: list[str] = []
|
||||||
|
self._skip_depth = 0
|
||||||
|
|
||||||
|
def handle_starttag(self, tag: str, attrs: list) -> None:
|
||||||
|
if tag in self.SKIP_TAGS:
|
||||||
|
self._skip_depth += 1
|
||||||
|
elif tag in self.BLOCK_TAGS and self._skip_depth == 0:
|
||||||
|
self._parts.append("\n")
|
||||||
|
|
||||||
|
def handle_endtag(self, tag: str) -> None:
|
||||||
|
if tag in self.SKIP_TAGS and self._skip_depth > 0:
|
||||||
|
self._skip_depth -= 1
|
||||||
|
|
||||||
|
def handle_data(self, data: str) -> None:
|
||||||
|
if self._skip_depth == 0:
|
||||||
|
self._parts.append(data)
|
||||||
|
|
||||||
|
def get_text(self) -> str:
|
||||||
|
raw = "".join(self._parts)
|
||||||
|
# Collapse runs of whitespace/blank lines to single spaces / newlines
|
||||||
|
raw = re.sub(r"[ \t]+", " ", raw)
|
||||||
|
raw = re.sub(r"\n{3,}", "\n\n", raw)
|
||||||
|
return raw.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def _strip_html(html: str) -> str:
|
||||||
|
"""Strip HTML tags and return clean plain text."""
|
||||||
|
if not html or not ("<" in html and ">" in html):
|
||||||
|
return html # Already plain text — skip parsing
|
||||||
|
stripper = _HTMLStripper()
|
||||||
|
try:
|
||||||
|
stripper.feed(html)
|
||||||
|
return stripper.get_text()
|
||||||
|
except Exception:
|
||||||
|
# Fallback: crude tag removal
|
||||||
|
return re.sub(r"<[^>]+>", " ", html).strip()
|
||||||
|
|
||||||
|
|
||||||
def extract_item(entry) -> dict:
|
def extract_item(entry) -> dict:
|
||||||
"""Extract a clean item dict from a feedparser entry object."""
|
"""Extract a clean item dict from a feedparser entry object."""
|
||||||
# Prefer full content over summary (feedparser uses a list of Content objects)
|
# Prefer full content over summary (feedparser uses a list of Content objects)
|
||||||
@@ -26,7 +79,7 @@ def extract_item(entry) -> dict:
|
|||||||
content = raw_content[0].value
|
content = raw_content[0].value
|
||||||
else:
|
else:
|
||||||
content = entry.get("summary", "")
|
content = entry.get("summary", "")
|
||||||
content = content[:CONTENT_MAX_CHARS]
|
content = _strip_html(content)[:CONTENT_MAX_CHARS]
|
||||||
|
|
||||||
pub = None
|
pub = None
|
||||||
if entry.published_parsed:
|
if entry.published_parsed:
|
||||||
|
|||||||
Reference in New Issue
Block a user