From 0b05b039875a88b96728160ad19c3303697a28ba Mon Sep 17 00:00:00 2001
From: Bryan Van Deusen <bryan.vandeusen@fabledsword.com>
Date: Mon, 30 Mar 2026 16:15:49 -0400
Subject: [PATCH] fix: strip HTML from RSS item content during ingestion
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

feedparser returns HTML in content/summary fields for many feeds.
Raw tags were being stored in the DB and passed to the LLM/embeddings.
Added a stdlib HTMLParser-based stripper in extract_item() — block elements
become newlines, script/style content is dropped, plain text passes through.
No new dependencies required.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 src/fabledassistant/services/rss.py | 55 ++++++++++++++++++++++++++++-
 1 file changed, 54 insertions(+), 1 deletion(-)

diff --git a/src/fabledassistant/services/rss.py b/src/fabledassistant/services/rss.py
index 5bb79ef..8674d6f 100644
--- a/src/fabledassistant/services/rss.py
+++ b/src/fabledassistant/services/rss.py
@@ -2,7 +2,9 @@
 
 import asyncio
 import logging
+import re
 from datetime import datetime, timezone
+from html.parser import HTMLParser
 
 import feedparser
 from sqlalchemy import select, text
@@ -17,6 +19,57 @@ CONTENT_MAX_CHARS = 2000
 ITEM_MAX_AGE_DAYS = 90
 
 
+class _HTMLStripper(HTMLParser):
+    """Minimal HTML-to-text converter using stdlib only."""
+
+    # Tags that should produce a line break in the output
+    BLOCK_TAGS = frozenset({
+        "p", "br", "li", "h1", "h2", "h3", "h4", "h5", "h6",
+        "div", "section", "article", "blockquote", "tr",
+    })
+    # Tags whose inner text should be skipped entirely
+    SKIP_TAGS = frozenset({"script", "style", "noscript"})
+
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self._parts: list[str] = []
+        self._skip_depth = 0
+
+    def handle_starttag(self, tag: str, attrs: list) -> None:
+        if tag in self.SKIP_TAGS:
+            self._skip_depth += 1
+        elif tag in self.BLOCK_TAGS and self._skip_depth == 0:
+            self._parts.append("\n")
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag in self.SKIP_TAGS and self._skip_depth > 0:
+            self._skip_depth -= 1
+
+    def handle_data(self, data: str) -> None:
+        if self._skip_depth == 0:
+            self._parts.append(data)
+
+    def get_text(self) -> str:
+        raw = "".join(self._parts)
+        # Collapse runs of whitespace/blank lines to single spaces / newlines
+        raw = re.sub(r"[ \t]+", " ", raw)
+        raw = re.sub(r"\n{3,}", "\n\n", raw)
+        return raw.strip()
+
+
+def _strip_html(html: str) -> str:
+    """Strip HTML tags and return clean plain text."""
+    if not html or not ("<" in html and ">" in html):
+        return html  # Already plain text — skip parsing
+    stripper = _HTMLStripper()
+    try:
+        stripper.feed(html)
+        return stripper.get_text()
+    except Exception:
+        # Fallback: crude tag removal
+        return re.sub(r"<[^>]+>", " ", html).strip()
+
+
 def extract_item(entry) -> dict:
     """Extract a clean item dict from a feedparser entry object."""
     # Prefer full content over summary (feedparser uses a list of Content objects)
@@ -26,7 +79,7 @@ def extract_item(entry) -> dict:
         content = raw_content[0].value
     else:
         content = entry.get("summary", "")
-    content = content[:CONTENT_MAX_CHARS]
+    content = _strip_html(content)[:CONTENT_MAX_CHARS]
 
     pub = None
     if entry.published_parsed: