fc3e: html_to_plain + truncate_at_word utilities (stdlib html.parser, no new deps)

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

backend/app/utils/text.py

@@ -0,0 +1,81 @@
+"""HTML→plain converter and word-boundary truncator.
+
+Used by PostFeedService to sanitize Post.description for the unified
+posts feed. Stdlib-only (no bleach dependency) — strips ALL tags and
+collapses whitespace.
+"""
+from __future__ import annotations
+
+import re
+from html.parser import HTMLParser
+
+# <br>, </p>, </div>, </li>, </h1..6> → soft block separators we emit as \n
+# so paragraph breaks survive the strip.
+_BLOCK_END_TAGS = frozenset({
+    "p", "div", "li", "h1", "h2", "h3", "h4", "h5", "h6",
+    "blockquote", "tr", "ul", "ol",
+})
+_VOID_NEWLINE_TAGS = frozenset({"br"})
+
+_MULTI_WS_RE = re.compile(r"[^\S\n]+")  # runs of horizontal whitespace
+_MULTI_NL_RE = re.compile(r"\n{3,}")    # cap consecutive newlines at 2
+
+
+class _PlainTextExtractor(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__()
+        self._chunks: list[str] = []
+
+    def handle_data(self, data: str) -> None:
+        self._chunks.append(data)
+
+    def handle_starttag(self, tag: str, attrs) -> None:
+        if tag in _VOID_NEWLINE_TAGS:
+            self._chunks.append("\n")
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag in _BLOCK_END_TAGS:
+            self._chunks.append("\n")
+
+    def get_text(self) -> str:
+        return "".join(self._chunks)
+
+
+def html_to_plain(s: str | None) -> str:
+    """Strip HTML, normalize whitespace. Returns "" for None.
+
+    - All tags removed; entities decoded (stdlib parser does this).
+    - <br> and block-closing tags (</p>, </div>, </li>, </h1..6>, ...)
+      become "\\n" so paragraph breaks survive.
+    - Horizontal whitespace runs collapse to a single space.
+    - Three+ newlines compress to two; leading/trailing whitespace stripped.
+    - Idempotent on plain-text input.
+    """
+    if s is None:
+        return ""
+    parser = _PlainTextExtractor()
+    parser.feed(s)
+    parser.close()
+    text = parser.get_text()
+    # Collapse horizontal whitespace per line, then cap consecutive newlines.
+    text = _MULTI_WS_RE.sub(" ", text)
+    text = _MULTI_NL_RE.sub("\n\n", text)
+    # Strip leading/trailing whitespace on the whole string AND each line.
+    text = "\n".join(line.strip() for line in text.split("\n")).strip()
+    return text
+
+
+def truncate_at_word(s: str | None, limit: int) -> tuple[str, bool]:
+    """Return (truncated, was_truncated). Cuts at last whitespace before
+    ``limit``; falls back to a hard cut if no whitespace is found.
+    Appends an ellipsis when truncation occurs. Empty/None → ("", False).
+    """
+    if not s:
+        return ("", False)
+    if len(s) <= limit:
+        return (s, False)
+    head = s[:limit]
+    last_ws = head.rfind(" ")
+    if last_ws > 0:
+        head = head[:last_ws]
+    return (head + "…", True)

tests/test_text_utils.py

@@ -0,0 +1,56 @@
+"""FC-3e: HTML-strip + word-boundary truncator unit tests."""
+
+from backend.app.utils.text import html_to_plain, truncate_at_word
+
+
+def test_html_to_plain_strips_tags():
+    assert html_to_plain("<p>hi</p>") == "hi"
+
+
+def test_html_to_plain_converts_br_to_newline():
+    assert html_to_plain("a<br>b") == "a\nb"
+
+
+def test_html_to_plain_converts_p_close_to_newline():
+    assert html_to_plain("<p>one</p><p>two</p>") == "one\ntwo"
+
+
+def test_html_to_plain_collapses_runs_of_whitespace():
+    assert html_to_plain("a   b") == "a b"
+
+
+def test_html_to_plain_preserves_intentional_newlines_between_blocks():
+    assert html_to_plain("alpha<br><br>beta") == "alpha\n\nbeta"
+
+
+def test_html_to_plain_idempotent_on_plain_text():
+    assert html_to_plain("plain text") == "plain text"
+
+
+def test_html_to_plain_none_returns_empty():
+    assert html_to_plain(None) == ""
+
+
+def test_html_to_plain_decodes_entities():
+    assert html_to_plain("a &amp; b") == "a & b"
+
+
+def test_truncate_at_word_under_limit_no_change():
+    assert truncate_at_word("short", 100) == ("short", False)
+
+
+def test_truncate_at_word_breaks_at_whitespace():
+    assert truncate_at_word("alpha beta gamma", 8) == ("alpha…", True)
+
+
+def test_truncate_at_word_no_whitespace_hard_cuts():
+    # No space to break on within the limit — hard-cut at the limit.
+    assert truncate_at_word("abcdefghij", 5) == ("abcde…", True)
+
+
+def test_truncate_at_word_handles_none():
+    assert truncate_at_word(None, 10) == ("", False)
+
+
+def test_truncate_at_word_handles_empty_string():
+    assert truncate_at_word("", 10) == ("", False)