From 07edb733733dc03f5a11243fb3ec25ddd3715b8e Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 21 May 2026 18:53:35 -0400 Subject: [PATCH] fc3e: html_to_plain + truncate_at_word utilities (stdlib html.parser, no new deps) Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/utils/text.py | 81 +++++++++++++++++++++++++++++++++++++++ tests/test_text_utils.py | 56 +++++++++++++++++++++++++++ 2 files changed, 137 insertions(+) create mode 100644 backend/app/utils/text.py create mode 100644 tests/test_text_utils.py diff --git a/backend/app/utils/text.py b/backend/app/utils/text.py new file mode 100644 index 0000000..3aea5b5 --- /dev/null +++ b/backend/app/utils/text.py @@ -0,0 +1,81 @@ +"""HTML→plain converter and word-boundary truncator. + +Used by PostFeedService to sanitize Post.description for the unified +posts feed. Stdlib-only (no bleach dependency) — strips ALL tags and +collapses whitespace. +""" +from __future__ import annotations + +import re +from html.parser import HTMLParser + +#
,

, , , → soft block separators we emit as \n +# so paragraph breaks survive the strip. +_BLOCK_END_TAGS = frozenset({ + "p", "div", "li", "h1", "h2", "h3", "h4", "h5", "h6", + "blockquote", "tr", "ul", "ol", +}) +_VOID_NEWLINE_TAGS = frozenset({"br"}) + +_MULTI_WS_RE = re.compile(r"[^\S\n]+") # runs of horizontal whitespace +_MULTI_NL_RE = re.compile(r"\n{3,}") # cap consecutive newlines at 2 + + +class _PlainTextExtractor(HTMLParser): + def __init__(self) -> None: + super().__init__() + self._chunks: list[str] = [] + + def handle_data(self, data: str) -> None: + self._chunks.append(data) + + def handle_starttag(self, tag: str, attrs) -> None: + if tag in _VOID_NEWLINE_TAGS: + self._chunks.append("\n") + + def handle_endtag(self, tag: str) -> None: + if tag in _BLOCK_END_TAGS: + self._chunks.append("\n") + + def get_text(self) -> str: + return "".join(self._chunks) + + +def html_to_plain(s: str | None) -> str: + """Strip HTML, normalize whitespace. Returns "" for None. + + - All tags removed; entities decoded (stdlib parser does this). + -
and block-closing tags (

, , , , ...) + become "\\n" so paragraph breaks survive. + - Horizontal whitespace runs collapse to a single space. + - Three+ newlines compress to two; leading/trailing whitespace stripped. + - Idempotent on plain-text input. + """ + if s is None: + return "" + parser = _PlainTextExtractor() + parser.feed(s) + parser.close() + text = parser.get_text() + # Collapse horizontal whitespace per line, then cap consecutive newlines. + text = _MULTI_WS_RE.sub(" ", text) + text = _MULTI_NL_RE.sub("\n\n", text) + # Strip leading/trailing whitespace on the whole string AND each line. + text = "\n".join(line.strip() for line in text.split("\n")).strip() + return text + + +def truncate_at_word(s: str | None, limit: int) -> tuple[str, bool]: + """Return (truncated, was_truncated). Cuts at last whitespace before + ``limit``; falls back to a hard cut if no whitespace is found. + Appends an ellipsis when truncation occurs. Empty/None → ("", False). + """ + if not s: + return ("", False) + if len(s) <= limit: + return (s, False) + head = s[:limit] + last_ws = head.rfind(" ") + if last_ws > 0: + head = head[:last_ws] + return (head + "…", True) diff --git a/tests/test_text_utils.py b/tests/test_text_utils.py new file mode 100644 index 0000000..18bf2bc --- /dev/null +++ b/tests/test_text_utils.py @@ -0,0 +1,56 @@ +"""FC-3e: HTML-strip + word-boundary truncator unit tests.""" + +from backend.app.utils.text import html_to_plain, truncate_at_word + + +def test_html_to_plain_strips_tags(): + assert html_to_plain("

hi

") == "hi" + + +def test_html_to_plain_converts_br_to_newline(): + assert html_to_plain("a
b") == "a\nb" + + +def test_html_to_plain_converts_p_close_to_newline(): + assert html_to_plain("

one

two

") == "one\ntwo" + + +def test_html_to_plain_collapses_runs_of_whitespace(): + assert html_to_plain("a b") == "a b" + + +def test_html_to_plain_preserves_intentional_newlines_between_blocks(): + assert html_to_plain("alpha

beta") == "alpha\n\nbeta" + + +def test_html_to_plain_idempotent_on_plain_text(): + assert html_to_plain("plain text") == "plain text" + + +def test_html_to_plain_none_returns_empty(): + assert html_to_plain(None) == "" + + +def test_html_to_plain_decodes_entities(): + assert html_to_plain("a & b") == "a & b" + + +def test_truncate_at_word_under_limit_no_change(): + assert truncate_at_word("short", 100) == ("short", False) + + +def test_truncate_at_word_breaks_at_whitespace(): + assert truncate_at_word("alpha beta gamma", 8) == ("alpha…", True) + + +def test_truncate_at_word_no_whitespace_hard_cuts(): + # No space to break on within the limit — hard-cut at the limit. + assert truncate_at_word("abcdefghij", 5) == ("abcde…", True) + + +def test_truncate_at_word_handles_none(): + assert truncate_at_word(None, 10) == ("", False) + + +def test_truncate_at_word_handles_empty_string(): + assert truncate_at_word("", 10) == ("", False)