82 lines
2.7 KiB
Python
82 lines
2.7 KiB
Python
"""HTML→plain converter and word-boundary truncator.
|
|
|
|
Used by PostFeedService to sanitize Post.description for the unified
|
|
posts feed. Stdlib-only (no bleach dependency) — strips ALL tags and
|
|
collapses whitespace.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from html.parser import HTMLParser
|
|
|
|
# <br>, </p>, </div>, </li>, </h1..6> → soft block separators we emit as \n
|
|
# so paragraph breaks survive the strip.
|
|
_BLOCK_END_TAGS = frozenset({
|
|
"p", "div", "li", "h1", "h2", "h3", "h4", "h5", "h6",
|
|
"blockquote", "tr", "ul", "ol",
|
|
})
|
|
_VOID_NEWLINE_TAGS = frozenset({"br"})
|
|
|
|
_MULTI_WS_RE = re.compile(r"[^\S\n]+") # runs of horizontal whitespace
|
|
_MULTI_NL_RE = re.compile(r"\n{3,}") # cap consecutive newlines at 2
|
|
|
|
|
|
class _PlainTextExtractor(HTMLParser):
|
|
def __init__(self) -> None:
|
|
super().__init__()
|
|
self._chunks: list[str] = []
|
|
|
|
def handle_data(self, data: str) -> None:
|
|
self._chunks.append(data)
|
|
|
|
def handle_starttag(self, tag: str, attrs) -> None:
|
|
if tag in _VOID_NEWLINE_TAGS:
|
|
self._chunks.append("\n")
|
|
|
|
def handle_endtag(self, tag: str) -> None:
|
|
if tag in _BLOCK_END_TAGS:
|
|
self._chunks.append("\n")
|
|
|
|
def get_text(self) -> str:
|
|
return "".join(self._chunks)
|
|
|
|
|
|
def html_to_plain(s: str | None) -> str:
|
|
"""Strip HTML, normalize whitespace. Returns "" for None.
|
|
|
|
- All tags removed; entities decoded (stdlib parser does this).
|
|
- <br> and block-closing tags (</p>, </div>, </li>, </h1..6>, ...)
|
|
become "\\n" so paragraph breaks survive.
|
|
- Horizontal whitespace runs collapse to a single space.
|
|
- Three+ newlines compress to two; leading/trailing whitespace stripped.
|
|
- Idempotent on plain-text input.
|
|
"""
|
|
if s is None:
|
|
return ""
|
|
parser = _PlainTextExtractor()
|
|
parser.feed(s)
|
|
parser.close()
|
|
text = parser.get_text()
|
|
# Collapse horizontal whitespace per line, then cap consecutive newlines.
|
|
text = _MULTI_WS_RE.sub(" ", text)
|
|
text = _MULTI_NL_RE.sub("\n\n", text)
|
|
# Strip leading/trailing whitespace on the whole string AND each line.
|
|
text = "\n".join(line.strip() for line in text.split("\n")).strip()
|
|
return text
|
|
|
|
|
|
def truncate_at_word(s: str | None, limit: int) -> tuple[str, bool]:
|
|
"""Return (truncated, was_truncated). Cuts at last whitespace before
|
|
``limit``; falls back to a hard cut if no whitespace is found.
|
|
Appends an ellipsis when truncation occurs. Empty/None → ("", False).
|
|
"""
|
|
if not s:
|
|
return ("", False)
|
|
if len(s) <= limit:
|
|
return (s, False)
|
|
head = s[:limit]
|
|
last_ws = head.rfind(" ")
|
|
if last_ws > 0:
|
|
head = head[:last_ws]
|
|
return (head + "…", True)
|