fix(sidecar): synthesize post_title from content first-line when title is empty (subscribestar)
SubscribeStar gallery-dl always writes `title: ""` and embeds the leading sentence inside `content` HTML. Confirmed against the operator's /mnt/Data/Patreon/Cheunart/subscribestar/ dump 2026-05-27: every post's JSON has `title: ""` and a content like `<div>Lets say hello to you guys with my Belle <br><br><br></div>`. FC's sidecar parser, treating empty strings as missing, had been leaving post_title NULL on every subscribestar post since FC-3 shipped. Fix at two layers: 1. `backend/app/utils/sidecar.py` — new `_first_line_text(body, limit)` helper strips HTML tags, collapses whitespace, returns the first non-empty line truncated to 120 chars with ellipsis. `parse_sidecar` now falls back to this when `title` resolves to None and a `content`/`description`/`caption` value is present. Patreon's non-empty titles short-circuit the fallback so existing behavior is unchanged. Four new tests in test_sidecar_util.py pin: derivation from content, truncation at 120 chars, explicit-title precedence, no-content no-fallback. 2. `alembic 0024_backfill_post_title_from_description` — backfills the same logic across existing Post rows where `post_title IS NULL OR post_title = ''` AND description is present. Idempotent (re-running is a no-op once titles are populated). Downgrade is a no-op since there's no safe way to tell derived rows from genuine ones. After deploy + migration: subscribestar posts will surface a meaningful title in PostCard, post feed search, etc.
This commit is contained in:
@@ -55,6 +55,33 @@ def _first_str(data: dict, keys: tuple[str, ...]) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
# Strip HTML tags + collapse whitespace + take the first non-empty line.
|
||||
# Used to derive a display title from a body when the platform doesn't
|
||||
# expose a separate title field (subscribestar posts always write
|
||||
# `title: ""` and put the leading sentence inside `content` as HTML).
|
||||
# Truncated to 120 chars with an ellipsis if longer — long enough to be
|
||||
# meaningful in a feed, short enough to fit a row.
|
||||
_TAG_RE = re.compile(r"<[^>]+>")
|
||||
_WS_RE = re.compile(r"\s+")
|
||||
|
||||
|
||||
def _first_line_text(body: str, limit: int = 120) -> str | None:
|
||||
if not body:
|
||||
return None
|
||||
text = _TAG_RE.sub(" ", body)
|
||||
text = text.replace("\xa0", " ")
|
||||
# Split on hard line breaks first; the body-stripped HTML often
|
||||
# collapses to one logical line, in which case the first sentence
|
||||
# split is the next-best heuristic.
|
||||
for line in text.splitlines():
|
||||
line = _WS_RE.sub(" ", line).strip()
|
||||
if line:
|
||||
if len(line) > limit:
|
||||
return line[: limit - 1].rstrip() + "…"
|
||||
return line
|
||||
return None
|
||||
|
||||
|
||||
def _parse_date(v) -> datetime | None:
|
||||
if isinstance(v, bool):
|
||||
return None
|
||||
@@ -111,12 +138,24 @@ def parse_sidecar(data: dict) -> SidecarData:
|
||||
if post_date is not None:
|
||||
break
|
||||
|
||||
description = _first_str(data, ("content", "description", "caption"))
|
||||
|
||||
# SubscribeStar posts always write `title: ""` and put the leading
|
||||
# sentence inside `content` (confirmed against the operator's
|
||||
# /mnt/Data/Patreon/Cheunart/subscribestar/ dump 2026-05-27). When
|
||||
# no explicit title is present, synthesize one from the description
|
||||
# body's first non-empty line. Patreon retains its explicit titles
|
||||
# because they're non-empty and short-circuit the fallback.
|
||||
post_title = _first_str(data, ("title",))
|
||||
if post_title is None and description:
|
||||
post_title = _first_line_text(description)
|
||||
|
||||
return SidecarData(
|
||||
platform=platform,
|
||||
external_post_id=external_post_id,
|
||||
post_url=_first_str(data, ("url", "post_url")),
|
||||
post_title=_first_str(data, ("title",)),
|
||||
description=_first_str(data, ("content", "description", "caption")),
|
||||
post_title=post_title,
|
||||
description=description,
|
||||
attachment_count=attachment_count,
|
||||
post_date=post_date,
|
||||
raw=data,
|
||||
|
||||
Reference in New Issue
Block a user