diff --git a/alembic/versions/0024_backfill_post_title_from_description.py b/alembic/versions/0024_backfill_post_title_from_description.py new file mode 100644 index 0000000..2b1385c --- /dev/null +++ b/alembic/versions/0024_backfill_post_title_from_description.py @@ -0,0 +1,80 @@ +"""backfill post.post_title from description first-line — 2026-05-27 + +Revision ID: 0024 +Revises: 0023 +Create Date: 2026-05-27 + +SubscribeStar gallery-dl always writes `title: ""` and embeds the leading +sentence inside `content` HTML. FC's sidecar parser was leaving +post_title NULL for every SubscribeStar post since FC-3 shipped. The +parser fix (sidecar._first_line_text fallback) now synthesizes a title +at parse time; this migration applies the same logic retroactively to +existing rows. + +Operator-flagged 2026-05-27 after inspecting +/mnt/Data/Patreon/Cheunart/subscribestar/ sidecars. + +Idempotent: only touches rows where post_title IS NULL or empty AND +description IS NOT NULL. Re-running the migration is a no-op. +""" +from __future__ import annotations + +import re +from typing import Sequence, Union + +from alembic import op +from sqlalchemy import text + +revision: str = "0024" +down_revision: Union[str, None] = "0023" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +_TAG_RE = re.compile(r"<[^>]+>") +_WS_RE = re.compile(r"\s+") + + +def _first_line_text(body: str, limit: int = 120) -> str | None: + """Mirror of sidecar._first_line_text. Kept inline so the migration + doesn't carry a runtime import dependency from app code that may + have moved by the time the migration is replayed years from now.""" + if not body: + return None + text_ = _TAG_RE.sub(" ", body) + text_ = text_.replace("\xa0", " ") + for line in text_.splitlines(): + line = _WS_RE.sub(" ", line).strip() + if line: + if len(line) > limit: + return line[: limit - 1].rstrip() + "…" + return line + return None + + +def upgrade() -> None: + bind = op.get_bind() + rows = bind.execute( + text( + "SELECT id, description FROM post " + "WHERE (post_title IS NULL OR post_title = '') " + "AND description IS NOT NULL AND description <> ''" + ) + ).fetchall() + updated = 0 + for row in rows: + derived = _first_line_text(row.description) + if not derived: + continue + bind.execute( + text("UPDATE post SET post_title = :t WHERE id = :id"), + {"t": derived, "id": row.id}, + ) + updated += 1 + print(f"0024: backfilled post_title on {updated} row(s)") + + +def downgrade() -> None: + # No safe restore — we can't tell which post_titles were derived vs + # genuinely present. Leave the column alone on rollback. + pass diff --git a/backend/app/utils/sidecar.py b/backend/app/utils/sidecar.py index 8df78b4..19b1878 100644 --- a/backend/app/utils/sidecar.py +++ b/backend/app/utils/sidecar.py @@ -55,6 +55,33 @@ def _first_str(data: dict, keys: tuple[str, ...]) -> str | None: return None +# Strip HTML tags + collapse whitespace + take the first non-empty line. +# Used to derive a display title from a body when the platform doesn't +# expose a separate title field (subscribestar posts always write +# `title: ""` and put the leading sentence inside `content` as HTML). +# Truncated to 120 chars with an ellipsis if longer — long enough to be +# meaningful in a feed, short enough to fit a row. +_TAG_RE = re.compile(r"<[^>]+>") +_WS_RE = re.compile(r"\s+") + + +def _first_line_text(body: str, limit: int = 120) -> str | None: + if not body: + return None + text = _TAG_RE.sub(" ", body) + text = text.replace("\xa0", " ") + # Split on hard line breaks first; the body-stripped HTML often + # collapses to one logical line, in which case the first sentence + # split is the next-best heuristic. + for line in text.splitlines(): + line = _WS_RE.sub(" ", line).strip() + if line: + if len(line) > limit: + return line[: limit - 1].rstrip() + "…" + return line + return None + + def _parse_date(v) -> datetime | None: if isinstance(v, bool): return None @@ -111,12 +138,24 @@ def parse_sidecar(data: dict) -> SidecarData: if post_date is not None: break + description = _first_str(data, ("content", "description", "caption")) + + # SubscribeStar posts always write `title: ""` and put the leading + # sentence inside `content` (confirmed against the operator's + # /mnt/Data/Patreon/Cheunart/subscribestar/ dump 2026-05-27). When + # no explicit title is present, synthesize one from the description + # body's first non-empty line. Patreon retains its explicit titles + # because they're non-empty and short-circuit the fallback. + post_title = _first_str(data, ("title",)) + if post_title is None and description: + post_title = _first_line_text(description) + return SidecarData( platform=platform, external_post_id=external_post_id, post_url=_first_str(data, ("url", "post_url")), - post_title=_first_str(data, ("title",)), - description=_first_str(data, ("content", "description", "caption")), + post_title=post_title, + description=description, attachment_count=attachment_count, post_date=post_date, raw=data, diff --git a/tests/test_sidecar_util.py b/tests/test_sidecar_util.py index 3abff31..2edd259 100644 --- a/tests/test_sidecar_util.py +++ b/tests/test_sidecar_util.py @@ -110,6 +110,39 @@ def test_parse_date_epoch_and_unparseable_and_naive(): assert naive is not None and naive.utcoffset().total_seconds() == 0 +def test_parse_title_derived_from_content_when_empty(): + """SubscribeStar gallery-dl writes `title: ""` and puts the leading + sentence in `content` HTML. When `title` is empty, synthesize the + post title from the content body's first non-empty text line.""" + sd = parse_sidecar({ + "title": "", + "content": "\n
body line
"}) + assert sd.post_title == "Real Title" + + +def test_parse_title_no_fallback_when_no_content(): + sd = parse_sidecar({"title": ""}) + assert sd.post_title is None + + def test_parse_ignores_non_str_junk(): sd = parse_sidecar({"category": 5, "title": 7, "page_count": "x", "id": True})