ae8c78ae09
SubscribeStar gallery-dl always writes `title: ""` and embeds the leading sentence inside `content` HTML. Confirmed against the operator's /mnt/Data/Patreon/Cheunart/subscribestar/ dump 2026-05-27: every post's JSON has `title: ""` and a content like `<div>Lets say hello to you guys with my Belle <br><br><br></div>`. FC's sidecar parser, treating empty strings as missing, had been leaving post_title NULL on every subscribestar post since FC-3 shipped. Fix at two layers: 1. `backend/app/utils/sidecar.py` — new `_first_line_text(body, limit)` helper strips HTML tags, collapses whitespace, returns the first non-empty line truncated to 120 chars with ellipsis. `parse_sidecar` now falls back to this when `title` resolves to None and a `content`/`description`/`caption` value is present. Patreon's non-empty titles short-circuit the fallback so existing behavior is unchanged. Four new tests in test_sidecar_util.py pin: derivation from content, truncation at 120 chars, explicit-title precedence, no-content no-fallback. 2. `alembic 0024_backfill_post_title_from_description` — backfills the same logic across existing Post rows where `post_title IS NULL OR post_title = ''` AND description is present. Idempotent (re-running is a no-op once titles are populated). Downgrade is a no-op since there's no safe way to tell derived rows from genuine ones. After deploy + migration: subscribestar posts will surface a meaningful title in PostCard, post feed search, etc.
154 lines
5.6 KiB
Python
154 lines
5.6 KiB
Python
from backend.app.utils.sidecar import SidecarData, find_sidecar, parse_sidecar
|
|
|
|
|
|
def test_find_sidecar_stem_json(tmp_path):
|
|
media = tmp_path / "photo.jpg"
|
|
media.write_bytes(b"x")
|
|
sc = tmp_path / "photo.json"
|
|
sc.write_text("{}")
|
|
assert find_sidecar(media) == sc
|
|
|
|
|
|
def test_find_sidecar_full_name_json(tmp_path):
|
|
media = tmp_path / "photo.jpg"
|
|
media.write_bytes(b"x")
|
|
sc = tmp_path / "photo.jpg.json"
|
|
sc.write_text("{}")
|
|
assert find_sidecar(media) == sc
|
|
|
|
|
|
def test_find_sidecar_none(tmp_path):
|
|
media = tmp_path / "photo.jpg"
|
|
media.write_bytes(b"x")
|
|
assert find_sidecar(media) is None
|
|
|
|
|
|
def test_find_sidecar_gallerydl_numbered_prefix(tmp_path):
|
|
"""gallery-dl prefixes media filenames with NN_ for in-post ordering
|
|
(e.g. `01_HOLLOW-ICHIGO.png`) but writes the post-level sidecar under
|
|
the attachment stem WITHOUT the prefix (`HOLLOW-ICHIGO.json`).
|
|
Confirmed against real Patreon downloads 2026-05-26 — operator's deep
|
|
scan produced 24 refresh calls but 0 Posts because the unprefixed
|
|
sidecar was invisible to find_sidecar."""
|
|
media = tmp_path / "01_HOLLOW-ICHIGO.png"
|
|
media.write_bytes(b"x")
|
|
sc = tmp_path / "HOLLOW-ICHIGO.json"
|
|
sc.write_text("{}")
|
|
assert find_sidecar(media) == sc
|
|
|
|
|
|
def test_find_sidecar_multidigit_prefix(tmp_path):
|
|
"""Numbering prefix can be wider than 2 digits (`001_...`); the strip
|
|
handles any \\d+_ form."""
|
|
media = tmp_path / "001_mirko-sketch.png"
|
|
media.write_bytes(b"x")
|
|
sc = tmp_path / "mirko-sketch.json"
|
|
sc.write_text("{}")
|
|
assert find_sidecar(media) == sc
|
|
|
|
|
|
def test_find_sidecar_prefers_attachment_level_over_post_level(tmp_path):
|
|
"""If BOTH a per-attachment sidecar and a post-level sidecar exist,
|
|
the attachment-level one wins (it's more specific)."""
|
|
media = tmp_path / "01_image.png"
|
|
media.write_bytes(b"x")
|
|
per_attachment = tmp_path / "01_image.json"
|
|
per_attachment.write_text('{"specific": true}')
|
|
post_level = tmp_path / "image.json"
|
|
post_level.write_text('{"specific": false}')
|
|
assert find_sidecar(media) == per_attachment
|
|
|
|
|
|
def test_find_sidecar_no_underscore_not_treated_as_prefix(tmp_path):
|
|
"""`01.png` (just digits, no underscore-separated stem) shouldn't
|
|
match. The regex requires NN_<something>."""
|
|
media = tmp_path / "01.png"
|
|
media.write_bytes(b"x")
|
|
(tmp_path / ".json").write_text("{}") # would be matched only if buggy
|
|
assert find_sidecar(media) is None
|
|
|
|
|
|
def test_parse_empty_dict_all_none():
|
|
sd = parse_sidecar({})
|
|
assert isinstance(sd, SidecarData)
|
|
assert (sd.platform, sd.external_post_id, sd.post_url, sd.post_title,
|
|
sd.description, sd.attachment_count, sd.post_date) == (
|
|
None, None, None, None, None, None, None)
|
|
assert sd.raw == {}
|
|
|
|
|
|
def test_parse_core_fields_and_id_priority():
|
|
sd = parse_sidecar({
|
|
"category": "patreon",
|
|
"id": 12345, "post_id": 999,
|
|
"url": "https://patreon.com/posts/12345",
|
|
"title": " Hello ",
|
|
"content": "<p>body</p>",
|
|
"page_count": 4,
|
|
"published_at": "2023-08-01T04:20:02Z",
|
|
})
|
|
assert sd.platform == "patreon"
|
|
assert sd.external_post_id == "12345" # 'id' wins over 'post_id'
|
|
assert sd.post_url == "https://patreon.com/posts/12345"
|
|
assert sd.post_title == "Hello"
|
|
assert sd.description == "<p>body</p>"
|
|
assert sd.attachment_count == 4
|
|
assert sd.post_date.year == 2023 and sd.post_date.tzinfo is not None
|
|
|
|
|
|
def test_parse_description_precedence_and_images_count():
|
|
sd = parse_sidecar({"description": "d", "caption": "c",
|
|
"images": [1, 2, 3]})
|
|
assert sd.description == "d" # content>description>caption
|
|
assert sd.attachment_count == 3 # len(images) fallback
|
|
|
|
|
|
def test_parse_date_epoch_and_unparseable_and_naive():
|
|
assert parse_sidecar({"timestamp": 1690857602}).post_date.tzinfo is not None
|
|
assert parse_sidecar({"date": "not-a-date"}).post_date is None
|
|
naive = parse_sidecar({"date": "2023-08-01T00:00:00"}).post_date
|
|
assert naive is not None and naive.utcoffset().total_seconds() == 0
|
|
|
|
|
|
def test_parse_title_derived_from_content_when_empty():
|
|
"""SubscribeStar gallery-dl writes `title: ""` and puts the leading
|
|
sentence in `content` HTML. When `title` is empty, synthesize the
|
|
post title from the content body's first non-empty text line."""
|
|
sd = parse_sidecar({
|
|
"title": "",
|
|
"content": "\n<div>Lets say hello to you guys with my Belle <br><br><br>\n</div>\n",
|
|
})
|
|
assert sd.post_title == "Lets say hello to you guys with my Belle"
|
|
assert sd.description == (
|
|
"<div>Lets say hello to you guys with my Belle <br><br><br>\n</div>"
|
|
)
|
|
|
|
|
|
def test_parse_title_derived_truncates_long_content():
|
|
long = "x" * 200
|
|
sd = parse_sidecar({"title": "", "content": long})
|
|
assert sd.post_title is not None
|
|
assert len(sd.post_title) <= 120
|
|
assert sd.post_title.endswith("…")
|
|
|
|
|
|
def test_parse_title_explicit_wins_over_content_fallback():
|
|
"""If `title` is non-empty, the fallback never runs."""
|
|
sd = parse_sidecar({"title": "Real Title", "content": "<p>body line</p>"})
|
|
assert sd.post_title == "Real Title"
|
|
|
|
|
|
def test_parse_title_no_fallback_when_no_content():
|
|
sd = parse_sidecar({"title": ""})
|
|
assert sd.post_title is None
|
|
|
|
|
|
def test_parse_ignores_non_str_junk():
|
|
sd = parse_sidecar({"category": 5, "title": 7, "page_count": "x",
|
|
"id": True})
|
|
assert sd.platform is None and sd.post_title is None
|
|
assert sd.attachment_count is None
|
|
assert sd.external_post_id is None # bool is not a valid id
|
|
assert sd.raw == {"category": 5, "title": 7, "page_count": "x",
|
|
"id": True}
|