fix(subscribestar): port gallery-dl's content + preview-skip extraction faithfully

Body rendered as a bogus '264 / 265' on every post: our balanced-</div> body regex either returned empty or over-captured into sibling upload divs and the 'View next posts (N / M)' pagination counter. Replace it with gallery-dl's exact _data_from_post rule — content between the post_content-text wrapper and the youtube-uploads div (literal markers), then strip the trix editor's <html><body>…</body></html> document wrapper to its inner. Verified against the live cheunart sample: clean per-post bodies, empty for genuinely text-less posts. Also port gallery-dl's _media_from_post preview guard: skip gallery items whose URL is under /previews (locked/blurred teasers) — the SubscribeStar analog of the Patreon gated-preview bug (#874); this is why a locked post yields no media. Tests: body must not bleed into the pagination counter; trix html-document wrapper stripped; /previews items skipped. Fixture now includes the youtube- uploads close marker present in real markup. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-17 16:06:52 -04:00
parent 559d29fe1c
commit 8771364cee
2 changed files with 88 additions and 9 deletions
@@ -53,7 +53,12 @@ def _post_html(post_id="111", date="May 01, 2026 12:00 pm", body="<p>hello</p>",
        f'<div class="post-date">{date}</div>'
        f'<div class="post-body" data-role="post-body">'
        f'<div class="post-content" data-role="post_content-text">'
-        f'<div class="trix-content">{body}</div></div></div>'
+        f'<div class="trix-content">{body}</div></div>'
+        # The youtube-uploads div always follows post-content in real markup; it
+        # is the close marker our content extraction (and gallery-dl's) keys on.
+        f'<div class="post-uploads for-youtube" '
+        f'data-role="post_content-youtube_uploads"></div>'
+        f'</div>'
        f'{gallery}</div>'
    )

@@ -206,6 +211,22 @@ def test_extract_media_from_gallery():
    assert items[0].post_id == "111"


+def test_extract_media_skips_locked_previews():
+    # gallery-dl's _media_from_post skips gallery items under /previews (locked/
+    # blurred teasers) — the SubscribeStar gated-preview safeguard (#874).
+    client = SubscribeStarClient(None)
+    media = [
+        {"id": 1, "type": "image", "original_filename": "real.jpg",
+         "url": "/post_uploads?payload=ABC"},
+        {"id": 2, "type": "image", "original_filename": "locked.jpg",
+         "url": "/previews/abc/locked.jpg"},
+    ]
+    [post] = client._parse_posts(_feed_page(_post_html("111", media=media)))
+    post["_base"] = "https://subscribestar.adult"
+    items = client.extract_media(post, {})
+    assert [m.media_id for m in items] == ["1"]
+
+
 def test_text_post_has_no_media_but_keeps_body():
    client = SubscribeStarClient(None)
    [post] = client._parse_posts(_feed_page(_post_html("111", body="<p>text only</p>")))
@@ -214,6 +235,33 @@ def test_text_post_has_no_media_but_keeps_body():
    assert "text only" in post["attributes"]["content"]


+def test_body_does_not_bleed_into_pagination_counter():
+    # Regression (cheunart 2026-06-17): the body must stop at the post's own
+    # content, not run into sibling upload divs or the "View next posts (N / M)"
+    # pagination counter (which rendered as a bogus "264 / 265" body/title).
+    client = SubscribeStarClient(None)
+    page = _feed_page(_post_html("700", body="<p>real body</p>"), next_href="/p?page=2")
+    page = page.replace(
+        '<div data-role="infinite_scroll-next_page" href="/p?page=2"></div>',
+        '<div data-role="infinite_scroll-next_page" href="/p?page=2">'
+        "View next posts (264 / 265)</div>",
+    )
+    [parsed] = client._parse_posts(page)
+    content = parsed["attributes"]["content"]
+    assert "real body" in content
+    assert "264 / 265" not in content
+    assert "View next posts" not in content
+
+
+def test_body_strips_trix_html_document_wrapper():
+    # The trix editor serializes rich bodies as a full HTML document; keep only
+    # the <body> inner (gallery-dl parity).
+    client = SubscribeStarClient(None)
+    wrapped = "<!DOCTYPE html><html><body>\n<div>hello world</div>\n</body></html>\n"
+    [parsed] = client._parse_posts(_feed_page(_post_html("701", body=wrapped)))
+    assert parsed["attributes"]["content"] == "<div>hello world</div>"
+
+
 # -- client: gating + record key --------------------------------------------

 def test_post_is_gated():