fix(subscribestar): port gallery-dl's content + preview-skip extraction faithfully

Body rendered as a bogus '264 / 265' on every post: our balanced-</div> body regex either returned empty or over-captured into sibling upload divs and the 'View next posts (N / M)' pagination counter. Replace it with gallery-dl's exact _data_from_post rule — content between the post_content-text wrapper and the youtube-uploads div (literal markers), then strip the trix editor's <html><body>…</body></html> document wrapper to its inner. Verified against the live cheunart sample: clean per-post bodies, empty for genuinely text-less posts. Also port gallery-dl's _media_from_post preview guard: skip gallery items whose URL is under /previews (locked/blurred teasers) — the SubscribeStar analog of the Patreon gated-preview bug (#874); this is why a locked post yields no media. Tests: body must not bleed into the pagination counter; trix html-document wrapper stripped; /previews items skipped. Fixture now includes the youtube- uploads close marker present in real markup. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-17 16:06:52 -04:00
parent 559d29fe1c
commit 8771364cee
2 changed files with 88 additions and 9 deletions
@@ -87,12 +87,15 @@ _GDL_HEADERS = {
 _POST_OPEN = '<div class="post '
 _POST_ID_RE = re.compile(r'data-id="(\d+)"')
 _POST_DATE_RE = re.compile(r'<div class="post-date">([^<]+)</div>')
-# The body: inner of `.post-content .trix-content`. Non-greedy to the LAST
-# </div></div> in the chunk would over-capture; instead capture from trix-content
-# to the close of the post-content wrapper, which the post-body block ends with.
-_BODY_RE = re.compile(
-    r'<div class="post-content"[^>]*>(.*?)</div>\s*</div>\s*</div>', re.DOTALL
-)
+# The body lives between these two LITERAL markers — exactly gallery-dl's
+# `_data_from_post` extraction: from the post_content-text wrapper open to the
+# youtube-uploads div that always follows post-content. A balanced-</div> regex
+# either returned empty or over-captured into sibling upload divs AND the
+# "View next posts (N / M)" pagination counter (the "264 / 265" body bug,
+# cheunart 2026-06-17). The trix editor wraps rich bodies in a full
+# `<html><body>…</body></html>` document, so strip to the body inner when present.
+_CONTENT_OPEN = '<div class="post-content" data-role="post_content-text">'
+_CONTENT_CLOSE = '</div><div class="post-uploads for-youtube"'
 _GALLERY_RE = re.compile(r'data-gallery="([^"]*)"')
 _NEXT_PAGE_RE = re.compile(
    r'data-role="infinite_scroll-next_page"\s+href="([^"]+)"'
@@ -165,6 +168,29 @@ class MediaItem:
    media_id: str


+def _extr(text: str, start: str, end: str) -> str:
+    """Substring between the first `start` and the next `end` after it (gallery-
+    dl's `text.extr`); '' when either marker is absent."""
+    i = text.find(start)
+    if i < 0:
+        return ""
+    i += len(start)
+    j = text.find(end, i)
+    if j < 0:
+        return ""
+    return text[i:j]
+
+
+def _extract_content(chunk: str) -> str:
+    """The post body HTML — gallery-dl's `_data_from_post` content rule: between
+    the post_content-text wrapper and the youtube-uploads div, with the trix
+    editor's `<html><body>…</body></html>` wrapper stripped to its inner."""
+    content = _extr(chunk, _CONTENT_OPEN, _CONTENT_CLOSE)
+    if "<html><body>" in content:
+        content = _extr(content, "<body>", "</body>")
+    return content.strip()
+
+
 def _split_creator_url(campaign_id: str) -> tuple[str, str]:
    """`campaign_id` is the creator URL → (base, slug).

@@ -316,8 +342,7 @@ class SubscribeStarClient:
        post_id = m.group(1)
        date_m = _POST_DATE_RE.search(chunk)
        published = _parse_ss_datetime(date_m.group(1)) if date_m else None
-        body_m = _BODY_RE.search(chunk)
-        content = body_m.group(1).strip() if body_m else ""
+        content = _extract_content(chunk)
        return {
            "id": post_id,
            "attributes": {
@@ -368,6 +393,12 @@ class SubscribeStarClient:
                rel = it.get("url")
                if not isinstance(rel, str) or not rel:
                    continue
+                # gallery-dl's _media_from_post: a gallery item whose URL is under
+                # /previews is a locked/blurred TEASER, not the real file — skip it
+                # (the SubscribeStar analog of the Patreon gated-preview bug #874).
+                # This is why a locked post yields no downloadable media.
+                if "/previews" in rel:
+                    continue
                url = urljoin(base + "/", rel)
                media_id = str(it.get("id") or "")
                name = it.get("original_filename")
@@ -53,7 +53,12 @@ def _post_html(post_id="111", date="May 01, 2026 12:00 pm", body="<p>hello</p>",
        f'<div class="post-date">{date}</div>'
        f'<div class="post-body" data-role="post-body">'
        f'<div class="post-content" data-role="post_content-text">'
-        f'<div class="trix-content">{body}</div></div></div>'
+        f'<div class="trix-content">{body}</div></div>'
+        # The youtube-uploads div always follows post-content in real markup; it
+        # is the close marker our content extraction (and gallery-dl's) keys on.
+        f'<div class="post-uploads for-youtube" '
+        f'data-role="post_content-youtube_uploads"></div>'
+        f'</div>'
        f'{gallery}</div>'
    )

@@ -206,6 +211,22 @@ def test_extract_media_from_gallery():
    assert items[0].post_id == "111"


+def test_extract_media_skips_locked_previews():
+    # gallery-dl's _media_from_post skips gallery items under /previews (locked/
+    # blurred teasers) — the SubscribeStar gated-preview safeguard (#874).
+    client = SubscribeStarClient(None)
+    media = [
+        {"id": 1, "type": "image", "original_filename": "real.jpg",
+         "url": "/post_uploads?payload=ABC"},
+        {"id": 2, "type": "image", "original_filename": "locked.jpg",
+         "url": "/previews/abc/locked.jpg"},
+    ]
+    [post] = client._parse_posts(_feed_page(_post_html("111", media=media)))
+    post["_base"] = "https://subscribestar.adult"
+    items = client.extract_media(post, {})
+    assert [m.media_id for m in items] == ["1"]
+
+
 def test_text_post_has_no_media_but_keeps_body():
    client = SubscribeStarClient(None)
    [post] = client._parse_posts(_feed_page(_post_html("111", body="<p>text only</p>")))
@@ -214,6 +235,33 @@ def test_text_post_has_no_media_but_keeps_body():
    assert "text only" in post["attributes"]["content"]


+def test_body_does_not_bleed_into_pagination_counter():
+    # Regression (cheunart 2026-06-17): the body must stop at the post's own
+    # content, not run into sibling upload divs or the "View next posts (N / M)"
+    # pagination counter (which rendered as a bogus "264 / 265" body/title).
+    client = SubscribeStarClient(None)
+    page = _feed_page(_post_html("700", body="<p>real body</p>"), next_href="/p?page=2")
+    page = page.replace(
+        '<div data-role="infinite_scroll-next_page" href="/p?page=2"></div>',
+        '<div data-role="infinite_scroll-next_page" href="/p?page=2">'
+        "View next posts (264 / 265)</div>",
+    )
+    [parsed] = client._parse_posts(page)
+    content = parsed["attributes"]["content"]
+    assert "real body" in content
+    assert "264 / 265" not in content
+    assert "View next posts" not in content
+
+
+def test_body_strips_trix_html_document_wrapper():
+    # The trix editor serializes rich bodies as a full HTML document; keep only
+    # the <body> inner (gallery-dl parity).
+    client = SubscribeStarClient(None)
+    wrapped = "<!DOCTYPE html><html><body>\n<div>hello world</div>\n</body></html>\n"
+    [parsed] = client._parse_posts(_feed_page(_post_html("701", body=wrapped)))
+    assert parsed["attributes"]["content"] == "<div>hello world</div>"
+
+
 # -- client: gating + record key --------------------------------------------

 def test_post_is_gated():