fix(subscribestar): port gallery-dl's content + preview-skip extraction faithfully

Body rendered as a bogus '264 / 265' on every post: our balanced-</div> body regex either returned empty or over-captured into sibling upload divs and the 'View next posts (N / M)' pagination counter. Replace it with gallery-dl's exact _data_from_post rule — content between the post_content-text wrapper and the youtube-uploads div (literal markers), then strip the trix editor's <html><body>…</body></html> document wrapper to its inner. Verified against the live cheunart sample: clean per-post bodies, empty for genuinely text-less posts. Also port gallery-dl's _media_from_post preview guard: skip gallery items whose URL is under /previews (locked/blurred teasers) — the SubscribeStar analog of the Patreon gated-preview bug (#874); this is why a locked post yields no media. Tests: body must not bleed into the pagination counter; trix html-document wrapper stripped; /previews items skipped. Fixture now includes the youtube- uploads close marker present in real markup. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-17 16:06:52 -04:00
parent 559d29fe1c
commit 8771364cee
2 changed files with 88 additions and 9 deletions
@@ -87,12 +87,15 @@ _GDL_HEADERS = {
 _POST_OPEN = '<div class="post '
 _POST_ID_RE = re.compile(r'data-id="(\d+)"')
 _POST_DATE_RE = re.compile(r'<div class="post-date">([^<]+)</div>')
-# The body: inner of `.post-content .trix-content`. Non-greedy to the LAST
+# The body lives between these two LITERAL markers — exactly gallery-dl's
-# </div></div> in the chunk would over-capture; instead capture from trix-content
+# `_data_from_post` extraction: from the post_content-text wrapper open to the
-# to the close of the post-content wrapper, which the post-body block ends with.
+# youtube-uploads div that always follows post-content. A balanced-</div> regex
-_BODY_RE = re.compile(
+# either returned empty or over-captured into sibling upload divs AND the
-    r'<div class="post-content"[^>]*>(.*?)</div>\s*</div>\s*</div>', re.DOTALL
+# "View next posts (N / M)" pagination counter (the "264 / 265" body bug,
-)
+# cheunart 2026-06-17). The trix editor wraps rich bodies in a full
 # `<html><body>…</body></html>` document, so strip to the body inner when present.
 _CONTENT_OPEN = '<div class="post-content" data-role="post_content-text">'
 _CONTENT_CLOSE = '</div><div class="post-uploads for-youtube"'
 _GALLERY_RE = re.compile(r'data-gallery="([^"]*)"')
 _NEXT_PAGE_RE = re.compile(
    r'data-role="infinite_scroll-next_page"\s+href="([^"]+)"'
@@ -165,6 +168,29 @@ class MediaItem:
    media_id: str
 def _extr(text: str, start: str, end: str) -> str:
    """Substring between the first `start` and the next `end` after it (gallery-
    dl's `text.extr`); '' when either marker is absent."""
    i = text.find(start)
    if i < 0:
        return ""
    i += len(start)
    j = text.find(end, i)
    if j < 0:
        return ""
    return text[i:j]
 def _extract_content(chunk: str) -> str:
    """The post body HTML — gallery-dl's `_data_from_post` content rule: between
    the post_content-text wrapper and the youtube-uploads div, with the trix
    editor's `<html><body>…</body></html>` wrapper stripped to its inner."""
    content = _extr(chunk, _CONTENT_OPEN, _CONTENT_CLOSE)
    if "<html><body>" in content:
        content = _extr(content, "<body>", "</body>")
    return content.strip()
 def _split_creator_url(campaign_id: str) -> tuple[str, str]:
    """`campaign_id` is the creator URL → (base, slug).
@@ -316,8 +342,7 @@ class SubscribeStarClient:
        post_id = m.group(1)
        date_m = _POST_DATE_RE.search(chunk)
        published = _parse_ss_datetime(date_m.group(1)) if date_m else None
-        body_m = _BODY_RE.search(chunk)
+        content = _extract_content(chunk)
        content = body_m.group(1).strip() if body_m else ""
        return {
            "id": post_id,
            "attributes": {
@@ -368,6 +393,12 @@ class SubscribeStarClient:
                rel = it.get("url")
                if not isinstance(rel, str) or not rel:
                    continue
                # gallery-dl's _media_from_post: a gallery item whose URL is under
                # /previews is a locked/blurred TEASER, not the real file — skip it
                # (the SubscribeStar analog of the Patreon gated-preview bug #874).
                # This is why a locked post yields no downloadable media.
                if "/previews" in rel:
                    continue
                url = urljoin(base + "/", rel)
                media_id = str(it.get("id") or "")
                name = it.get("original_filename")
@@ -53,7 +53,12 @@ def _post_html(post_id="111", date="May 01, 2026 12:00 pm", body="<p>hello</p>",
        f'<div class="post-date">{date}</div>'
        f'<div class="post-body" data-role="post-body">'
        f'<div class="post-content" data-role="post_content-text">'
-        f'<div class="trix-content">{body}</div></div></div>'
+        f'<div class="trix-content">{body}</div></div>'
        # The youtube-uploads div always follows post-content in real markup; it
        # is the close marker our content extraction (and gallery-dl's) keys on.
        f'<div class="post-uploads for-youtube" '
        f'data-role="post_content-youtube_uploads"></div>'
        f'</div>'
        f'{gallery}</div>'
    )
@@ -206,6 +211,22 @@ def test_extract_media_from_gallery():
    assert items[0].post_id == "111"
 def test_extract_media_skips_locked_previews():
    # gallery-dl's _media_from_post skips gallery items under /previews (locked/
    # blurred teasers) — the SubscribeStar gated-preview safeguard (#874).
    client = SubscribeStarClient(None)
    media = [
        {"id": 1, "type": "image", "original_filename": "real.jpg",
         "url": "/post_uploads?payload=ABC"},
        {"id": 2, "type": "image", "original_filename": "locked.jpg",
         "url": "/previews/abc/locked.jpg"},
    ]
    [post] = client._parse_posts(_feed_page(_post_html("111", media=media)))
    post["_base"] = "https://subscribestar.adult"
    items = client.extract_media(post, {})
    assert [m.media_id for m in items] == ["1"]
 def test_text_post_has_no_media_but_keeps_body():
    client = SubscribeStarClient(None)
    [post] = client._parse_posts(_feed_page(_post_html("111", body="<p>text only</p>")))
@@ -214,6 +235,33 @@ def test_text_post_has_no_media_but_keeps_body():
    assert "text only" in post["attributes"]["content"]
 def test_body_does_not_bleed_into_pagination_counter():
    # Regression (cheunart 2026-06-17): the body must stop at the post's own
    # content, not run into sibling upload divs or the "View next posts (N / M)"
    # pagination counter (which rendered as a bogus "264 / 265" body/title).
    client = SubscribeStarClient(None)
    page = _feed_page(_post_html("700", body="<p>real body</p>"), next_href="/p?page=2")
    page = page.replace(
        '<div data-role="infinite_scroll-next_page" href="/p?page=2"></div>',
        '<div data-role="infinite_scroll-next_page" href="/p?page=2">'
        "View next posts (264 / 265)</div>",
    )
    [parsed] = client._parse_posts(page)
    content = parsed["attributes"]["content"]
    assert "real body" in content
    assert "264 / 265" not in content
    assert "View next posts" not in content
 def test_body_strips_trix_html_document_wrapper():
    # The trix editor serializes rich bodies as a full HTML document; keep only
    # the <body> inner (gallery-dl parity).
    client = SubscribeStarClient(None)
    wrapped = "<!DOCTYPE html><html><body>\n<div>hello world</div>\n</body></html>\n"
    [parsed] = client._parse_posts(_feed_page(_post_html("701", body=wrapped)))
    assert parsed["attributes"]["content"] == "<div>hello world</div>"
 # -- client: gating + record key --------------------------------------------
 def test_post_is_gated():