From 8771364ceec8b31e77eb102bd9e99f58add89cc6 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 16:06:52 -0400 Subject: [PATCH] fix(subscribestar): port gallery-dl's content + preview-skip extraction faithfully MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Body rendered as a bogus '264 / 265' on every post: our balanced- body regex either returned empty or over-captured into sibling upload divs and the 'View next posts (N / M)' pagination counter. Replace it with gallery-dl's exact _data_from_post rule — content between the post_content-text wrapper and the youtube-uploads div (literal markers), then strip the trix editor's … document wrapper to its inner. Verified against the live cheunart sample: clean per-post bodies, empty for genuinely text-less posts. Also port gallery-dl's _media_from_post preview guard: skip gallery items whose URL is under /previews (locked/blurred teasers) — the SubscribeStar analog of the Patreon gated-preview bug (#874); this is why a locked post yields no media. Tests: body must not bleed into the pagination counter; trix html-document wrapper stripped; /previews items skipped. Fixture now includes the youtube- uploads close marker present in real markup. Co-Authored-By: Claude Opus 4.8 --- backend/app/services/subscribestar_client.py | 47 ++++++++++++++---- tests/test_subscribestar_native.py | 50 +++++++++++++++++++- 2 files changed, 88 insertions(+), 9 deletions(-) diff --git a/backend/app/services/subscribestar_client.py b/backend/app/services/subscribestar_client.py index e247c92..abf93c5 100644 --- a/backend/app/services/subscribestar_client.py +++ b/backend/app/services/subscribestar_client.py @@ -87,12 +87,15 @@ _GDL_HEADERS = { _POST_OPEN = '
([^<]+)
') -# The body: inner of `.post-content .trix-content`. Non-greedy to the LAST -# in the chunk would over-capture; instead capture from trix-content -# to the close of the post-content wrapper, which the post-body block ends with. -_BODY_RE = re.compile( - r'
]*>(.*?)
\s*\s*', re.DOTALL -) +# The body lives between these two LITERAL markers — exactly gallery-dl's +# `_data_from_post` extraction: from the post_content-text wrapper open to the +# youtube-uploads div that always follows post-content. A balanced- regex +# either returned empty or over-captured into sibling upload divs AND the +# "View next posts (N / M)" pagination counter (the "264 / 265" body bug, +# cheunart 2026-06-17). The trix editor wraps rich bodies in a full +# `…` document, so strip to the body inner when present. +_CONTENT_OPEN = '
' +_CONTENT_CLOSE = '
str: + """Substring between the first `start` and the next `end` after it (gallery- + dl's `text.extr`); '' when either marker is absent.""" + i = text.find(start) + if i < 0: + return "" + i += len(start) + j = text.find(end, i) + if j < 0: + return "" + return text[i:j] + + +def _extract_content(chunk: str) -> str: + """The post body HTML — gallery-dl's `_data_from_post` content rule: between + the post_content-text wrapper and the youtube-uploads div, with the trix + editor's `…` wrapper stripped to its inner.""" + content = _extr(chunk, _CONTENT_OPEN, _CONTENT_CLOSE) + if "" in content: + content = _extr(content, "", "") + return content.strip() + + def _split_creator_url(campaign_id: str) -> tuple[str, str]: """`campaign_id` is the creator URL → (base, slug). @@ -316,8 +342,7 @@ class SubscribeStarClient: post_id = m.group(1) date_m = _POST_DATE_RE.search(chunk) published = _parse_ss_datetime(date_m.group(1)) if date_m else None - body_m = _BODY_RE.search(chunk) - content = body_m.group(1).strip() if body_m else "" + content = _extract_content(chunk) return { "id": post_id, "attributes": { @@ -368,6 +393,12 @@ class SubscribeStarClient: rel = it.get("url") if not isinstance(rel, str) or not rel: continue + # gallery-dl's _media_from_post: a gallery item whose URL is under + # /previews is a locked/blurred TEASER, not the real file — skip it + # (the SubscribeStar analog of the Patreon gated-preview bug #874). + # This is why a locked post yields no downloadable media. + if "/previews" in rel: + continue url = urljoin(base + "/", rel) media_id = str(it.get("id") or "") name = it.get("original_filename") diff --git a/tests/test_subscribestar_native.py b/tests/test_subscribestar_native.py index be9742f..7e21f87 100644 --- a/tests/test_subscribestar_native.py +++ b/tests/test_subscribestar_native.py @@ -53,7 +53,12 @@ def _post_html(post_id="111", date="May 01, 2026 12:00 pm", body="

hello

", f'
{date}
' f'
' f'
' - f'
{body}
' + f'
{body}
' + # The youtube-uploads div always follows post-content in real markup; it + # is the close marker our content extraction (and gallery-dl's) keys on. + f'
' + f'' f'{gallery}' ) @@ -206,6 +211,22 @@ def test_extract_media_from_gallery(): assert items[0].post_id == "111" +def test_extract_media_skips_locked_previews(): + # gallery-dl's _media_from_post skips gallery items under /previews (locked/ + # blurred teasers) — the SubscribeStar gated-preview safeguard (#874). + client = SubscribeStarClient(None) + media = [ + {"id": 1, "type": "image", "original_filename": "real.jpg", + "url": "/post_uploads?payload=ABC"}, + {"id": 2, "type": "image", "original_filename": "locked.jpg", + "url": "/previews/abc/locked.jpg"}, + ] + [post] = client._parse_posts(_feed_page(_post_html("111", media=media))) + post["_base"] = "https://subscribestar.adult" + items = client.extract_media(post, {}) + assert [m.media_id for m in items] == ["1"] + + def test_text_post_has_no_media_but_keeps_body(): client = SubscribeStarClient(None) [post] = client._parse_posts(_feed_page(_post_html("111", body="

text only

"))) @@ -214,6 +235,33 @@ def test_text_post_has_no_media_but_keeps_body(): assert "text only" in post["attributes"]["content"] +def test_body_does_not_bleed_into_pagination_counter(): + # Regression (cheunart 2026-06-17): the body must stop at the post's own + # content, not run into sibling upload divs or the "View next posts (N / M)" + # pagination counter (which rendered as a bogus "264 / 265" body/title). + client = SubscribeStarClient(None) + page = _feed_page(_post_html("700", body="

real body

"), next_href="/p?page=2") + page = page.replace( + '
', + '
' + "View next posts (264 / 265)
", + ) + [parsed] = client._parse_posts(page) + content = parsed["attributes"]["content"] + assert "real body" in content + assert "264 / 265" not in content + assert "View next posts" not in content + + +def test_body_strips_trix_html_document_wrapper(): + # The trix editor serializes rich bodies as a full HTML document; keep only + # the inner (gallery-dl parity). + client = SubscribeStarClient(None) + wrapped = "\n
hello world
\n\n" + [parsed] = client._parse_posts(_feed_page(_post_html("701", body=wrapped))) + assert parsed["attributes"]["content"] == "
hello world
" + + # -- client: gating + record key -------------------------------------------- def test_post_is_gated():