From 9201b7b5393e5cdabbd44ccc0ca70c0ac17be952 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 14:52:28 -0400 Subject: [PATCH 1/2] diag(subscribestar): name the interstitial in the drift error (title + type) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The XHR fix worked (we now get a real 93KB HTML page, not JSON) but cheunart still drifts — we're being served a full HTML page that isn't the feed. Add _describe_page(): the drift error now reports the page + which known interstitial it resembles (cloudflare/bot-challenge, age-gate, login, captcha), so the next run names the actual cause instead of "markup changed". Strong suspicion: a Cloudflare challenge (python-requests has no JS; cf_clearance is UA-locked and our hardcoded UA likely differs from the cookie-capturing browser). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- backend/app/services/subscribestar_client.py | 38 ++++++++++++++++---- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/backend/app/services/subscribestar_client.py b/backend/app/services/subscribestar_client.py index 7cd7043..6fe7ce5 100644 --- a/backend/app/services/subscribestar_client.py +++ b/backend/app/services/subscribestar_client.py @@ -85,6 +85,33 @@ _NEXT_PAGE_RE = re.compile( # age cookie missing) rather than a real — possibly empty — creator feed. _LOGIN_MARKERS = ("/session/new", 'data-role="sign_in"', "age_confirmation_warning") +# An interstitial served INSTEAD of the feed — characterized so a drift error +# reports the actual cause (bot challenge / age gate / login) rather than a bare +# "markup changed". (name, substrings to look for, case-insensitive.) +_INTERSTITIAL_MARKERS = ( + ("cloudflare/bot-challenge", ( + "just a moment", "cf-challenge", "challenge-platform", "cf_chl", + "attention required", "enable javascript and cookies", + )), + ("age-gate", ( + "18 or older", "adult content", "age_confirmation", "i am over", + "confirm your age", "must be 18", + )), + ("login", ("/session/new", 'data-role="sign_in"', "sign in", "log in")), + ("captcha", ("g-recaptcha", "hcaptcha", "captcha")), +) + + +def _describe_page(html: str) -> str: + """A short, log-safe description of an unexpected page: its <title> + which + known interstitial it resembles (bot challenge / age gate / login / captcha).""" + m = re.search(r"<title[^>]*>(.*?)", html, re.IGNORECASE | re.DOTALL) + title = unescape(m.group(1).strip())[:120] if m else "(no )" + low = html.lower() + hits = [name for name, needles in _INTERSTITIAL_MARKERS + if any(n.lower() in low for n in needles)] + return f"title={title!r}; resembles: {'/'.join(hits) if hits else 'unrecognized'}" + class SubscribeStarAPIError(NativeIngestError): """Base for native SubscribeStar client failures. status_code / retry_after @@ -393,16 +420,15 @@ class SubscribeStarClient: # body canary catches systematic emptiness across a populated feed; # here we only raise if the feed container itself is missing. if 'data-role="posts_container-list"' not in html: - # Surface what we actually got (length + a JSON hint) — the - # first live run tripped this because an XHR-flagged GET - # returned a non-HTML body, which is invisible from a bare - # "markup changed?" message. + # Report what we actually got — length + the page's identity — + # so a served interstitial (bot challenge / age gate / login) + # is named instead of mislabeled "markup changed". looks_json = html.lstrip()[:1] in ("{", "[") raise SubscribeStarDriftError( f"SubscribeStar feed for {slug!r} had no posts and no " f"recognizable feed container ({len(html)} bytes" - f"{', looks like JSON' if looks_json else ''}) — " - "markup changed, or the feed wasn't served as a full page?" + f"{', looks like JSON' if looks_json else ''}; " + f"{_describe_page(html)})" ) for post in posts: post["_base"] = base From 204d341a9969b4f5a7781bfe28233203e9717227 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen <bryan.vandeusen@fabledsword.com> Date: Wed, 17 Jun 2026 15:04:05 -0400 Subject: [PATCH 2/2] fix(subscribestar): match gallery-dl's generic post delimiter (live-feed drift) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The native client split the feed on `<div class="post is-shown`, but `is-shown` is added by SubscribeStar's infinite-scroll JS when a post scrolls into view — present in a browser-SAVED page (what the Step-0 characterization used) but ABSENT from the raw server HTML we and gallery-dl actually fetch. So the live feed (cheunart) parsed to zero posts and raised a false SubscribeStarDriftError. Align with gallery-dl's proven `_pagination`: split on the generic `<div class="post ` (trailing space rules out the hyphenated post-content/ post-date/post-body siblings). Also mirror gallery-dl's redirect-based gating detection (/verify_subscriber, /age_confirmation_warning => auth, not drift). Regression tests: raw server markup without is-shown now parses; an age-wall redirect raises SubscribeStarAuthError. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- backend/app/services/subscribestar_client.py | 24 +++++++++- tests/test_subscribestar_native.py | 50 ++++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/backend/app/services/subscribestar_client.py b/backend/app/services/subscribestar_client.py index 6fe7ce5..c857300 100644 --- a/backend/app/services/subscribestar_client.py +++ b/backend/app/services/subscribestar_client.py @@ -68,7 +68,16 @@ _LOADMORE_HEADERS = { # A post block opens with this wrapper; we slice the page between consecutive # occurrences (regex can't match the balanced close of nested divs, so chunk- # per-post is the robust approach gallery-dl also uses). -_POST_OPEN = '<div class="post is-shown' +# +# Delimiter is the GENERIC `<div class="post ` (trailing space — matches the post +# container regardless of the classes that follow), exactly what gallery-dl's +# `_pagination` splits on. We previously keyed on `<div class="post is-shown`, +# but `is-shown` is added by SubscribeStar's infinite-scroll JS when a post +# scrolls into view — it's present in a browser-SAVED page but ABSENT from the +# raw server HTML we (and gallery-dl) actually fetch, so the raw feed parsed to +# zero posts → false drift (cheunart, 2026-06-17). The space rules out the +# hyphenated siblings (`post-content`/`post-date`/`post-body`/`post-uploads`). +_POST_OPEN = '<div class="post ' _POST_ID_RE = re.compile(r'data-id="(\d+)"') _POST_DATE_RE = re.compile(r'<div class="post-date">([^<]+)</div>') # The body: inner of `.post-content .trix-content`. Non-greedy to the LAST @@ -216,6 +225,19 @@ class SubscribeStarClient: time.sleep(delay) continue break + # gallery-dl's own gating signal: SubscribeStar 302-redirects an + # unauthenticated / age-unconfirmed request to /verify_subscriber or + # /age_confirmation_warning (lands as a 200 on that URL). Treat it as auth, + # not drift — the fix is to rotate cookies / set the age cookie. + if resp.history and ( + "/verify_subscriber" in resp.url + or "/age_confirmation_warning" in resp.url + ): + raise SubscribeStarAuthError( + f"SubscribeStar redirected to {resp.url} — auth/age wall " + f"(rotate cookies or set the 18+ age cookie; requested {url})", + status_code=resp.status_code, + ) if resp.status_code in (401, 403): raise SubscribeStarAuthError( f"SubscribeStar returned HTTP {resp.status_code} — auth rejected " diff --git a/tests/test_subscribestar_native.py b/tests/test_subscribestar_native.py index 8e00576..364bd55 100644 --- a/tests/test_subscribestar_native.py +++ b/tests/test_subscribestar_native.py @@ -127,6 +127,56 @@ def test_iter_posts_parses_and_paginates(monkeypatch): assert out[0][0]["_base"] == "https://subscribestar.adult" +def test_iter_posts_parses_raw_server_html_without_is_shown(monkeypatch): + # Regression (cheunart, 2026-06-17): the RAW server HTML opens posts as + # `<div class="post ...">` WITHOUT the `is-shown` class — that class is added + # by SubscribeStar's infinite-scroll JS when a post scrolls into view, so it's + # present in a browser-SAVED page but absent from the feed we actually fetch. + # The delimiter must match the generic `<div class="post ` (gallery-dl parity), + # else the live feed parses to zero posts → false drift. + client = SubscribeStarClient(None) + raw_post = ( + '<div class="post for-subscribers" data-id="555" ' + 'data-infinite-scroll-id="555">' + '<div class="post-date">May 01, 2026 12:00 pm</div>' + '<div class="post-body" data-role="post-body">' + '<div class="post-content" data-role="post_content-text">' + '<div class="trix-content"><p>raw</p></div></div></div></div>' + ) + page = ( + '<html><body><div data-role="posts_container-list">' + f'{raw_post}</div></body></html>' + ) + monkeypatch.setattr(client, "_feed_html", lambda url: page) + out = list(client.iter_posts("https://subscribestar.adult/x")) + assert [p["id"] for p, _inc, _cur in out] == ["555"] + + +def test_redirect_to_age_wall_is_auth_error(): + # gallery-dl parity: SubscribeStar redirects an unauthenticated / age- + # unconfirmed request to /age_confirmation_warning (or /verify_subscriber). + # That's auth, not drift — rotate cookies / set the age cookie. + client = SubscribeStarClient(None) + + class _Resp: + status_code = 200 + url = "https://subscribestar.adult/age_confirmation_warning" + history = [object()] + text = "" + headers: dict = {} + + class _Session: + def get(self, url, timeout=None, headers=None): + return _Resp() + + client._session = _Session() + try: + client._get("https://subscribestar.adult/x") + except SubscribeStarAuthError: + return + raise AssertionError("expected SubscribeStarAuthError") + + def test_iter_posts_drift_when_no_container(monkeypatch): client = SubscribeStarClient(None) monkeypatch.setattr(client, "_feed_html", lambda url: "<html>nothing</html>")