diff --git a/backend/app/services/subscribestar_client.py b/backend/app/services/subscribestar_client.py index 7cd7043..c857300 100644 --- a/backend/app/services/subscribestar_client.py +++ b/backend/app/services/subscribestar_client.py @@ -68,7 +68,16 @@ _LOADMORE_HEADERS = { # A post block opens with this wrapper; we slice the page between consecutive # occurrences (regex can't match the balanced close of nested divs, so chunk- # per-post is the robust approach gallery-dl also uses). -_POST_OPEN = '
([^<]+)
') # The body: inner of `.post-content .trix-content`. Non-greedy to the LAST @@ -85,6 +94,33 @@ _NEXT_PAGE_RE = re.compile( # age cookie missing) rather than a real — possibly empty — creator feed. _LOGIN_MARKERS = ("/session/new", 'data-role="sign_in"', "age_confirmation_warning") +# An interstitial served INSTEAD of the feed — characterized so a drift error +# reports the actual cause (bot challenge / age gate / login) rather than a bare +# "markup changed". (name, substrings to look for, case-insensitive.) +_INTERSTITIAL_MARKERS = ( + ("cloudflare/bot-challenge", ( + "just a moment", "cf-challenge", "challenge-platform", "cf_chl", + "attention required", "enable javascript and cookies", + )), + ("age-gate", ( + "18 or older", "adult content", "age_confirmation", "i am over", + "confirm your age", "must be 18", + )), + ("login", ("/session/new", 'data-role="sign_in"', "sign in", "log in")), + ("captcha", ("g-recaptcha", "hcaptcha", "captcha")), +) + + +def _describe_page(html: str) -> str: + """A short, log-safe description of an unexpected page: its + which + known interstitial it resembles (bot challenge / age gate / login / captcha).""" + m = re.search(r"<title[^>]*>(.*?)", html, re.IGNORECASE | re.DOTALL) + title = unescape(m.group(1).strip())[:120] if m else "(no )" + low = html.lower() + hits = [name for name, needles in _INTERSTITIAL_MARKERS + if any(n.lower() in low for n in needles)] + return f"title={title!r}; resembles: {'/'.join(hits) if hits else 'unrecognized'}" + class SubscribeStarAPIError(NativeIngestError): """Base for native SubscribeStar client failures. status_code / retry_after @@ -189,6 +225,19 @@ class SubscribeStarClient: time.sleep(delay) continue break + # gallery-dl's own gating signal: SubscribeStar 302-redirects an + # unauthenticated / age-unconfirmed request to /verify_subscriber or + # /age_confirmation_warning (lands as a 200 on that URL). Treat it as auth, + # not drift — the fix is to rotate cookies / set the age cookie. + if resp.history and ( + "/verify_subscriber" in resp.url + or "/age_confirmation_warning" in resp.url + ): + raise SubscribeStarAuthError( + f"SubscribeStar redirected to {resp.url} — auth/age wall " + f"(rotate cookies or set the 18+ age cookie; requested {url})", + status_code=resp.status_code, + ) if resp.status_code in (401, 403): raise SubscribeStarAuthError( f"SubscribeStar returned HTTP {resp.status_code} — auth rejected " @@ -393,16 +442,15 @@ class SubscribeStarClient: # body canary catches systematic emptiness across a populated feed; # here we only raise if the feed container itself is missing. if 'data-role="posts_container-list"' not in html: - # Surface what we actually got (length + a JSON hint) — the - # first live run tripped this because an XHR-flagged GET - # returned a non-HTML body, which is invisible from a bare - # "markup changed?" message. + # Report what we actually got — length + the page's identity — + # so a served interstitial (bot challenge / age gate / login) + # is named instead of mislabeled "markup changed". looks_json = html.lstrip()[:1] in ("{", "[") raise SubscribeStarDriftError( f"SubscribeStar feed for {slug!r} had no posts and no " f"recognizable feed container ({len(html)} bytes" - f"{', looks like JSON' if looks_json else ''}) — " - "markup changed, or the feed wasn't served as a full page?" + f"{', looks like JSON' if looks_json else ''}; " + f"{_describe_page(html)})" ) for post in posts: post["_base"] = base diff --git a/tests/test_subscribestar_native.py b/tests/test_subscribestar_native.py index 8e00576..364bd55 100644 --- a/tests/test_subscribestar_native.py +++ b/tests/test_subscribestar_native.py @@ -127,6 +127,56 @@ def test_iter_posts_parses_and_paginates(monkeypatch): assert out[0][0]["_base"] == "https://subscribestar.adult" +def test_iter_posts_parses_raw_server_html_without_is_shown(monkeypatch): + # Regression (cheunart, 2026-06-17): the RAW server HTML opens posts as + # `<div class="post ...">` WITHOUT the `is-shown` class — that class is added + # by SubscribeStar's infinite-scroll JS when a post scrolls into view, so it's + # present in a browser-SAVED page but absent from the feed we actually fetch. + # The delimiter must match the generic `<div class="post ` (gallery-dl parity), + # else the live feed parses to zero posts → false drift. + client = SubscribeStarClient(None) + raw_post = ( + '<div class="post for-subscribers" data-id="555" ' + 'data-infinite-scroll-id="555">' + '<div class="post-date">May 01, 2026 12:00 pm</div>' + '<div class="post-body" data-role="post-body">' + '<div class="post-content" data-role="post_content-text">' + '<div class="trix-content"><p>raw</p></div></div></div></div>' + ) + page = ( + '<html><body><div data-role="posts_container-list">' + f'{raw_post}</div></body></html>' + ) + monkeypatch.setattr(client, "_feed_html", lambda url: page) + out = list(client.iter_posts("https://subscribestar.adult/x")) + assert [p["id"] for p, _inc, _cur in out] == ["555"] + + +def test_redirect_to_age_wall_is_auth_error(): + # gallery-dl parity: SubscribeStar redirects an unauthenticated / age- + # unconfirmed request to /age_confirmation_warning (or /verify_subscriber). + # That's auth, not drift — rotate cookies / set the age cookie. + client = SubscribeStarClient(None) + + class _Resp: + status_code = 200 + url = "https://subscribestar.adult/age_confirmation_warning" + history = [object()] + text = "" + headers: dict = {} + + class _Session: + def get(self, url, timeout=None, headers=None): + return _Resp() + + client._session = _Session() + try: + client._get("https://subscribestar.adult/x") + except SubscribeStarAuthError: + return + raise AssertionError("expected SubscribeStarAuthError") + + def test_iter_posts_drift_when_no_container(monkeypatch): client = SubscribeStarClient(None) monkeypatch.setattr(client, "_feed_html", lambda url: "<html>nothing</html>")