diff --git a/backend/app/services/subscribestar_client.py b/backend/app/services/subscribestar_client.py index c857300..e247c92 100644 --- a/backend/app/services/subscribestar_client.py +++ b/backend/app/services/subscribestar_client.py @@ -54,15 +54,22 @@ from .native_ingest_common import ( log = logging.getLogger(__name__) _TIMEOUT_SECONDS = 30.0 -# The INITIAL creator-page GET is a browser-like NAVIGATION (Accept: html, NO -# X-Requested-With). SubscribeStar is Rails: an XHR-flagged request to the full -# page content-negotiates to a non-HTML response with no posts_container — which -# tripped the drift guard on the first live run (cheunart, 2026-06-17). The -# "load more" endpoint IS a real XHR and gets these headers per-request. -_NAV_ACCEPT = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" -_LOADMORE_HEADERS = { - "X-Requested-With": "XMLHttpRequest", - "Accept": "application/json, text/javascript, */*; q=0.01", +# Match gallery-dl's default (cookies-only) request profile EXACTLY — the proven +# way to read SubscribeStar without tripping its /verify_subscriber gate. In that +# mode gallery-dl's base Extractor._init_session sends a Firefox UA, Accept: */*, +# Accept-Language, and a same-site Referer (root/) on EVERY request — including +# the first creator-page GET — and uses NO X-Requested-With on either the creator +# page or the "load more" JSON endpoint (it GETs and parses the body as JSON). +# Our prior Chrome UA + missing Referer + XHR toggling looked enough unlike a +# browser that SubscribeStar 302'd the adult-creator page to //verify_ +# subscriber even with valid cookies (cheunart, 2026-06-17). +_FIREFOX_UA = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) " + "Gecko/20100101 Firefox/140.0" +) +_GDL_HEADERS = { + "User-Agent": _FIREFOX_UA, + "Accept-Language": "en-US,en;q=0.5", } # A post block opens with this wrapper; we slice the page between consecutive @@ -196,9 +203,11 @@ class SubscribeStarClient: max_retries: int = _MAX_429_RETRIES, ): self.cookies_path = str(cookies_path) if cookies_path else None - # Browser-like navigation defaults; the load-more XHR headers are applied - # per-request (NOT session-wide) so the initial page GET isn't flagged XHR. - self._session = make_session(cookies_path, accept=_NAV_ACCEPT) + # gallery-dl-parity request profile (Firefox UA, Accept: */*, Accept- + # Language; Referer is stamped per-walk once the creator base is known). + self._session = make_session( + cookies_path, accept="*/*", extra_headers=_GDL_HEADERS + ) self._request_sleep = request_sleep or 0.0 self._max_retries = max_retries @@ -272,8 +281,9 @@ class SubscribeStarClient: return text def _loadmore_html(self, url: str) -> str: - """Subsequent pages: a real XHR GET that returns JSON {"html": "..."}.""" - resp = self._get(url, headers=_LOADMORE_HEADERS) + """Subsequent pages: a plain GET (gallery-dl uses no XHR header) to the + `/posts?...` endpoint, which returns JSON {"html": "..."}.""" + resp = self._get(url) try: payload = resp.json() except ValueError as exc: @@ -427,6 +437,8 @@ class SubscribeStarClient: raise SubscribeStarDriftError( f"Could not extract a creator slug from {campaign_id!r}" ) + # Same-site Referer (gallery-dl sends root/ on every request). + self._session.headers["Referer"] = f"{base}/" current = cursor first_page = True while True: @@ -469,6 +481,7 @@ class SubscribeStarClient: base, slug = _split_creator_url(campaign_id) if not slug: return None, f"Couldn't parse a SubscribeStar creator from {campaign_id!r}" + self._session.headers["Referer"] = f"{base}/" try: html = self._feed_html(f"{base}/{slug}") except SubscribeStarAuthError as exc: diff --git a/tests/test_subscribestar_native.py b/tests/test_subscribestar_native.py index 364bd55..be9742f 100644 --- a/tests/test_subscribestar_native.py +++ b/tests/test_subscribestar_native.py @@ -79,29 +79,29 @@ def test_parse_ss_datetime_variants(): # -- client: request headers (regression: live-fetch drift) ----------------- -def test_initial_fetch_is_navigation_loadmore_is_xhr(monkeypatch): - # The initial creator-page GET must NOT be flagged XHR (Rails content- - # negotiates an XHR full-page request to a non-HTML body → drift; first live - # run, cheunart 2026-06-17). Only the "load more" endpoint is a real XHR. +def test_request_profile_matches_gallery_dl(monkeypatch): + # gallery-dl's default (cookies-only) mode: Firefox UA, Accept: */*, a same- + # site Referer (root/), and NO X-Requested-With on EITHER the creator page or + # the "load more" JSON endpoint. That profile is what avoids SubscribeStar's + # /verify_subscriber 302 with valid cookies (cheunart, 2026-06-17). client = SubscribeStarClient(None) - seen = [] + assert "Firefox" in client._session.headers["User-Agent"] + assert client._session.headers["Accept"] == "*/*" + assert "X-Requested-With" not in client._session.headers class _R: status_code = 200 - text = '
' + history: list = [] + url = "https://subscribestar.adult/x" + text = '
' def json(self): return {"html": ""} - def fake_get(url, *, headers=None): - seen.append(headers) - return _R() - - monkeypatch.setattr(client, "_get", fake_get) - client._feed_html("https://subscribestar.adult/x") - client._loadmore_html("https://subscribestar.adult/posts?page=2") - assert seen[0] is None # navigation — no per-request XHR header - assert seen[1]["X-Requested-With"] == "XMLHttpRequest" + monkeypatch.setattr(client, "_get", lambda url, *, headers=None: _R()) + list(client.iter_posts("https://subscribestar.adult/x")) + # Referer is stamped to the creator base for the walk. + assert client._session.headers["Referer"] == "https://subscribestar.adult/" # -- client: iteration + pagination -----------------------------------------