Merge pull request 'fix(subscribestar): mirror gallery-dl's full request profile (verify_subscriber gate)' (#120) from dev into main

backend/app/services/subscribestar_client.py

@@ -54,15 +54,22 @@ from .native_ingest_common import (
 log = logging.getLogger(__name__)
 
 _TIMEOUT_SECONDS = 30.0
-# The INITIAL creator-page GET is a browser-like NAVIGATION (Accept: html, NO
-# X-Requested-With). SubscribeStar is Rails: an XHR-flagged request to the full
-# page content-negotiates to a non-HTML response with no posts_container — which
-# tripped the drift guard on the first live run (cheunart, 2026-06-17). The
-# "load more" endpoint IS a real XHR and gets these headers per-request.
-_NAV_ACCEPT = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
-_LOADMORE_HEADERS = {
-    "X-Requested-With": "XMLHttpRequest",
-    "Accept": "application/json, text/javascript, */*; q=0.01",
+# Match gallery-dl's default (cookies-only) request profile EXACTLY — the proven
+# way to read SubscribeStar without tripping its /verify_subscriber gate. In that
+# mode gallery-dl's base Extractor._init_session sends a Firefox UA, Accept: */*,
+# Accept-Language, and a same-site Referer (root/) on EVERY request — including
+# the first creator-page GET — and uses NO X-Requested-With on either the creator
+# page or the "load more" JSON endpoint (it GETs and parses the body as JSON).
+# Our prior Chrome UA + missing Referer + XHR toggling looked enough unlike a
+# browser that SubscribeStar 302'd the adult-creator page to /<slug>/verify_
+# subscriber even with valid cookies (cheunart, 2026-06-17).
+_FIREFOX_UA = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) "
+    "Gecko/20100101 Firefox/140.0"
+)
+_GDL_HEADERS = {
+    "User-Agent": _FIREFOX_UA,
+    "Accept-Language": "en-US,en;q=0.5",
 }
 
 # A post block opens with this wrapper; we slice the page between consecutive
@@ -196,9 +203,11 @@ class SubscribeStarClient:
         max_retries: int = _MAX_429_RETRIES,
     ):
         self.cookies_path = str(cookies_path) if cookies_path else None
-        # Browser-like navigation defaults; the load-more XHR headers are applied
-        # per-request (NOT session-wide) so the initial page GET isn't flagged XHR.
-        self._session = make_session(cookies_path, accept=_NAV_ACCEPT)
+        # gallery-dl-parity request profile (Firefox UA, Accept: */*, Accept-
+        # Language; Referer is stamped per-walk once the creator base is known).
+        self._session = make_session(
+            cookies_path, accept="*/*", extra_headers=_GDL_HEADERS
+        )
         self._request_sleep = request_sleep or 0.0
         self._max_retries = max_retries
 
@@ -272,8 +281,9 @@ class SubscribeStarClient:
         return text
 
     def _loadmore_html(self, url: str) -> str:
-        """Subsequent pages: a real XHR GET that returns JSON {"html": "..."}."""
-        resp = self._get(url, headers=_LOADMORE_HEADERS)
+        """Subsequent pages: a plain GET (gallery-dl uses no XHR header) to the
+        `/posts?...` endpoint, which returns JSON {"html": "..."}."""
+        resp = self._get(url)
         try:
             payload = resp.json()
         except ValueError as exc:
@@ -427,6 +437,8 @@ class SubscribeStarClient:
             raise SubscribeStarDriftError(
                 f"Could not extract a creator slug from {campaign_id!r}"
             )
+        # Same-site Referer (gallery-dl sends root/ on every request).
+        self._session.headers["Referer"] = f"{base}/"
         current = cursor
         first_page = True
         while True:
@@ -469,6 +481,7 @@ class SubscribeStarClient:
         base, slug = _split_creator_url(campaign_id)
         if not slug:
             return None, f"Couldn't parse a SubscribeStar creator from {campaign_id!r}"
+        self._session.headers["Referer"] = f"{base}/"
         try:
             html = self._feed_html(f"{base}/{slug}")
         except SubscribeStarAuthError as exc:

tests/test_subscribestar_native.py

@@ -79,29 +79,29 @@ def test_parse_ss_datetime_variants():
 
 # -- client: request headers (regression: live-fetch drift) -----------------
 
-def test_initial_fetch_is_navigation_loadmore_is_xhr(monkeypatch):
-    # The initial creator-page GET must NOT be flagged XHR (Rails content-
-    # negotiates an XHR full-page request to a non-HTML body → drift; first live
-    # run, cheunart 2026-06-17). Only the "load more" endpoint is a real XHR.
+def test_request_profile_matches_gallery_dl(monkeypatch):
+    # gallery-dl's default (cookies-only) mode: Firefox UA, Accept: */*, a same-
+    # site Referer (root/), and NO X-Requested-With on EITHER the creator page or
+    # the "load more" JSON endpoint. That profile is what avoids SubscribeStar's
+    # /verify_subscriber 302 with valid cookies (cheunart, 2026-06-17).
     client = SubscribeStarClient(None)
-    seen = []
+    assert "Firefox" in client._session.headers["User-Agent"]
+    assert client._session.headers["Accept"] == "*/*"
+    assert "X-Requested-With" not in client._session.headers
 
     class _R:
         status_code = 200
-        text = '<div data-role="posts_container-list"></div>'
+        history: list = []
+        url = "https://subscribestar.adult/x"
+        text = '<div class="post " data-id="1"></div>'
 
         def json(self):
             return {"html": ""}
 
-    def fake_get(url, *, headers=None):
-        seen.append(headers)
-        return _R()
-
-    monkeypatch.setattr(client, "_get", fake_get)
-    client._feed_html("https://subscribestar.adult/x")
-    client._loadmore_html("https://subscribestar.adult/posts?page=2")
-    assert seen[0] is None  # navigation — no per-request XHR header
-    assert seen[1]["X-Requested-With"] == "XMLHttpRequest"
+    monkeypatch.setattr(client, "_get", lambda url, *, headers=None: _R())
+    list(client.iter_posts("https://subscribestar.adult/x"))
+    # Referer is stamped to the creator base for the walk.
+    assert client._session.headers["Referer"] == "https://subscribestar.adult/"
 
 
 # -- client: iteration + pagination -----------------------------------------