Merge pull request 'fix(subscribestar): match gallery-dl's generic post delimiter (live-feed drift)' (#119) from dev into main

backend/app/services/subscribestar_client.py

@@ -68,7 +68,16 @@ _LOADMORE_HEADERS = {
 # A post block opens with this wrapper; we slice the page between consecutive
 # occurrences (regex can't match the balanced close of nested divs, so chunk-
 # per-post is the robust approach gallery-dl also uses).
-_POST_OPEN = '<div class="post is-shown'
+#
+# Delimiter is the GENERIC `<div class="post ` (trailing space — matches the post
+# container regardless of the classes that follow), exactly what gallery-dl's
+# `_pagination` splits on. We previously keyed on `<div class="post is-shown`,
+# but `is-shown` is added by SubscribeStar's infinite-scroll JS when a post
+# scrolls into view — it's present in a browser-SAVED page but ABSENT from the
+# raw server HTML we (and gallery-dl) actually fetch, so the raw feed parsed to
+# zero posts → false drift (cheunart, 2026-06-17). The space rules out the
+# hyphenated siblings (`post-content`/`post-date`/`post-body`/`post-uploads`).
+_POST_OPEN = '<div class="post '
 _POST_ID_RE = re.compile(r'data-id="(\d+)"')
 _POST_DATE_RE = re.compile(r'<div class="post-date">([^<]+)</div>')
 # The body: inner of `.post-content .trix-content`. Non-greedy to the LAST
@@ -85,6 +94,33 @@ _NEXT_PAGE_RE = re.compile(
 # age cookie missing) rather than a real — possibly empty — creator feed.
 _LOGIN_MARKERS = ("/session/new", 'data-role="sign_in"', "age_confirmation_warning")
 
+# An interstitial served INSTEAD of the feed — characterized so a drift error
+# reports the actual cause (bot challenge / age gate / login) rather than a bare
+# "markup changed". (name, substrings to look for, case-insensitive.)
+_INTERSTITIAL_MARKERS = (
+    ("cloudflare/bot-challenge", (
+        "just a moment", "cf-challenge", "challenge-platform", "cf_chl",
+        "attention required", "enable javascript and cookies",
+    )),
+    ("age-gate", (
+        "18 or older", "adult content", "age_confirmation", "i am over",
+        "confirm your age", "must be 18",
+    )),
+    ("login", ("/session/new", 'data-role="sign_in"', "sign in", "log in")),
+    ("captcha", ("g-recaptcha", "hcaptcha", "captcha")),
+)
+
+
+def _describe_page(html: str) -> str:
+    """A short, log-safe description of an unexpected page: its <title> + which
+    known interstitial it resembles (bot challenge / age gate / login / captcha)."""
+    m = re.search(r"<title[^>]*>(.*?)</title>", html, re.IGNORECASE | re.DOTALL)
+    title = unescape(m.group(1).strip())[:120] if m else "(no <title>)"
+    low = html.lower()
+    hits = [name for name, needles in _INTERSTITIAL_MARKERS
+            if any(n.lower() in low for n in needles)]
+    return f"title={title!r}; resembles: {'/'.join(hits) if hits else 'unrecognized'}"
+
 
 class SubscribeStarAPIError(NativeIngestError):
     """Base for native SubscribeStar client failures. status_code / retry_after
@@ -189,6 +225,19 @@ class SubscribeStarClient:
                 time.sleep(delay)
                 continue
             break
+        # gallery-dl's own gating signal: SubscribeStar 302-redirects an
+        # unauthenticated / age-unconfirmed request to /verify_subscriber or
+        # /age_confirmation_warning (lands as a 200 on that URL). Treat it as auth,
+        # not drift — the fix is to rotate cookies / set the age cookie.
+        if resp.history and (
+            "/verify_subscriber" in resp.url
+            or "/age_confirmation_warning" in resp.url
+        ):
+            raise SubscribeStarAuthError(
+                f"SubscribeStar redirected to {resp.url} — auth/age wall "
+                f"(rotate cookies or set the 18+ age cookie; requested {url})",
+                status_code=resp.status_code,
+            )
         if resp.status_code in (401, 403):
             raise SubscribeStarAuthError(
                 f"SubscribeStar returned HTTP {resp.status_code} — auth rejected "
@@ -393,16 +442,15 @@ class SubscribeStarClient:
                 # body canary catches systematic emptiness across a populated feed;
                 # here we only raise if the feed container itself is missing.
                 if 'data-role="posts_container-list"' not in html:
-                    # Surface what we actually got (length + a JSON hint) — the
-                    # first live run tripped this because an XHR-flagged GET
-                    # returned a non-HTML body, which is invisible from a bare
-                    # "markup changed?" message.
+                    # Report what we actually got — length + the page's identity —
+                    # so a served interstitial (bot challenge / age gate / login)
+                    # is named instead of mislabeled "markup changed".
                     looks_json = html.lstrip()[:1] in ("{", "[")
                     raise SubscribeStarDriftError(
                         f"SubscribeStar feed for {slug!r} had no posts and no "
                         f"recognizable feed container ({len(html)} bytes"
-                        f"{', looks like JSON' if looks_json else ''}) — "
-                        "markup changed, or the feed wasn't served as a full page?"
+                        f"{', looks like JSON' if looks_json else ''}; "
+                        f"{_describe_page(html)})"
                     )
             for post in posts:
                 post["_base"] = base

tests/test_subscribestar_native.py

@@ -127,6 +127,56 @@ def test_iter_posts_parses_and_paginates(monkeypatch):
     assert out[0][0]["_base"] == "https://subscribestar.adult"
 
 
+def test_iter_posts_parses_raw_server_html_without_is_shown(monkeypatch):
+    # Regression (cheunart, 2026-06-17): the RAW server HTML opens posts as
+    # `<div class="post ...">` WITHOUT the `is-shown` class — that class is added
+    # by SubscribeStar's infinite-scroll JS when a post scrolls into view, so it's
+    # present in a browser-SAVED page but absent from the feed we actually fetch.
+    # The delimiter must match the generic `<div class="post ` (gallery-dl parity),
+    # else the live feed parses to zero posts → false drift.
+    client = SubscribeStarClient(None)
+    raw_post = (
+        '<div class="post for-subscribers" data-id="555" '
+        'data-infinite-scroll-id="555">'
+        '<div class="post-date">May 01, 2026 12:00 pm</div>'
+        '<div class="post-body" data-role="post-body">'
+        '<div class="post-content" data-role="post_content-text">'
+        '<div class="trix-content"><p>raw</p></div></div></div></div>'
+    )
+    page = (
+        '<html><body><div data-role="posts_container-list">'
+        f'{raw_post}</div></body></html>'
+    )
+    monkeypatch.setattr(client, "_feed_html", lambda url: page)
+    out = list(client.iter_posts("https://subscribestar.adult/x"))
+    assert [p["id"] for p, _inc, _cur in out] == ["555"]
+
+
+def test_redirect_to_age_wall_is_auth_error():
+    # gallery-dl parity: SubscribeStar redirects an unauthenticated / age-
+    # unconfirmed request to /age_confirmation_warning (or /verify_subscriber).
+    # That's auth, not drift — rotate cookies / set the age cookie.
+    client = SubscribeStarClient(None)
+
+    class _Resp:
+        status_code = 200
+        url = "https://subscribestar.adult/age_confirmation_warning"
+        history = [object()]
+        text = ""
+        headers: dict = {}
+
+    class _Session:
+        def get(self, url, timeout=None, headers=None):
+            return _Resp()
+
+    client._session = _Session()
+    try:
+        client._get("https://subscribestar.adult/x")
+    except SubscribeStarAuthError:
+        return
+    raise AssertionError("expected SubscribeStarAuthError")
+
+
 def test_iter_posts_drift_when_no_container(monkeypatch):
     client = SubscribeStarClient(None)
     monkeypatch.setattr(client, "_feed_html", lambda url: "<html>nothing</html>")