Merge pull request 'fix(subscribestar): initial feed GET is a navigation, not XHR (first-run drift)' (#118) from dev into main

First live run (cheunart) tripped the drift guard: "no posts and no recognizable
feed container". The browser-saved page was normal (6 posts + posts_container-list),
so the parser was fine — our live HTTP fetch got a different response. Cause: the
client set X-Requested-With: XMLHttpRequest (+ a JSON Accept) session-wide, so the
initial creator-page GET was sent as an XHR. SubscribeStar (Rails) content-
negotiates an XHR full-page request to a non-HTML body → no container → drift.

Fix: the session now uses browser-like navigation headers (Accept: html, NO
X-Requested-With); the XHR header + JSON Accept are applied PER-REQUEST only on
the "load more" endpoint (which is a genuine XHR). Drift message now reports the
response length + a JSON hint so a recurrence is self-explaining. Regression test
pins the header split (navigation initial GET, XHR load-more).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

backend/app/services/subscribestar_client.py

@@ -54,9 +54,16 @@ from .native_ingest_common import (
 log = logging.getLogger(__name__)
 
 _TIMEOUT_SECONDS = 30.0
-# Accept header for the HTML feed + JSON "load more" + the XHR marker.
-_ACCEPT = "text/html,application/xhtml+xml,application/json,*/*"
-_XHR_HEADERS = {"X-Requested-With": "XMLHttpRequest"}
+# The INITIAL creator-page GET is a browser-like NAVIGATION (Accept: html, NO
+# X-Requested-With). SubscribeStar is Rails: an XHR-flagged request to the full
+# page content-negotiates to a non-HTML response with no posts_container — which
+# tripped the drift guard on the first live run (cheunart, 2026-06-17). The
+# "load more" endpoint IS a real XHR and gets these headers per-request.
+_NAV_ACCEPT = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+_LOADMORE_HEADERS = {
+    "X-Requested-With": "XMLHttpRequest",
+    "Accept": "application/json, text/javascript, */*; q=0.01",
+}
 
 # A post block opens with this wrapper; we slice the page between consecutive
 # occurrences (regex can't match the balanced close of nested divs, so chunk-
@@ -153,21 +160,21 @@ class SubscribeStarClient:
         max_retries: int = _MAX_429_RETRIES,
     ):
         self.cookies_path = str(cookies_path) if cookies_path else None
-        self._session = make_session(
-            cookies_path, accept=_ACCEPT, extra_headers=_XHR_HEADERS
-        )
+        # Browser-like navigation defaults; the load-more XHR headers are applied
+        # per-request (NOT session-wide) so the initial page GET isn't flagged XHR.
+        self._session = make_session(cookies_path, accept=_NAV_ACCEPT)
         self._request_sleep = request_sleep or 0.0
         self._max_retries = max_retries
 
     # -- request -----------------------------------------------------------
 
-    def _get(self, url: str) -> requests.Response:
+    def _get(self, url: str, *, headers: dict | None = None) -> requests.Response:
         if self._request_sleep > 0:
             time.sleep(self._request_sleep)
         attempt = 0
         while True:
             try:
-                resp = self._session.get(url, timeout=_TIMEOUT_SECONDS)
+                resp = self._session.get(url, timeout=_TIMEOUT_SECONDS, headers=headers)
             except requests.RequestException as exc:
                 raise SubscribeStarAPIError(
                     f"SubscribeStar request failed ({url}): {exc}"
@@ -216,8 +223,8 @@ class SubscribeStarClient:
         return text
 
     def _loadmore_html(self, url: str) -> str:
-        """Subsequent pages: GET returns JSON {"html": "<fragment>"}."""
-        resp = self._get(url)
+        """Subsequent pages: a real XHR GET that returns JSON {"html": "..."}."""
+        resp = self._get(url, headers=_LOADMORE_HEADERS)
         try:
             payload = resp.json()
         except ValueError as exc:
@@ -386,9 +393,16 @@ class SubscribeStarClient:
                 # body canary catches systematic emptiness across a populated feed;
                 # here we only raise if the feed container itself is missing.
                 if 'data-role="posts_container-list"' not in html:
+                    # Surface what we actually got (length + a JSON hint) — the
+                    # first live run tripped this because an XHR-flagged GET
+                    # returned a non-HTML body, which is invisible from a bare
+                    # "markup changed?" message.
+                    looks_json = html.lstrip()[:1] in ("{", "[")
                     raise SubscribeStarDriftError(
                         f"SubscribeStar feed for {slug!r} had no posts and no "
-                        "recognizable feed container — markup changed?"
+                        f"recognizable feed container ({len(html)} bytes"
+                        f"{', looks like JSON' if looks_json else ''}) — "
+                        "markup changed, or the feed wasn't served as a full page?"
                     )
             for post in posts:
                 post["_base"] = base

tests/test_subscribestar_native.py

@@ -77,6 +77,33 @@ def test_parse_ss_datetime_variants():
     assert _parse_ss_datetime("nonsense") is None
 
 
+# -- client: request headers (regression: live-fetch drift) -----------------
+
+def test_initial_fetch_is_navigation_loadmore_is_xhr(monkeypatch):
+    # The initial creator-page GET must NOT be flagged XHR (Rails content-
+    # negotiates an XHR full-page request to a non-HTML body → drift; first live
+    # run, cheunart 2026-06-17). Only the "load more" endpoint is a real XHR.
+    client = SubscribeStarClient(None)
+    seen = []
+
+    class _R:
+        status_code = 200
+        text = '<div data-role="posts_container-list"></div>'
+
+        def json(self):
+            return {"html": ""}
+
+    def fake_get(url, *, headers=None):
+        seen.append(headers)
+        return _R()
+
+    monkeypatch.setattr(client, "_get", fake_get)
+    client._feed_html("https://subscribestar.adult/x")
+    client._loadmore_html("https://subscribestar.adult/posts?page=2")
+    assert seen[0] is None  # navigation — no per-request XHR header
+    assert seen[1]["X-Requested-With"] == "XMLHttpRequest"
+
+
 # -- client: iteration + pagination -----------------------------------------
 
 def test_iter_posts_parses_and_paginates(monkeypatch):