From 78a3977f8ac7e94035e852235900568d39620b3d Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 14:25:40 -0400 Subject: [PATCH] fix(subscribestar): initial feed GET is a navigation, not XHR (first-run drift) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First live run (cheunart) tripped the drift guard: "no posts and no recognizable feed container". The browser-saved page was normal (6 posts + posts_container-list), so the parser was fine — our live HTTP fetch got a different response. Cause: the client set X-Requested-With: XMLHttpRequest (+ a JSON Accept) session-wide, so the initial creator-page GET was sent as an XHR. SubscribeStar (Rails) content- negotiates an XHR full-page request to a non-HTML body → no container → drift. Fix: the session now uses browser-like navigation headers (Accept: html, NO X-Requested-With); the XHR header + JSON Accept are applied PER-REQUEST only on the "load more" endpoint (which is a genuine XHR). Drift message now reports the response length + a JSON hint so a recurrence is self-explaining. Regression test pins the header split (navigation initial GET, XHR load-more). Co-Authored-By: Claude Opus 4.8 --- backend/app/services/subscribestar_client.py | 36 ++++++++++++++------ tests/test_subscribestar_native.py | 27 +++++++++++++++ 2 files changed, 52 insertions(+), 11 deletions(-) diff --git a/backend/app/services/subscribestar_client.py b/backend/app/services/subscribestar_client.py index fad3dc2..7cd7043 100644 --- a/backend/app/services/subscribestar_client.py +++ b/backend/app/services/subscribestar_client.py @@ -54,9 +54,16 @@ from .native_ingest_common import ( log = logging.getLogger(__name__) _TIMEOUT_SECONDS = 30.0 -# Accept header for the HTML feed + JSON "load more" + the XHR marker. -_ACCEPT = "text/html,application/xhtml+xml,application/json,*/*" -_XHR_HEADERS = {"X-Requested-With": "XMLHttpRequest"} +# The INITIAL creator-page GET is a browser-like NAVIGATION (Accept: html, NO +# X-Requested-With). SubscribeStar is Rails: an XHR-flagged request to the full +# page content-negotiates to a non-HTML response with no posts_container — which +# tripped the drift guard on the first live run (cheunart, 2026-06-17). The +# "load more" endpoint IS a real XHR and gets these headers per-request. +_NAV_ACCEPT = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" +_LOADMORE_HEADERS = { + "X-Requested-With": "XMLHttpRequest", + "Accept": "application/json, text/javascript, */*; q=0.01", +} # A post block opens with this wrapper; we slice the page between consecutive # occurrences (regex can't match the balanced close of nested divs, so chunk- @@ -153,21 +160,21 @@ class SubscribeStarClient: max_retries: int = _MAX_429_RETRIES, ): self.cookies_path = str(cookies_path) if cookies_path else None - self._session = make_session( - cookies_path, accept=_ACCEPT, extra_headers=_XHR_HEADERS - ) + # Browser-like navigation defaults; the load-more XHR headers are applied + # per-request (NOT session-wide) so the initial page GET isn't flagged XHR. + self._session = make_session(cookies_path, accept=_NAV_ACCEPT) self._request_sleep = request_sleep or 0.0 self._max_retries = max_retries # -- request ----------------------------------------------------------- - def _get(self, url: str) -> requests.Response: + def _get(self, url: str, *, headers: dict | None = None) -> requests.Response: if self._request_sleep > 0: time.sleep(self._request_sleep) attempt = 0 while True: try: - resp = self._session.get(url, timeout=_TIMEOUT_SECONDS) + resp = self._session.get(url, timeout=_TIMEOUT_SECONDS, headers=headers) except requests.RequestException as exc: raise SubscribeStarAPIError( f"SubscribeStar request failed ({url}): {exc}" @@ -216,8 +223,8 @@ class SubscribeStarClient: return text def _loadmore_html(self, url: str) -> str: - """Subsequent pages: GET returns JSON {"html": ""}.""" - resp = self._get(url) + """Subsequent pages: a real XHR GET that returns JSON {"html": "..."}.""" + resp = self._get(url, headers=_LOADMORE_HEADERS) try: payload = resp.json() except ValueError as exc: @@ -386,9 +393,16 @@ class SubscribeStarClient: # body canary catches systematic emptiness across a populated feed; # here we only raise if the feed container itself is missing. if 'data-role="posts_container-list"' not in html: + # Surface what we actually got (length + a JSON hint) — the + # first live run tripped this because an XHR-flagged GET + # returned a non-HTML body, which is invisible from a bare + # "markup changed?" message. + looks_json = html.lstrip()[:1] in ("{", "[") raise SubscribeStarDriftError( f"SubscribeStar feed for {slug!r} had no posts and no " - "recognizable feed container — markup changed?" + f"recognizable feed container ({len(html)} bytes" + f"{', looks like JSON' if looks_json else ''}) — " + "markup changed, or the feed wasn't served as a full page?" ) for post in posts: post["_base"] = base diff --git a/tests/test_subscribestar_native.py b/tests/test_subscribestar_native.py index c84cfd7..8e00576 100644 --- a/tests/test_subscribestar_native.py +++ b/tests/test_subscribestar_native.py @@ -77,6 +77,33 @@ def test_parse_ss_datetime_variants(): assert _parse_ss_datetime("nonsense") is None +# -- client: request headers (regression: live-fetch drift) ----------------- + +def test_initial_fetch_is_navigation_loadmore_is_xhr(monkeypatch): + # The initial creator-page GET must NOT be flagged XHR (Rails content- + # negotiates an XHR full-page request to a non-HTML body → drift; first live + # run, cheunart 2026-06-17). Only the "load more" endpoint is a real XHR. + client = SubscribeStarClient(None) + seen = [] + + class _R: + status_code = 200 + text = '
' + + def json(self): + return {"html": ""} + + def fake_get(url, *, headers=None): + seen.append(headers) + return _R() + + monkeypatch.setattr(client, "_get", fake_get) + client._feed_html("https://subscribestar.adult/x") + client._loadmore_html("https://subscribestar.adult/posts?page=2") + assert seen[0] is None # navigation — no per-request XHR header + assert seen[1]["X-Requested-With"] == "XMLHttpRequest" + + # -- client: iteration + pagination ----------------------------------------- def test_iter_posts_parses_and_paginates(monkeypatch):