From 9201b7b5393e5cdabbd44ccc0ca70c0ac17be952 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 14:52:28 -0400 Subject: [PATCH] diag(subscribestar): name the interstitial in the drift error (title + type) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The XHR fix worked (we now get a real 93KB HTML page, not JSON) but cheunart still drifts — we're being served a full HTML page that isn't the feed. Add _describe_page(): the drift error now reports the page + which known interstitial it resembles (cloudflare/bot-challenge, age-gate, login, captcha), so the next run names the actual cause instead of "markup changed". Strong suspicion: a Cloudflare challenge (python-requests has no JS; cf_clearance is UA-locked and our hardcoded UA likely differs from the cookie-capturing browser). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --- backend/app/services/subscribestar_client.py | 38 ++++++++++++++++---- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/backend/app/services/subscribestar_client.py b/backend/app/services/subscribestar_client.py index 7cd7043..6fe7ce5 100644 --- a/backend/app/services/subscribestar_client.py +++ b/backend/app/services/subscribestar_client.py @@ -85,6 +85,33 @@ _NEXT_PAGE_RE = re.compile( # age cookie missing) rather than a real — possibly empty — creator feed. _LOGIN_MARKERS = ("/session/new", 'data-role="sign_in"', "age_confirmation_warning") +# An interstitial served INSTEAD of the feed — characterized so a drift error +# reports the actual cause (bot challenge / age gate / login) rather than a bare +# "markup changed". (name, substrings to look for, case-insensitive.) +_INTERSTITIAL_MARKERS = ( + ("cloudflare/bot-challenge", ( + "just a moment", "cf-challenge", "challenge-platform", "cf_chl", + "attention required", "enable javascript and cookies", + )), + ("age-gate", ( + "18 or older", "adult content", "age_confirmation", "i am over", + "confirm your age", "must be 18", + )), + ("login", ("/session/new", 'data-role="sign_in"', "sign in", "log in")), + ("captcha", ("g-recaptcha", "hcaptcha", "captcha")), +) + + +def _describe_page(html: str) -> str: + """A short, log-safe description of an unexpected page: its <title> + which + known interstitial it resembles (bot challenge / age gate / login / captcha).""" + m = re.search(r"<title[^>]*>(.*?)", html, re.IGNORECASE | re.DOTALL) + title = unescape(m.group(1).strip())[:120] if m else "(no )" + low = html.lower() + hits = [name for name, needles in _INTERSTITIAL_MARKERS + if any(n.lower() in low for n in needles)] + return f"title={title!r}; resembles: {'/'.join(hits) if hits else 'unrecognized'}" + class SubscribeStarAPIError(NativeIngestError): """Base for native SubscribeStar client failures. status_code / retry_after @@ -393,16 +420,15 @@ class SubscribeStarClient: # body canary catches systematic emptiness across a populated feed; # here we only raise if the feed container itself is missing. if 'data-role="posts_container-list"' not in html: - # Surface what we actually got (length + a JSON hint) — the - # first live run tripped this because an XHR-flagged GET - # returned a non-HTML body, which is invisible from a bare - # "markup changed?" message. + # Report what we actually got — length + the page's identity — + # so a served interstitial (bot challenge / age gate / login) + # is named instead of mislabeled "markup changed". looks_json = html.lstrip()[:1] in ("{", "[") raise SubscribeStarDriftError( f"SubscribeStar feed for {slug!r} had no posts and no " f"recognizable feed container ({len(html)} bytes" - f"{', looks like JSON' if looks_json else ''}) — " - "markup changed, or the feed wasn't served as a full page?" + f"{', looks like JSON' if looks_json else ''}; " + f"{_describe_page(html)})" ) for post in posts: post["_base"] = base