diag(subscribestar): name the interstitial in the drift error (title + type)
The XHR fix worked (we now get a real 93KB HTML page, not JSON) but cheunart still drifts — we're being served a full HTML page that isn't the feed. Add _describe_page(): the drift error now reports the page <title> + which known interstitial it resembles (cloudflare/bot-challenge, age-gate, login, captcha), so the next run names the actual cause instead of "markup changed". Strong suspicion: a Cloudflare challenge (python-requests has no JS; cf_clearance is UA-locked and our hardcoded UA likely differs from the cookie-capturing browser). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -85,6 +85,33 @@ _NEXT_PAGE_RE = re.compile(
|
|||||||
# age cookie missing) rather than a real — possibly empty — creator feed.
|
# age cookie missing) rather than a real — possibly empty — creator feed.
|
||||||
_LOGIN_MARKERS = ("/session/new", 'data-role="sign_in"', "age_confirmation_warning")
|
_LOGIN_MARKERS = ("/session/new", 'data-role="sign_in"', "age_confirmation_warning")
|
||||||
|
|
||||||
|
# An interstitial served INSTEAD of the feed — characterized so a drift error
|
||||||
|
# reports the actual cause (bot challenge / age gate / login) rather than a bare
|
||||||
|
# "markup changed". (name, substrings to look for, case-insensitive.)
|
||||||
|
_INTERSTITIAL_MARKERS = (
|
||||||
|
("cloudflare/bot-challenge", (
|
||||||
|
"just a moment", "cf-challenge", "challenge-platform", "cf_chl",
|
||||||
|
"attention required", "enable javascript and cookies",
|
||||||
|
)),
|
||||||
|
("age-gate", (
|
||||||
|
"18 or older", "adult content", "age_confirmation", "i am over",
|
||||||
|
"confirm your age", "must be 18",
|
||||||
|
)),
|
||||||
|
("login", ("/session/new", 'data-role="sign_in"', "sign in", "log in")),
|
||||||
|
("captcha", ("g-recaptcha", "hcaptcha", "captcha")),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _describe_page(html: str) -> str:
|
||||||
|
"""A short, log-safe description of an unexpected page: its <title> + which
|
||||||
|
known interstitial it resembles (bot challenge / age gate / login / captcha)."""
|
||||||
|
m = re.search(r"<title[^>]*>(.*?)</title>", html, re.IGNORECASE | re.DOTALL)
|
||||||
|
title = unescape(m.group(1).strip())[:120] if m else "(no <title>)"
|
||||||
|
low = html.lower()
|
||||||
|
hits = [name for name, needles in _INTERSTITIAL_MARKERS
|
||||||
|
if any(n.lower() in low for n in needles)]
|
||||||
|
return f"title={title!r}; resembles: {'/'.join(hits) if hits else 'unrecognized'}"
|
||||||
|
|
||||||
|
|
||||||
class SubscribeStarAPIError(NativeIngestError):
|
class SubscribeStarAPIError(NativeIngestError):
|
||||||
"""Base for native SubscribeStar client failures. status_code / retry_after
|
"""Base for native SubscribeStar client failures. status_code / retry_after
|
||||||
@@ -393,16 +420,15 @@ class SubscribeStarClient:
|
|||||||
# body canary catches systematic emptiness across a populated feed;
|
# body canary catches systematic emptiness across a populated feed;
|
||||||
# here we only raise if the feed container itself is missing.
|
# here we only raise if the feed container itself is missing.
|
||||||
if 'data-role="posts_container-list"' not in html:
|
if 'data-role="posts_container-list"' not in html:
|
||||||
# Surface what we actually got (length + a JSON hint) — the
|
# Report what we actually got — length + the page's identity —
|
||||||
# first live run tripped this because an XHR-flagged GET
|
# so a served interstitial (bot challenge / age gate / login)
|
||||||
# returned a non-HTML body, which is invisible from a bare
|
# is named instead of mislabeled "markup changed".
|
||||||
# "markup changed?" message.
|
|
||||||
looks_json = html.lstrip()[:1] in ("{", "[")
|
looks_json = html.lstrip()[:1] in ("{", "[")
|
||||||
raise SubscribeStarDriftError(
|
raise SubscribeStarDriftError(
|
||||||
f"SubscribeStar feed for {slug!r} had no posts and no "
|
f"SubscribeStar feed for {slug!r} had no posts and no "
|
||||||
f"recognizable feed container ({len(html)} bytes"
|
f"recognizable feed container ({len(html)} bytes"
|
||||||
f"{', looks like JSON' if looks_json else ''}) — "
|
f"{', looks like JSON' if looks_json else ''}; "
|
||||||
"markup changed, or the feed wasn't served as a full page?"
|
f"{_describe_page(html)})"
|
||||||
)
|
)
|
||||||
for post in posts:
|
for post in posts:
|
||||||
post["_base"] = base
|
post["_base"] = base
|
||||||
|
|||||||
Reference in New Issue
Block a user