fix(subscribestar): match gallery-dl's generic post delimiter (live-feed drift)
The native client split the feed on `<div class="post is-shown`, but `is-shown` is added by SubscribeStar's infinite-scroll JS when a post scrolls into view — present in a browser-SAVED page (what the Step-0 characterization used) but ABSENT from the raw server HTML we and gallery-dl actually fetch. So the live feed (cheunart) parsed to zero posts and raised a false SubscribeStarDriftError. Align with gallery-dl's proven `_pagination`: split on the generic `<div class="post ` (trailing space rules out the hyphenated post-content/ post-date/post-body siblings). Also mirror gallery-dl's redirect-based gating detection (/verify_subscriber, /age_confirmation_warning => auth, not drift). Regression tests: raw server markup without is-shown now parses; an age-wall redirect raises SubscribeStarAuthError. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -68,7 +68,16 @@ _LOADMORE_HEADERS = {
|
||||
# A post block opens with this wrapper; we slice the page between consecutive
|
||||
# occurrences (regex can't match the balanced close of nested divs, so chunk-
|
||||
# per-post is the robust approach gallery-dl also uses).
|
||||
_POST_OPEN = '<div class="post is-shown'
|
||||
#
|
||||
# Delimiter is the GENERIC `<div class="post ` (trailing space — matches the post
|
||||
# container regardless of the classes that follow), exactly what gallery-dl's
|
||||
# `_pagination` splits on. We previously keyed on `<div class="post is-shown`,
|
||||
# but `is-shown` is added by SubscribeStar's infinite-scroll JS when a post
|
||||
# scrolls into view — it's present in a browser-SAVED page but ABSENT from the
|
||||
# raw server HTML we (and gallery-dl) actually fetch, so the raw feed parsed to
|
||||
# zero posts → false drift (cheunart, 2026-06-17). The space rules out the
|
||||
# hyphenated siblings (`post-content`/`post-date`/`post-body`/`post-uploads`).
|
||||
_POST_OPEN = '<div class="post '
|
||||
_POST_ID_RE = re.compile(r'data-id="(\d+)"')
|
||||
_POST_DATE_RE = re.compile(r'<div class="post-date">([^<]+)</div>')
|
||||
# The body: inner of `.post-content .trix-content`. Non-greedy to the LAST
|
||||
@@ -216,6 +225,19 @@ class SubscribeStarClient:
|
||||
time.sleep(delay)
|
||||
continue
|
||||
break
|
||||
# gallery-dl's own gating signal: SubscribeStar 302-redirects an
|
||||
# unauthenticated / age-unconfirmed request to /verify_subscriber or
|
||||
# /age_confirmation_warning (lands as a 200 on that URL). Treat it as auth,
|
||||
# not drift — the fix is to rotate cookies / set the age cookie.
|
||||
if resp.history and (
|
||||
"/verify_subscriber" in resp.url
|
||||
or "/age_confirmation_warning" in resp.url
|
||||
):
|
||||
raise SubscribeStarAuthError(
|
||||
f"SubscribeStar redirected to {resp.url} — auth/age wall "
|
||||
f"(rotate cookies or set the 18+ age cookie; requested {url})",
|
||||
status_code=resp.status_code,
|
||||
)
|
||||
if resp.status_code in (401, 403):
|
||||
raise SubscribeStarAuthError(
|
||||
f"SubscribeStar returned HTTP {resp.status_code} — auth rejected "
|
||||
|
||||
@@ -127,6 +127,56 @@ def test_iter_posts_parses_and_paginates(monkeypatch):
|
||||
assert out[0][0]["_base"] == "https://subscribestar.adult"
|
||||
|
||||
|
||||
def test_iter_posts_parses_raw_server_html_without_is_shown(monkeypatch):
|
||||
# Regression (cheunart, 2026-06-17): the RAW server HTML opens posts as
|
||||
# `<div class="post ...">` WITHOUT the `is-shown` class — that class is added
|
||||
# by SubscribeStar's infinite-scroll JS when a post scrolls into view, so it's
|
||||
# present in a browser-SAVED page but absent from the feed we actually fetch.
|
||||
# The delimiter must match the generic `<div class="post ` (gallery-dl parity),
|
||||
# else the live feed parses to zero posts → false drift.
|
||||
client = SubscribeStarClient(None)
|
||||
raw_post = (
|
||||
'<div class="post for-subscribers" data-id="555" '
|
||||
'data-infinite-scroll-id="555">'
|
||||
'<div class="post-date">May 01, 2026 12:00 pm</div>'
|
||||
'<div class="post-body" data-role="post-body">'
|
||||
'<div class="post-content" data-role="post_content-text">'
|
||||
'<div class="trix-content"><p>raw</p></div></div></div></div>'
|
||||
)
|
||||
page = (
|
||||
'<html><body><div data-role="posts_container-list">'
|
||||
f'{raw_post}</div></body></html>'
|
||||
)
|
||||
monkeypatch.setattr(client, "_feed_html", lambda url: page)
|
||||
out = list(client.iter_posts("https://subscribestar.adult/x"))
|
||||
assert [p["id"] for p, _inc, _cur in out] == ["555"]
|
||||
|
||||
|
||||
def test_redirect_to_age_wall_is_auth_error():
|
||||
# gallery-dl parity: SubscribeStar redirects an unauthenticated / age-
|
||||
# unconfirmed request to /age_confirmation_warning (or /verify_subscriber).
|
||||
# That's auth, not drift — rotate cookies / set the age cookie.
|
||||
client = SubscribeStarClient(None)
|
||||
|
||||
class _Resp:
|
||||
status_code = 200
|
||||
url = "https://subscribestar.adult/age_confirmation_warning"
|
||||
history = [object()]
|
||||
text = ""
|
||||
headers: dict = {}
|
||||
|
||||
class _Session:
|
||||
def get(self, url, timeout=None, headers=None):
|
||||
return _Resp()
|
||||
|
||||
client._session = _Session()
|
||||
try:
|
||||
client._get("https://subscribestar.adult/x")
|
||||
except SubscribeStarAuthError:
|
||||
return
|
||||
raise AssertionError("expected SubscribeStarAuthError")
|
||||
|
||||
|
||||
def test_iter_posts_drift_when_no_container(monkeypatch):
|
||||
client = SubscribeStarClient(None)
|
||||
monkeypatch.setattr(client, "_feed_html", lambda url: "<html>nothing</html>")
|
||||
|
||||
Reference in New Issue
Block a user