bvandeusen
8771364cee
Body rendered as a bogus '264 / 265' on every post: our balanced-</div> body regex either returned empty or over-captured into sibling upload divs and the 'View next posts (N / M)' pagination counter. Replace it with gallery-dl's exact _data_from_post rule — content between the post_content-text wrapper and the youtube-uploads div (literal markers), then strip the trix editor's <html><body>…</body></html> document wrapper to its inner. Verified against the live cheunart sample: clean per-post bodies, empty for genuinely text-less posts. Also port gallery-dl's _media_from_post preview guard: skip gallery items whose URL is under /previews (locked/blurred teasers) — the SubscribeStar analog of the Patreon gated-preview bug (#874); this is why a locked post yields no media. Tests: body must not bleed into the pagination counter; trix html-document wrapper stripped; /previews items skipped. Fixture now includes the youtube- uploads close marker present in real markup. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
525 lines
23 KiB
Python
525 lines
23 KiB
Python
"""Native SubscribeStar client — the SubscribeStar adapter's read path.
|
|
|
|
Unlike Patreon (a clean JSON:API), SubscribeStar has NO public API: gallery-dl
|
|
scrapes its HTML, and so do we. The platform-agnostic core (`ingest_core`) only
|
|
calls `client.iter_posts` / `extract_media` / the optional seams, so the
|
|
HTML-scrape divergence is contained entirely to this module.
|
|
|
|
Feed shape (characterized from a live sample 2026-06-17 — Scribe note
|
|
"SubscribeStar HTML characterization"):
|
|
- Page 1 is the creator page HTML: GET <base>/<slug>.
|
|
- Each page embeds `data-role="infinite_scroll-next_page" href="/posts?...&page=N
|
|
&slug=<slug>&sort_by=newest"`. That path returns JSON {"html": "<...posts...>"};
|
|
the fragment carries the NEXT page link, until it runs out.
|
|
- A post is `<div class="post is-shown ..." data-id="<post_id>" ...>`; body lives
|
|
in `.post-content .trix-content`; date in `.post-date`; media in a `data-gallery`
|
|
JSON manifest on `.uploads-images`.
|
|
|
|
`campaign_id` for SubscribeStar is the full creator URL (e.g.
|
|
https://subscribestar.adult/sabu) — the host (.com vs .adult) and slug both come
|
|
from it, so no separate resolver is needed.
|
|
|
|
Drift is loud on purpose (mirrors patreon_client): an HTML login/age-gate where
|
|
the feed was expected is AUTH (rotate cookies); a feed whose post structure we
|
|
can't parse at all is DRIFT (the scraper needs updating). FC runs on a
|
|
plain-HTTP homelab; nothing here uses a secure-context Web API.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import re
|
|
import time
|
|
from collections.abc import Iterator
|
|
from dataclasses import dataclass
|
|
from datetime import datetime
|
|
from html import unescape
|
|
from pathlib import Path
|
|
from urllib.parse import urljoin, urlsplit
|
|
|
|
import requests
|
|
|
|
from ..utils.paths import filehash_from_url
|
|
from .native_ingest_common import (
|
|
_MAX_429_RETRIES,
|
|
NativeAuthError,
|
|
NativeDriftError,
|
|
NativeIngestError,
|
|
basename_from_url,
|
|
make_session,
|
|
retry_after_seconds,
|
|
)
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
_TIMEOUT_SECONDS = 30.0
|
|
# Match gallery-dl's default (cookies-only) request profile EXACTLY — the proven
|
|
# way to read SubscribeStar without tripping its /verify_subscriber gate. In that
|
|
# mode gallery-dl's base Extractor._init_session sends a Firefox UA, Accept: */*,
|
|
# Accept-Language, and a same-site Referer (root/) on EVERY request — including
|
|
# the first creator-page GET — and uses NO X-Requested-With on either the creator
|
|
# page or the "load more" JSON endpoint (it GETs and parses the body as JSON).
|
|
# Our prior Chrome UA + missing Referer + XHR toggling looked enough unlike a
|
|
# browser that SubscribeStar 302'd the adult-creator page to /<slug>/verify_
|
|
# subscriber even with valid cookies (cheunart, 2026-06-17).
|
|
_FIREFOX_UA = (
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) "
|
|
"Gecko/20100101 Firefox/140.0"
|
|
)
|
|
_GDL_HEADERS = {
|
|
"User-Agent": _FIREFOX_UA,
|
|
"Accept-Language": "en-US,en;q=0.5",
|
|
}
|
|
|
|
# A post block opens with this wrapper; we slice the page between consecutive
|
|
# occurrences (regex can't match the balanced close of nested divs, so chunk-
|
|
# per-post is the robust approach gallery-dl also uses).
|
|
#
|
|
# Delimiter is the GENERIC `<div class="post ` (trailing space — matches the post
|
|
# container regardless of the classes that follow), exactly what gallery-dl's
|
|
# `_pagination` splits on. We previously keyed on `<div class="post is-shown`,
|
|
# but `is-shown` is added by SubscribeStar's infinite-scroll JS when a post
|
|
# scrolls into view — it's present in a browser-SAVED page but ABSENT from the
|
|
# raw server HTML we (and gallery-dl) actually fetch, so the raw feed parsed to
|
|
# zero posts → false drift (cheunart, 2026-06-17). The space rules out the
|
|
# hyphenated siblings (`post-content`/`post-date`/`post-body`/`post-uploads`).
|
|
_POST_OPEN = '<div class="post '
|
|
_POST_ID_RE = re.compile(r'data-id="(\d+)"')
|
|
_POST_DATE_RE = re.compile(r'<div class="post-date">([^<]+)</div>')
|
|
# The body lives between these two LITERAL markers — exactly gallery-dl's
|
|
# `_data_from_post` extraction: from the post_content-text wrapper open to the
|
|
# youtube-uploads div that always follows post-content. A balanced-</div> regex
|
|
# either returned empty or over-captured into sibling upload divs AND the
|
|
# "View next posts (N / M)" pagination counter (the "264 / 265" body bug,
|
|
# cheunart 2026-06-17). The trix editor wraps rich bodies in a full
|
|
# `<html><body>…</body></html>` document, so strip to the body inner when present.
|
|
_CONTENT_OPEN = '<div class="post-content" data-role="post_content-text">'
|
|
_CONTENT_CLOSE = '</div><div class="post-uploads for-youtube"'
|
|
_GALLERY_RE = re.compile(r'data-gallery="([^"]*)"')
|
|
_NEXT_PAGE_RE = re.compile(
|
|
r'data-role="infinite_scroll-next_page"\s+href="([^"]+)"'
|
|
)
|
|
# Signals an auth/age wall served in place of the feed (cookies expired or the
|
|
# age cookie missing) rather than a real — possibly empty — creator feed.
|
|
_LOGIN_MARKERS = ("/session/new", 'data-role="sign_in"', "age_confirmation_warning")
|
|
|
|
# An interstitial served INSTEAD of the feed — characterized so a drift error
|
|
# reports the actual cause (bot challenge / age gate / login) rather than a bare
|
|
# "markup changed". (name, substrings to look for, case-insensitive.)
|
|
_INTERSTITIAL_MARKERS = (
|
|
("cloudflare/bot-challenge", (
|
|
"just a moment", "cf-challenge", "challenge-platform", "cf_chl",
|
|
"attention required", "enable javascript and cookies",
|
|
)),
|
|
("age-gate", (
|
|
"18 or older", "adult content", "age_confirmation", "i am over",
|
|
"confirm your age", "must be 18",
|
|
)),
|
|
("login", ("/session/new", 'data-role="sign_in"', "sign in", "log in")),
|
|
("captcha", ("g-recaptcha", "hcaptcha", "captcha")),
|
|
)
|
|
|
|
|
|
def _describe_page(html: str) -> str:
|
|
"""A short, log-safe description of an unexpected page: its <title> + which
|
|
known interstitial it resembles (bot challenge / age gate / login / captcha)."""
|
|
m = re.search(r"<title[^>]*>(.*?)</title>", html, re.IGNORECASE | re.DOTALL)
|
|
title = unescape(m.group(1).strip())[:120] if m else "(no <title>)"
|
|
low = html.lower()
|
|
hits = [name for name, needles in _INTERSTITIAL_MARKERS
|
|
if any(n.lower() in low for n in needles)]
|
|
return f"title={title!r}; resembles: {'/'.join(hits) if hits else 'unrecognized'}"
|
|
|
|
|
|
class SubscribeStarAPIError(NativeIngestError):
|
|
"""Base for native SubscribeStar client failures. status_code / retry_after
|
|
are inherited from NativeIngestError."""
|
|
|
|
|
|
class SubscribeStarAuthError(SubscribeStarAPIError, NativeAuthError):
|
|
"""Auth/authorization failure — expired cookies, missing age cookie, or an
|
|
HTML login/age wall served where the feed was expected. Fix = rotate the
|
|
credential, not update the scraper. Maps to error_type 'auth_error'."""
|
|
|
|
|
|
class SubscribeStarDriftError(SubscribeStarAPIError, NativeDriftError):
|
|
"""The feed HTML did not match the structure we scrape (no recognizable post
|
|
blocks AND no known empty-feed state). The scrape analog of API drift — fail
|
|
loud so the import step flags 'SubscribeStar changed its markup' instead of
|
|
silently importing nothing."""
|
|
|
|
|
|
@dataclass
|
|
class MediaItem:
|
|
"""One resolved downloadable item belonging to a SubscribeStar post.
|
|
|
|
Fields mirror patreon_client.MediaItem (so the downloader is structurally the
|
|
same) plus `media_id` — the stable per-upload gallery id. SubscribeStar's
|
|
full-res URL is an opaque `/post_uploads?payload=...` (not content-addressed),
|
|
so `filehash` is usually None and the ledger keys on `<post_id>:<media_id>`.
|
|
"""
|
|
|
|
url: str
|
|
filename: str
|
|
kind: str
|
|
filehash: str | None
|
|
post_id: str
|
|
media_id: str
|
|
|
|
|
|
def _extr(text: str, start: str, end: str) -> str:
|
|
"""Substring between the first `start` and the next `end` after it (gallery-
|
|
dl's `text.extr`); '' when either marker is absent."""
|
|
i = text.find(start)
|
|
if i < 0:
|
|
return ""
|
|
i += len(start)
|
|
j = text.find(end, i)
|
|
if j < 0:
|
|
return ""
|
|
return text[i:j]
|
|
|
|
|
|
def _extract_content(chunk: str) -> str:
|
|
"""The post body HTML — gallery-dl's `_data_from_post` content rule: between
|
|
the post_content-text wrapper and the youtube-uploads div, with the trix
|
|
editor's `<html><body>…</body></html>` wrapper stripped to its inner."""
|
|
content = _extr(chunk, _CONTENT_OPEN, _CONTENT_CLOSE)
|
|
if "<html><body>" in content:
|
|
content = _extr(content, "<body>", "</body>")
|
|
return content.strip()
|
|
|
|
|
|
def _split_creator_url(campaign_id: str) -> tuple[str, str]:
|
|
"""`campaign_id` is the creator URL → (base, slug).
|
|
|
|
base = scheme://host (preserving .com vs .adult); slug = first path segment.
|
|
"""
|
|
parts = urlsplit(campaign_id)
|
|
base = f"{parts.scheme or 'https'}://{parts.netloc}"
|
|
slug = parts.path.strip("/").split("/")[0] if parts.path else ""
|
|
return base, slug
|
|
|
|
|
|
def _parse_ss_datetime(text: str) -> str | None:
|
|
"""SubscribeStar renders human dates: 'Jun 17, 2026 03:19 am' (and an
|
|
'Updated on <date>' variant). Return ISO-8601 (UTC-naive) or None."""
|
|
s = unescape(text or "").strip()
|
|
if s.lower().startswith("updated on "):
|
|
s = s[len("updated on "):].strip()
|
|
for fmt in ("%b %d, %Y %I:%M %p", "%b %d, %Y"):
|
|
try:
|
|
return datetime.strptime(s, fmt).isoformat()
|
|
except ValueError:
|
|
continue
|
|
return None
|
|
|
|
|
|
class SubscribeStarClient:
|
|
"""Synchronous SubscribeStar HTML-scrape read client. Construct with a path
|
|
to a Netscape cookies.txt (the same file CredentialService.get_cookies_path
|
|
materializes, already carrying the age cookie via augment_cookies)."""
|
|
|
|
def __init__(
|
|
self,
|
|
cookies_path: str | Path | None,
|
|
*,
|
|
request_sleep: float = 0.0,
|
|
max_retries: int = _MAX_429_RETRIES,
|
|
):
|
|
self.cookies_path = str(cookies_path) if cookies_path else None
|
|
# gallery-dl-parity request profile (Firefox UA, Accept: */*, Accept-
|
|
# Language; Referer is stamped per-walk once the creator base is known).
|
|
self._session = make_session(
|
|
cookies_path, accept="*/*", extra_headers=_GDL_HEADERS
|
|
)
|
|
self._request_sleep = request_sleep or 0.0
|
|
self._max_retries = max_retries
|
|
|
|
# -- request -----------------------------------------------------------
|
|
|
|
def _get(self, url: str, *, headers: dict | None = None) -> requests.Response:
|
|
if self._request_sleep > 0:
|
|
time.sleep(self._request_sleep)
|
|
attempt = 0
|
|
while True:
|
|
try:
|
|
resp = self._session.get(url, timeout=_TIMEOUT_SECONDS, headers=headers)
|
|
except requests.RequestException as exc:
|
|
raise SubscribeStarAPIError(
|
|
f"SubscribeStar request failed ({url}): {exc}"
|
|
) from exc
|
|
if resp.status_code == 429 and attempt < self._max_retries:
|
|
attempt += 1
|
|
delay = retry_after_seconds(resp, attempt)
|
|
log.warning(
|
|
"SubscribeStar 429 (%s) — backing off %.1fs (retry %d/%d)",
|
|
url, delay, attempt, self._max_retries,
|
|
)
|
|
time.sleep(delay)
|
|
continue
|
|
break
|
|
# gallery-dl's own gating signal: SubscribeStar 302-redirects an
|
|
# unauthenticated / age-unconfirmed request to /verify_subscriber or
|
|
# /age_confirmation_warning (lands as a 200 on that URL). Treat it as auth,
|
|
# not drift — the fix is to rotate cookies / set the age cookie.
|
|
if resp.history and (
|
|
"/verify_subscriber" in resp.url
|
|
or "/age_confirmation_warning" in resp.url
|
|
):
|
|
raise SubscribeStarAuthError(
|
|
f"SubscribeStar redirected to {resp.url} — auth/age wall "
|
|
f"(rotate cookies or set the 18+ age cookie; requested {url})",
|
|
status_code=resp.status_code,
|
|
)
|
|
if resp.status_code in (401, 403):
|
|
raise SubscribeStarAuthError(
|
|
f"SubscribeStar returned HTTP {resp.status_code} — auth rejected "
|
|
f"(cookies expired or tier insufficient; {url})",
|
|
status_code=resp.status_code,
|
|
)
|
|
if resp.status_code != 200:
|
|
retry_after = None
|
|
if resp.status_code == 429:
|
|
hdr = resp.headers.get("Retry-After")
|
|
if hdr:
|
|
try:
|
|
retry_after = float(hdr)
|
|
except (TypeError, ValueError):
|
|
retry_after = None
|
|
raise SubscribeStarAPIError(
|
|
f"SubscribeStar returned HTTP {resp.status_code} ({url})",
|
|
status_code=resp.status_code,
|
|
retry_after=retry_after,
|
|
)
|
|
return resp
|
|
|
|
def _feed_html(self, url: str) -> str:
|
|
"""Page 1: the creator page (full HTML document)."""
|
|
resp = self._get(url)
|
|
text = resp.text or ""
|
|
if any(m in text for m in _LOGIN_MARKERS) and _POST_OPEN not in text:
|
|
raise SubscribeStarAuthError(
|
|
f"SubscribeStar served a login/age wall instead of the feed "
|
|
f"(cookies expired or age cookie missing; {url})"
|
|
)
|
|
return text
|
|
|
|
def _loadmore_html(self, url: str) -> str:
|
|
"""Subsequent pages: a plain GET (gallery-dl uses no XHR header) to the
|
|
`/posts?...` endpoint, which returns JSON {"html": "..."}."""
|
|
resp = self._get(url)
|
|
try:
|
|
payload = resp.json()
|
|
except ValueError as exc:
|
|
raise SubscribeStarAuthError(
|
|
f"SubscribeStar 'load more' returned non-JSON (session expired?; "
|
|
f"{url}): {exc}"
|
|
) from exc
|
|
html = payload.get("html") if isinstance(payload, dict) else None
|
|
return html if isinstance(html, str) else ""
|
|
|
|
# -- parsing -----------------------------------------------------------
|
|
|
|
@staticmethod
|
|
def _post_chunks(html: str) -> list[str]:
|
|
"""Slice the page into one chunk per post (between consecutive wrapper
|
|
opens). Regex can't match a post's balanced close, so each chunk runs to
|
|
the next post's open (or end of fragment) — enough to scope per-post
|
|
field extraction."""
|
|
starts = [m.start() for m in re.finditer(re.escape(_POST_OPEN), html)]
|
|
chunks = []
|
|
for i, start in enumerate(starts):
|
|
end = starts[i + 1] if i + 1 < len(starts) else len(html)
|
|
chunks.append(html[start:end])
|
|
return chunks
|
|
|
|
def _parse_post(self, chunk: str) -> dict | None:
|
|
m = _POST_ID_RE.search(chunk)
|
|
if not m:
|
|
return None
|
|
post_id = m.group(1)
|
|
date_m = _POST_DATE_RE.search(chunk)
|
|
published = _parse_ss_datetime(date_m.group(1)) if date_m else None
|
|
content = _extract_content(chunk)
|
|
return {
|
|
"id": post_id,
|
|
"attributes": {
|
|
# SubscribeStar has no title field; the importer synthesizes a
|
|
# display title from the body's first line (sidecar util).
|
|
"title": "",
|
|
"content": content,
|
|
"published_at": published,
|
|
"post_type": "subscribestar",
|
|
},
|
|
# Raw chunk retained so extract_media parses the data-gallery manifest
|
|
# and post_is_gated can scan for the locked-teaser marker.
|
|
"_html": chunk,
|
|
}
|
|
|
|
def _parse_posts(self, html: str) -> list[dict]:
|
|
posts = []
|
|
for chunk in self._post_chunks(html):
|
|
post = self._parse_post(chunk)
|
|
if post is not None:
|
|
posts.append(post)
|
|
return posts
|
|
|
|
@staticmethod
|
|
def _next_page_href(html: str) -> str | None:
|
|
m = _NEXT_PAGE_RE.search(html)
|
|
return unescape(m.group(1)) if m else None
|
|
|
|
def extract_media(self, post: dict, included_index: dict) -> list[MediaItem]:
|
|
"""Resolve downloadable media from the post's `data-gallery` JSON
|
|
manifest. Each item carries a stable `id`, `original_filename`, `type`,
|
|
and a full-res `url` (`/post_uploads?payload=...`, relative — joined to the
|
|
post's base). `included_index` is unused (HTML carries media inline)."""
|
|
chunk = post.get("_html") or ""
|
|
base = post.get("_base") or "https://www.subscribestar.com"
|
|
post_id = str(post.get("id") or "")
|
|
items: list[MediaItem] = []
|
|
for gm in _GALLERY_RE.finditer(chunk):
|
|
try:
|
|
gallery = json.loads(unescape(gm.group(1)))
|
|
except (ValueError, TypeError):
|
|
continue
|
|
if not isinstance(gallery, list):
|
|
continue
|
|
for it in gallery:
|
|
if not isinstance(it, dict):
|
|
continue
|
|
rel = it.get("url")
|
|
if not isinstance(rel, str) or not rel:
|
|
continue
|
|
# gallery-dl's _media_from_post: a gallery item whose URL is under
|
|
# /previews is a locked/blurred TEASER, not the real file — skip it
|
|
# (the SubscribeStar analog of the Patreon gated-preview bug #874).
|
|
# This is why a locked post yields no downloadable media.
|
|
if "/previews" in rel:
|
|
continue
|
|
url = urljoin(base + "/", rel)
|
|
media_id = str(it.get("id") or "")
|
|
name = it.get("original_filename")
|
|
filename = name if isinstance(name, str) and name else basename_from_url(url)
|
|
items.append(
|
|
MediaItem(
|
|
url=url,
|
|
filename=filename,
|
|
kind=str(it.get("type") or "image"),
|
|
filehash=filehash_from_url(url),
|
|
post_id=post_id,
|
|
media_id=media_id,
|
|
)
|
|
)
|
|
return items
|
|
|
|
@staticmethod
|
|
def post_meta(post: dict) -> dict:
|
|
"""Title + date for the preview sample. Title is synthesized from the body
|
|
(SubscribeStar has no title field)."""
|
|
attrs = post.get("attributes") or {}
|
|
return {"title": None, "date": attrs.get("published_at")}
|
|
|
|
@staticmethod
|
|
def post_is_gated(post: dict) -> bool:
|
|
"""True when the subscriber cannot view this post (locked teaser). #874
|
|
"no stub for gated content": the core skips it ENTIRELY (no media, no
|
|
post-record), so we don't replicate a hollow teaser in Curator.
|
|
|
|
SubscribeStar does NOT expose downloadable preview media for locked posts
|
|
(no `data-gallery`), so the Patreon "downloaded blurred junk" failure
|
|
can't happen here — this gate only prevents capturing an empty teaser
|
|
stub. Detect the locked marker conservatively (default to NOT gated when
|
|
absent, so we never over-filter an accessible text post). The exact marker
|
|
is being confirmed against a live locked sample; until then we gate only on
|
|
the explicit lock-overlay class SubscribeStar renders on a paywalled post.
|
|
"""
|
|
chunk = post.get("_html") or ""
|
|
return 'class="post-content-locked"' in chunk or "for-locked_content" in chunk
|
|
|
|
@staticmethod
|
|
def post_record_key(post: dict) -> tuple[str, str] | None:
|
|
"""`(ledger_key, post_id)` for a post's seen-ledger entry (the `post:<id>`
|
|
synthetic key gates post-record capture through the same ledger as media),
|
|
or None when the post has no id."""
|
|
pid = post.get("id")
|
|
pid = str(pid) if pid is not None else ""
|
|
if not pid:
|
|
return None
|
|
return (f"post:{pid}", pid)
|
|
|
|
# -- iteration ---------------------------------------------------------
|
|
|
|
def iter_posts(
|
|
self, campaign_id: str, cursor: str | None = None
|
|
) -> Iterator[tuple[dict, dict, str | None]]:
|
|
"""Yield (post, {}, page_cursor) for every post in the feed.
|
|
|
|
`campaign_id` is the creator URL. `cursor` is the relative "load more"
|
|
href that fetches a page (None → page 1, the creator page HTML). The
|
|
yielded `page_cursor` is the href that FETCHED this post's page, so the
|
|
core checkpoints a value that re-fetches the same page on resume (matching
|
|
the Patreon cursor contract).
|
|
"""
|
|
base, slug = _split_creator_url(campaign_id)
|
|
if not slug:
|
|
raise SubscribeStarDriftError(
|
|
f"Could not extract a creator slug from {campaign_id!r}"
|
|
)
|
|
# Same-site Referer (gallery-dl sends root/ on every request).
|
|
self._session.headers["Referer"] = f"{base}/"
|
|
current = cursor
|
|
first_page = True
|
|
while True:
|
|
page_cursor = current
|
|
if current is None:
|
|
html = self._feed_html(f"{base}/{slug}")
|
|
else:
|
|
html = self._loadmore_html(urljoin(base + "/", current))
|
|
posts = self._parse_posts(html)
|
|
if first_page and not posts and _POST_OPEN not in html:
|
|
# Page 1 with no recognizable post wrappers at all: either a brand
|
|
# new creator with zero posts, or our scraper is stale. The core's
|
|
# body canary catches systematic emptiness across a populated feed;
|
|
# here we only raise if the feed container itself is missing.
|
|
if 'data-role="posts_container-list"' not in html:
|
|
# Report what we actually got — length + the page's identity —
|
|
# so a served interstitial (bot challenge / age gate / login)
|
|
# is named instead of mislabeled "markup changed".
|
|
looks_json = html.lstrip()[:1] in ("{", "[")
|
|
raise SubscribeStarDriftError(
|
|
f"SubscribeStar feed for {slug!r} had no posts and no "
|
|
f"recognizable feed container ({len(html)} bytes"
|
|
f"{', looks like JSON' if looks_json else ''}; "
|
|
f"{_describe_page(html)})"
|
|
)
|
|
for post in posts:
|
|
post["_base"] = base
|
|
yield post, {}, page_cursor
|
|
next_href = self._next_page_href(html)
|
|
if not next_href:
|
|
return
|
|
current = next_href
|
|
first_page = False
|
|
|
|
# -- verify ------------------------------------------------------------
|
|
|
|
def verify_auth(self, campaign_id: str) -> tuple[bool | None, str]:
|
|
"""Cheap auth probe: fetch the first feed page and report whether the
|
|
credential authenticated, without downloading anything."""
|
|
base, slug = _split_creator_url(campaign_id)
|
|
if not slug:
|
|
return None, f"Couldn't parse a SubscribeStar creator from {campaign_id!r}"
|
|
self._session.headers["Referer"] = f"{base}/"
|
|
try:
|
|
html = self._feed_html(f"{base}/{slug}")
|
|
except SubscribeStarAuthError as exc:
|
|
return False, f"SubscribeStar rejected the credential — {exc}"
|
|
except SubscribeStarAPIError as exc:
|
|
return None, f"Couldn't verify (network/HTTP issue): {exc}"
|
|
if _POST_OPEN in html or 'data-role="posts_container-list"' in html:
|
|
return True, "Credentials valid — the SubscribeStar feed loaded."
|
|
return None, "Couldn't verify — SubscribeStar feed shape unrecognized."
|