bvandeusen
7f6345dccf
Image posts wrap the post date in an <a> permalink (<div class="post-date"><a href="/posts/ID">DATE</a></div>); text-only posts don't. Our hand-written <div class="post-date">([^<]+)</div> regex matched ONLY the unwrapped case, so every image post got a null published_at and sorted to the top of the feed looking broken (cheunart 2026-06-17). Port gallery-dl's _data_from_post method: text up to the first </, then after the last > — handles both. Verified against the live raw feed (all 6 dates now parse). Robust logging (operator request): _parse_posts now logs per-page parse stats (posts / dated / with-body) and a WARNING canary when posts parse but NONE get a date or body while the raw markers are present — i.e. our extraction diverged from the live markup. Makes this failure class diagnosable from the worker log alone, no authed re-fetch needed. Test: a permalink-wrapped date parses to ISO. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
603 lines
27 KiB
Python
603 lines
27 KiB
Python
"""Native SubscribeStar client — the SubscribeStar adapter's read path.
|
|
|
|
Unlike Patreon (a clean JSON:API), SubscribeStar has NO public API: gallery-dl
|
|
scrapes its HTML, and so do we. The platform-agnostic core (`ingest_core`) only
|
|
calls `client.iter_posts` / `extract_media` / the optional seams, so the
|
|
HTML-scrape divergence is contained entirely to this module.
|
|
|
|
Feed shape (characterized from a live sample 2026-06-17 — Scribe note
|
|
"SubscribeStar HTML characterization"):
|
|
- Page 1 is the creator page HTML: GET <base>/<slug>.
|
|
- Each page embeds `data-role="infinite_scroll-next_page" href="/posts?...&page=N
|
|
&slug=<slug>&sort_by=newest"`. That path returns JSON {"html": "<...posts...>"};
|
|
the fragment carries the NEXT page link, until it runs out.
|
|
- A post is `<div class="post is-shown ..." data-id="<post_id>" ...>`; body lives
|
|
in `.post-content .trix-content`; date in `.post-date`; media in a `data-gallery`
|
|
JSON manifest on `.uploads-images`.
|
|
|
|
`campaign_id` for SubscribeStar is the full creator URL (e.g.
|
|
https://subscribestar.adult/sabu) — the host (.com vs .adult) and slug both come
|
|
from it, so no separate resolver is needed.
|
|
|
|
Drift is loud on purpose (mirrors patreon_client): an HTML login/age-gate where
|
|
the feed was expected is AUTH (rotate cookies); a feed whose post structure we
|
|
can't parse at all is DRIFT (the scraper needs updating). FC runs on a
|
|
plain-HTTP homelab; nothing here uses a secure-context Web API.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import re
|
|
import time
|
|
from collections.abc import Iterator
|
|
from dataclasses import dataclass
|
|
from datetime import datetime
|
|
from html import unescape
|
|
from pathlib import Path
|
|
from urllib.parse import urljoin, urlsplit
|
|
|
|
import requests
|
|
|
|
from ..utils.paths import filehash_from_url
|
|
from .native_ingest_common import (
|
|
_MAX_429_RETRIES,
|
|
NativeAuthError,
|
|
NativeDriftError,
|
|
NativeIngestError,
|
|
basename_from_url,
|
|
make_session,
|
|
retry_after_seconds,
|
|
)
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
_TIMEOUT_SECONDS = 30.0
|
|
# Match gallery-dl's default (cookies-only) request profile EXACTLY — the proven
|
|
# way to read SubscribeStar without tripping its /verify_subscriber gate. In that
|
|
# mode gallery-dl's base Extractor._init_session sends a Firefox UA, Accept: */*,
|
|
# Accept-Language, and a same-site Referer (root/) on EVERY request — including
|
|
# the first creator-page GET — and uses NO X-Requested-With on either the creator
|
|
# page or the "load more" JSON endpoint (it GETs and parses the body as JSON).
|
|
# Our prior Chrome UA + missing Referer + XHR toggling looked enough unlike a
|
|
# browser that SubscribeStar 302'd the adult-creator page to /<slug>/verify_
|
|
# subscriber even with valid cookies (cheunart, 2026-06-17).
|
|
_FIREFOX_UA = (
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) "
|
|
"Gecko/20100101 Firefox/140.0"
|
|
)
|
|
_GDL_HEADERS = {
|
|
"User-Agent": _FIREFOX_UA,
|
|
"Accept-Language": "en-US,en;q=0.5",
|
|
}
|
|
|
|
# A post block opens with this wrapper; we slice the page between consecutive
|
|
# occurrences (regex can't match the balanced close of nested divs, so chunk-
|
|
# per-post is the robust approach gallery-dl also uses).
|
|
#
|
|
# Delimiter is the GENERIC `<div class="post ` (trailing space — matches the post
|
|
# container regardless of the classes that follow), exactly what gallery-dl's
|
|
# `_pagination` splits on. We previously keyed on `<div class="post is-shown`,
|
|
# but `is-shown` is added by SubscribeStar's infinite-scroll JS when a post
|
|
# scrolls into view — it's present in a browser-SAVED page but ABSENT from the
|
|
# raw server HTML we (and gallery-dl) actually fetch, so the raw feed parsed to
|
|
# zero posts → false drift (cheunart, 2026-06-17). The space rules out the
|
|
# hyphenated siblings (`post-content`/`post-date`/`post-body`/`post-uploads`).
|
|
_POST_OPEN = '<div class="post '
|
|
_POST_ID_RE = re.compile(r'data-id="(\d+)"')
|
|
# The date may be plain text OR wrapped in an <a> permalink — image posts wrap it
|
|
# (`<div class="post-date"><a href="/posts/ID">DATE</a></div>`), text-only posts
|
|
# don't. gallery-dl's `_data_from_post` handles both: take text up to the first
|
|
# `</`, then whatever follows the last `>`. A naive
|
|
# `<div class="post-date">([^<]+)</div>` regex matched ONLY the unwrapped case, so
|
|
# every image post got a null date and sorted to the top (cheunart 2026-06-17).
|
|
_DATE_OPEN = 'class="post-date">'
|
|
# The body lives between these two LITERAL markers — exactly gallery-dl's
|
|
# `_data_from_post` extraction: from the post_content-text wrapper open to the
|
|
# youtube-uploads div that always follows post-content. A balanced-</div> regex
|
|
# either returned empty or over-captured into sibling upload divs AND the
|
|
# "View next posts (N / M)" pagination counter (the "264 / 265" body bug,
|
|
# cheunart 2026-06-17). The trix editor wraps rich bodies in a full
|
|
# `<html><body>…</body></html>` document, so strip to the body inner when present.
|
|
_CONTENT_OPEN = '<div class="post-content" data-role="post_content-text">'
|
|
_CONTENT_CLOSE = '</div><div class="post-uploads for-youtube"'
|
|
_GALLERY_RE = re.compile(r'data-gallery="([^"]*)"')
|
|
# Document + audio attachments are NOT in data-gallery — gallery-dl's
|
|
# _media_from_post scrapes them from their own `uploads-docs` / `uploads-audios`
|
|
# sections, splitting on each preview block. Some posts deliver content ONLY
|
|
# through these (PDFs/zips, audio), so we must walk them too.
|
|
_DOC_SPLIT_RE = re.compile(r'class="doc_preview[" ]')
|
|
_AUDIO_SPLIT_RE = re.compile(r'class="audio_preview-data[" ]')
|
|
_NEXT_PAGE_RE = re.compile(
|
|
r'data-role="infinite_scroll-next_page"\s+href="([^"]+)"'
|
|
)
|
|
# Signals an auth/age wall served in place of the feed (cookies expired or the
|
|
# age cookie missing) rather than a real — possibly empty — creator feed.
|
|
_LOGIN_MARKERS = ("/session/new", 'data-role="sign_in"', "age_confirmation_warning")
|
|
|
|
# An interstitial served INSTEAD of the feed — characterized so a drift error
|
|
# reports the actual cause (bot challenge / age gate / login) rather than a bare
|
|
# "markup changed". (name, substrings to look for, case-insensitive.)
|
|
_INTERSTITIAL_MARKERS = (
|
|
("cloudflare/bot-challenge", (
|
|
"just a moment", "cf-challenge", "challenge-platform", "cf_chl",
|
|
"attention required", "enable javascript and cookies",
|
|
)),
|
|
("age-gate", (
|
|
"18 or older", "adult content", "age_confirmation", "i am over",
|
|
"confirm your age", "must be 18",
|
|
)),
|
|
("login", ("/session/new", 'data-role="sign_in"', "sign in", "log in")),
|
|
("captcha", ("g-recaptcha", "hcaptcha", "captcha")),
|
|
)
|
|
|
|
|
|
def _describe_page(html: str) -> str:
|
|
"""A short, log-safe description of an unexpected page: its <title> + which
|
|
known interstitial it resembles (bot challenge / age gate / login / captcha)."""
|
|
m = re.search(r"<title[^>]*>(.*?)</title>", html, re.IGNORECASE | re.DOTALL)
|
|
title = unescape(m.group(1).strip())[:120] if m else "(no <title>)"
|
|
low = html.lower()
|
|
hits = [name for name, needles in _INTERSTITIAL_MARKERS
|
|
if any(n.lower() in low for n in needles)]
|
|
return f"title={title!r}; resembles: {'/'.join(hits) if hits else 'unrecognized'}"
|
|
|
|
|
|
class SubscribeStarAPIError(NativeIngestError):
|
|
"""Base for native SubscribeStar client failures. status_code / retry_after
|
|
are inherited from NativeIngestError."""
|
|
|
|
|
|
class SubscribeStarAuthError(SubscribeStarAPIError, NativeAuthError):
|
|
"""Auth/authorization failure — expired cookies, missing age cookie, or an
|
|
HTML login/age wall served where the feed was expected. Fix = rotate the
|
|
credential, not update the scraper. Maps to error_type 'auth_error'."""
|
|
|
|
|
|
class SubscribeStarDriftError(SubscribeStarAPIError, NativeDriftError):
|
|
"""The feed HTML did not match the structure we scrape (no recognizable post
|
|
blocks AND no known empty-feed state). The scrape analog of API drift — fail
|
|
loud so the import step flags 'SubscribeStar changed its markup' instead of
|
|
silently importing nothing."""
|
|
|
|
|
|
@dataclass
|
|
class MediaItem:
|
|
"""One resolved downloadable item belonging to a SubscribeStar post.
|
|
|
|
Fields mirror patreon_client.MediaItem (so the downloader is structurally the
|
|
same) plus `media_id` — the stable per-upload gallery id. SubscribeStar's
|
|
full-res URL is an opaque `/post_uploads?payload=...` (not content-addressed),
|
|
so `filehash` is usually None and the ledger keys on `<post_id>:<media_id>`.
|
|
"""
|
|
|
|
url: str
|
|
filename: str
|
|
kind: str
|
|
filehash: str | None
|
|
post_id: str
|
|
media_id: str
|
|
|
|
|
|
def _extr(text: str, start: str, end: str) -> str:
|
|
"""Substring between the first `start` and the next `end` after it (gallery-
|
|
dl's `text.extr`); '' when either marker is absent."""
|
|
i = text.find(start)
|
|
if i < 0:
|
|
return ""
|
|
i += len(start)
|
|
j = text.find(end, i)
|
|
if j < 0:
|
|
return ""
|
|
return text[i:j]
|
|
|
|
|
|
def _extract_content(chunk: str) -> str:
|
|
"""The post body HTML — gallery-dl's `_data_from_post` content rule: between
|
|
the post_content-text wrapper and the youtube-uploads div, with the trix
|
|
editor's `<html><body>…</body></html>` wrapper stripped to its inner."""
|
|
content = _extr(chunk, _CONTENT_OPEN, _CONTENT_CLOSE)
|
|
if "<html><body>" in content:
|
|
content = _extr(content, "<body>", "</body>")
|
|
return content.strip()
|
|
|
|
|
|
def _attachment_item(
|
|
frag: str, base: str, post_id: str, *, kind: str, title_marker: str, url_attr: str
|
|
) -> MediaItem | None:
|
|
"""One doc/audio attachment from its preview block — gallery-dl's
|
|
`_media_from_post` attachment/audio fields. `url_attr` is the URL-bearing
|
|
attribute (`href="` for docs, `src="` for audio); `title_marker` precedes the
|
|
display name. Returns None when the block carries no URL."""
|
|
rel = unescape(_extr(frag, url_attr, '"'))
|
|
if not rel:
|
|
return None
|
|
url = urljoin(base + "/", rel)
|
|
upload_id = _extr(frag, 'data-upload-id="', '"')
|
|
name = unescape(_extr(frag, title_marker, "<")).strip()
|
|
return MediaItem(
|
|
url=url,
|
|
filename=name or basename_from_url(url),
|
|
kind=kind,
|
|
filehash=filehash_from_url(url),
|
|
post_id=post_id,
|
|
media_id=str(upload_id or ""),
|
|
)
|
|
|
|
|
|
def _split_creator_url(campaign_id: str) -> tuple[str, str]:
|
|
"""`campaign_id` is the creator URL → (base, slug).
|
|
|
|
base = scheme://host (preserving .com vs .adult); slug = first path segment.
|
|
"""
|
|
parts = urlsplit(campaign_id)
|
|
base = f"{parts.scheme or 'https'}://{parts.netloc}"
|
|
slug = parts.path.strip("/").split("/")[0] if parts.path else ""
|
|
return base, slug
|
|
|
|
|
|
def _parse_ss_datetime(text: str) -> str | None:
|
|
"""SubscribeStar renders human dates: 'Jun 17, 2026 03:19 am' (and an
|
|
'Updated on <date>' variant). Return ISO-8601 (UTC-naive) or None."""
|
|
s = unescape(text or "").strip()
|
|
if s.lower().startswith("updated on "):
|
|
s = s[len("updated on "):].strip()
|
|
for fmt in ("%b %d, %Y %I:%M %p", "%b %d, %Y"):
|
|
try:
|
|
return datetime.strptime(s, fmt).isoformat()
|
|
except ValueError:
|
|
continue
|
|
return None
|
|
|
|
|
|
class SubscribeStarClient:
|
|
"""Synchronous SubscribeStar HTML-scrape read client. Construct with a path
|
|
to a Netscape cookies.txt (the same file CredentialService.get_cookies_path
|
|
materializes, already carrying the age cookie via augment_cookies)."""
|
|
|
|
def __init__(
|
|
self,
|
|
cookies_path: str | Path | None,
|
|
*,
|
|
request_sleep: float = 0.0,
|
|
max_retries: int = _MAX_429_RETRIES,
|
|
):
|
|
self.cookies_path = str(cookies_path) if cookies_path else None
|
|
# gallery-dl-parity request profile (Firefox UA, Accept: */*, Accept-
|
|
# Language; Referer is stamped per-walk once the creator base is known).
|
|
self._session = make_session(
|
|
cookies_path, accept="*/*", extra_headers=_GDL_HEADERS
|
|
)
|
|
self._request_sleep = request_sleep or 0.0
|
|
self._max_retries = max_retries
|
|
|
|
# -- request -----------------------------------------------------------
|
|
|
|
def _get(self, url: str, *, headers: dict | None = None) -> requests.Response:
|
|
if self._request_sleep > 0:
|
|
time.sleep(self._request_sleep)
|
|
attempt = 0
|
|
while True:
|
|
try:
|
|
resp = self._session.get(url, timeout=_TIMEOUT_SECONDS, headers=headers)
|
|
except requests.RequestException as exc:
|
|
raise SubscribeStarAPIError(
|
|
f"SubscribeStar request failed ({url}): {exc}"
|
|
) from exc
|
|
if resp.status_code == 429 and attempt < self._max_retries:
|
|
attempt += 1
|
|
delay = retry_after_seconds(resp, attempt)
|
|
log.warning(
|
|
"SubscribeStar 429 (%s) — backing off %.1fs (retry %d/%d)",
|
|
url, delay, attempt, self._max_retries,
|
|
)
|
|
time.sleep(delay)
|
|
continue
|
|
break
|
|
# gallery-dl's own gating signal: SubscribeStar 302-redirects an
|
|
# unauthenticated / age-unconfirmed request to /verify_subscriber or
|
|
# /age_confirmation_warning (lands as a 200 on that URL). Treat it as auth,
|
|
# not drift — the fix is to rotate cookies / set the age cookie.
|
|
if resp.history and (
|
|
"/verify_subscriber" in resp.url
|
|
or "/age_confirmation_warning" in resp.url
|
|
):
|
|
raise SubscribeStarAuthError(
|
|
f"SubscribeStar redirected to {resp.url} — auth/age wall "
|
|
f"(rotate cookies or set the 18+ age cookie; requested {url})",
|
|
status_code=resp.status_code,
|
|
)
|
|
if resp.status_code in (401, 403):
|
|
raise SubscribeStarAuthError(
|
|
f"SubscribeStar returned HTTP {resp.status_code} — auth rejected "
|
|
f"(cookies expired or tier insufficient; {url})",
|
|
status_code=resp.status_code,
|
|
)
|
|
if resp.status_code != 200:
|
|
retry_after = None
|
|
if resp.status_code == 429:
|
|
hdr = resp.headers.get("Retry-After")
|
|
if hdr:
|
|
try:
|
|
retry_after = float(hdr)
|
|
except (TypeError, ValueError):
|
|
retry_after = None
|
|
raise SubscribeStarAPIError(
|
|
f"SubscribeStar returned HTTP {resp.status_code} ({url})",
|
|
status_code=resp.status_code,
|
|
retry_after=retry_after,
|
|
)
|
|
return resp
|
|
|
|
def _feed_html(self, url: str) -> str:
|
|
"""Page 1: the creator page (full HTML document)."""
|
|
resp = self._get(url)
|
|
text = resp.text or ""
|
|
if any(m in text for m in _LOGIN_MARKERS) and _POST_OPEN not in text:
|
|
raise SubscribeStarAuthError(
|
|
f"SubscribeStar served a login/age wall instead of the feed "
|
|
f"(cookies expired or age cookie missing; {url})"
|
|
)
|
|
return text
|
|
|
|
def _loadmore_html(self, url: str) -> str:
|
|
"""Subsequent pages: a plain GET (gallery-dl uses no XHR header) to the
|
|
`/posts?...` endpoint, which returns JSON {"html": "..."}."""
|
|
resp = self._get(url)
|
|
try:
|
|
payload = resp.json()
|
|
except ValueError as exc:
|
|
raise SubscribeStarAuthError(
|
|
f"SubscribeStar 'load more' returned non-JSON (session expired?; "
|
|
f"{url}): {exc}"
|
|
) from exc
|
|
html = payload.get("html") if isinstance(payload, dict) else None
|
|
return html if isinstance(html, str) else ""
|
|
|
|
# -- parsing -----------------------------------------------------------
|
|
|
|
@staticmethod
|
|
def _post_chunks(html: str) -> list[str]:
|
|
"""Slice the page into one chunk per post (between consecutive wrapper
|
|
opens). Regex can't match a post's balanced close, so each chunk runs to
|
|
the next post's open (or end of fragment) — enough to scope per-post
|
|
field extraction."""
|
|
starts = [m.start() for m in re.finditer(re.escape(_POST_OPEN), html)]
|
|
chunks = []
|
|
for i, start in enumerate(starts):
|
|
end = starts[i + 1] if i + 1 < len(starts) else len(html)
|
|
chunks.append(html[start:end])
|
|
return chunks
|
|
|
|
def _parse_post(self, chunk: str) -> dict | None:
|
|
m = _POST_ID_RE.search(chunk)
|
|
if not m:
|
|
return None
|
|
post_id = m.group(1)
|
|
# gallery-dl date method: text up to first '</', then after the last '>'
|
|
# — handles both plain and <a>-wrapped dates (see _DATE_OPEN comment).
|
|
raw_date = _extr(chunk, _DATE_OPEN, "</").rpartition(">")[2]
|
|
published = _parse_ss_datetime(raw_date) if raw_date else None
|
|
content = _extract_content(chunk)
|
|
return {
|
|
"id": post_id,
|
|
"attributes": {
|
|
# SubscribeStar has no title field; the importer synthesizes a
|
|
# display title from the body's first line (sidecar util).
|
|
"title": "",
|
|
"content": content,
|
|
"published_at": published,
|
|
"post_type": "subscribestar",
|
|
},
|
|
# Raw chunk retained so extract_media parses the data-gallery manifest
|
|
# and post_is_gated can scan for the locked-teaser marker.
|
|
"_html": chunk,
|
|
}
|
|
|
|
def _parse_posts(self, html: str) -> list[dict]:
|
|
posts = []
|
|
for chunk in self._post_chunks(html):
|
|
post = self._parse_post(chunk)
|
|
if post is not None:
|
|
posts.append(post)
|
|
if posts:
|
|
dated = sum(1 for p in posts if p["attributes"].get("published_at"))
|
|
bodied = sum(1 for p in posts if p["attributes"].get("content"))
|
|
log.info(
|
|
"SubscribeStar parsed %d posts (%d dated, %d with body)",
|
|
len(posts), dated, bodied,
|
|
)
|
|
# Canary for this exact failure class: posts parsed but NONE got a
|
|
# date or a body, while the raw markers ARE present → our extraction
|
|
# diverged from the live markup (e.g. the <a>-wrapped date bug). Log
|
|
# the marker counts so the cause is diagnosable from the worker log
|
|
# alone, without re-fetching the authed page.
|
|
if dated == 0 or bodied == 0:
|
|
log.warning(
|
|
"SubscribeStar parse canary: %d posts but dated=%d bodied=%d; "
|
|
"raw markers post-date=%d post_content-text=%d data-gallery=%d "
|
|
"— extraction likely diverged from the live markup",
|
|
len(posts), dated, bodied,
|
|
html.count('class="post-date"'),
|
|
html.count("post_content-text"),
|
|
html.count("data-gallery"),
|
|
)
|
|
return posts
|
|
|
|
@staticmethod
|
|
def _next_page_href(html: str) -> str | None:
|
|
m = _NEXT_PAGE_RE.search(html)
|
|
return unescape(m.group(1)) if m else None
|
|
|
|
def extract_media(self, post: dict, included_index: dict) -> list[MediaItem]:
|
|
"""Resolve downloadable media (gallery-dl's `_media_from_post`): the
|
|
per-post `data-gallery` JSON manifest (images/videos), PLUS document
|
|
attachments (`uploads-docs`) and audio (`uploads-audios`) — some posts
|
|
deliver content only through the latter two. `/previews` (locked teaser)
|
|
gallery items are skipped. `included_index` is unused (media is inline)."""
|
|
chunk = post.get("_html") or ""
|
|
base = post.get("_base") or "https://www.subscribestar.com"
|
|
post_id = str(post.get("id") or "")
|
|
items: list[MediaItem] = []
|
|
for gm in _GALLERY_RE.finditer(chunk):
|
|
try:
|
|
gallery = json.loads(unescape(gm.group(1)))
|
|
except (ValueError, TypeError):
|
|
continue
|
|
if not isinstance(gallery, list):
|
|
continue
|
|
for it in gallery:
|
|
if not isinstance(it, dict):
|
|
continue
|
|
rel = it.get("url")
|
|
if not isinstance(rel, str) or not rel:
|
|
continue
|
|
# gallery-dl's _media_from_post: a gallery item whose URL is under
|
|
# /previews is a locked/blurred TEASER, not the real file — skip it
|
|
# (the SubscribeStar analog of the Patreon gated-preview bug #874).
|
|
# This is why a locked post yields no downloadable media.
|
|
if "/previews" in rel:
|
|
continue
|
|
url = urljoin(base + "/", rel)
|
|
media_id = str(it.get("id") or "")
|
|
name = it.get("original_filename")
|
|
filename = name if isinstance(name, str) and name else basename_from_url(url)
|
|
items.append(
|
|
MediaItem(
|
|
url=url,
|
|
filename=filename,
|
|
kind=str(it.get("type") or "image"),
|
|
filehash=filehash_from_url(url),
|
|
post_id=post_id,
|
|
media_id=media_id,
|
|
)
|
|
)
|
|
# Document attachments (uploads-docs → doc_preview blocks): href URL.
|
|
docs = _extr(chunk, 'class="uploads-docs"', 'class="post-edit_form"')
|
|
for frag in _DOC_SPLIT_RE.split(docs)[1:]:
|
|
item = _attachment_item(
|
|
frag, base, post_id, kind="attachment",
|
|
title_marker='doc_preview-title">', url_attr='href="',
|
|
)
|
|
if item is not None:
|
|
items.append(item)
|
|
# Audio attachments (uploads-audios → audio_preview-data blocks): src URL.
|
|
audios = _extr(chunk, 'class="uploads-audios"', 'class="post-edit_form"')
|
|
for frag in _AUDIO_SPLIT_RE.split(audios)[1:]:
|
|
item = _attachment_item(
|
|
frag, base, post_id, kind="audio",
|
|
title_marker='audio_preview-title">', url_attr='src="',
|
|
)
|
|
if item is not None:
|
|
items.append(item)
|
|
return items
|
|
|
|
@staticmethod
|
|
def post_meta(post: dict) -> dict:
|
|
"""Title + date for the preview sample. Title is synthesized from the body
|
|
(SubscribeStar has no title field)."""
|
|
attrs = post.get("attributes") or {}
|
|
return {"title": None, "date": attrs.get("published_at")}
|
|
|
|
@staticmethod
|
|
def post_is_gated(post: dict) -> bool:
|
|
"""True when the subscriber cannot view this post (locked teaser). #874
|
|
"no stub for gated content": the core skips it ENTIRELY (no media, no
|
|
post-record), so we don't replicate a hollow teaser in Curator.
|
|
|
|
SubscribeStar does NOT expose downloadable preview media for locked posts
|
|
(no `data-gallery`), so the Patreon "downloaded blurred junk" failure
|
|
can't happen here — this gate only prevents capturing an empty teaser
|
|
stub. Detect the locked marker conservatively (default to NOT gated when
|
|
absent, so we never over-filter an accessible text post). The exact marker
|
|
is being confirmed against a live locked sample; until then we gate only on
|
|
the explicit lock-overlay class SubscribeStar renders on a paywalled post.
|
|
"""
|
|
chunk = post.get("_html") or ""
|
|
return 'class="post-content-locked"' in chunk or "for-locked_content" in chunk
|
|
|
|
@staticmethod
|
|
def post_record_key(post: dict) -> tuple[str, str] | None:
|
|
"""`(ledger_key, post_id)` for a post's seen-ledger entry (the `post:<id>`
|
|
synthetic key gates post-record capture through the same ledger as media),
|
|
or None when the post has no id."""
|
|
pid = post.get("id")
|
|
pid = str(pid) if pid is not None else ""
|
|
if not pid:
|
|
return None
|
|
return (f"post:{pid}", pid)
|
|
|
|
# -- iteration ---------------------------------------------------------
|
|
|
|
def iter_posts(
|
|
self, campaign_id: str, cursor: str | None = None
|
|
) -> Iterator[tuple[dict, dict, str | None]]:
|
|
"""Yield (post, {}, page_cursor) for every post in the feed.
|
|
|
|
`campaign_id` is the creator URL. `cursor` is the relative "load more"
|
|
href that fetches a page (None → page 1, the creator page HTML). The
|
|
yielded `page_cursor` is the href that FETCHED this post's page, so the
|
|
core checkpoints a value that re-fetches the same page on resume (matching
|
|
the Patreon cursor contract).
|
|
"""
|
|
base, slug = _split_creator_url(campaign_id)
|
|
if not slug:
|
|
raise SubscribeStarDriftError(
|
|
f"Could not extract a creator slug from {campaign_id!r}"
|
|
)
|
|
# Same-site Referer (gallery-dl sends root/ on every request).
|
|
self._session.headers["Referer"] = f"{base}/"
|
|
current = cursor
|
|
first_page = True
|
|
while True:
|
|
page_cursor = current
|
|
if current is None:
|
|
html = self._feed_html(f"{base}/{slug}")
|
|
else:
|
|
html = self._loadmore_html(urljoin(base + "/", current))
|
|
posts = self._parse_posts(html)
|
|
if first_page and not posts and _POST_OPEN not in html:
|
|
# Page 1 with no recognizable post wrappers at all: either a brand
|
|
# new creator with zero posts, or our scraper is stale. The core's
|
|
# body canary catches systematic emptiness across a populated feed;
|
|
# here we only raise if the feed container itself is missing.
|
|
if 'data-role="posts_container-list"' not in html:
|
|
# Report what we actually got — length + the page's identity —
|
|
# so a served interstitial (bot challenge / age gate / login)
|
|
# is named instead of mislabeled "markup changed".
|
|
looks_json = html.lstrip()[:1] in ("{", "[")
|
|
raise SubscribeStarDriftError(
|
|
f"SubscribeStar feed for {slug!r} had no posts and no "
|
|
f"recognizable feed container ({len(html)} bytes"
|
|
f"{', looks like JSON' if looks_json else ''}; "
|
|
f"{_describe_page(html)})"
|
|
)
|
|
for post in posts:
|
|
post["_base"] = base
|
|
yield post, {}, page_cursor
|
|
next_href = self._next_page_href(html)
|
|
if not next_href:
|
|
return
|
|
current = next_href
|
|
first_page = False
|
|
|
|
# -- verify ------------------------------------------------------------
|
|
|
|
def verify_auth(self, campaign_id: str) -> tuple[bool | None, str]:
|
|
"""Cheap auth probe: fetch the first feed page and report whether the
|
|
credential authenticated, without downloading anything."""
|
|
base, slug = _split_creator_url(campaign_id)
|
|
if not slug:
|
|
return None, f"Couldn't parse a SubscribeStar creator from {campaign_id!r}"
|
|
self._session.headers["Referer"] = f"{base}/"
|
|
try:
|
|
html = self._feed_html(f"{base}/{slug}")
|
|
except SubscribeStarAuthError as exc:
|
|
return False, f"SubscribeStar rejected the credential — {exc}"
|
|
except SubscribeStarAPIError as exc:
|
|
return None, f"Couldn't verify (network/HTTP issue): {exc}"
|
|
if _POST_OPEN in html or 'data-role="posts_container-list"' in html:
|
|
return True, "Credentials valid — the SubscribeStar feed loaded."
|
|
return None, "Couldn't verify — SubscribeStar feed shape unrecognized."
|