`; body lives
in `.post-content .trix-content`; date in `.post-date`; media in a `data-gallery`
JSON manifest on `.uploads-images`.
`campaign_id` for SubscribeStar is the full creator URL (e.g.
https://subscribestar.adult/sabu) — the host (.com vs .adult) and slug both come
from it, so no separate resolver is needed.
Drift is loud on purpose (mirrors patreon_client): an HTML login/age-gate where
the feed was expected is AUTH (rotate cookies); a feed whose post structure we
can't parse at all is DRIFT (the scraper needs updating). FC runs on a
plain-HTTP homelab; nothing here uses a secure-context Web API.
"""
from __future__ import annotations
import json
import logging
import re
import time
from collections.abc import Iterator
from datetime import datetime
from html import unescape
from pathlib import Path
from urllib.parse import urljoin, urlsplit
import requests
from ..utils.paths import filehash_from_url
from .native_ingest_common import (
_MAX_429_RETRIES,
MediaItem,
Membership,
NativeAuthError,
NativeDriftError,
NativeIngestError,
basename_from_url,
make_session,
retry_after_seconds,
)
log = logging.getLogger(__name__)
_TIMEOUT_SECONDS = 30.0
# Match gallery-dl's default (cookies-only) request profile EXACTLY — the proven
# way to read SubscribeStar without tripping its /verify_subscriber gate. In that
# mode gallery-dl's base Extractor._init_session sends a Firefox UA, Accept: */*,
# Accept-Language, and a same-site Referer (root/) on EVERY request — including
# the first creator-page GET — and uses NO X-Requested-With on either the creator
# page or the "load more" JSON endpoint (it GETs and parses the body as JSON).
# Our prior Chrome UA + missing Referer + XHR toggling looked enough unlike a
# browser that SubscribeStar 302'd the adult-creator page to /
/verify_
# subscriber even with valid cookies (cheunart, 2026-06-17).
_FIREFOX_UA = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) "
"Gecko/20100101 Firefox/140.0"
)
_GDL_HEADERS = {
"User-Agent": _FIREFOX_UA,
"Accept-Language": "en-US,en;q=0.5",
}
# A post block opens with this wrapper; we slice the page between consecutive
# occurrences (regex can't match the balanced close of nested divs, so chunk-
# per-post is the robust approach gallery-dl also uses).
#
# Delimiter is the GENERIC ` regex
# either returned empty or over-captured into sibling upload divs AND the
# "View next posts (N / M)" pagination counter (the "264 / 265" body bug,
# cheunart 2026-06-17). The trix editor wraps rich bodies in a full
# `…` document, so strip to the body inner when present.
_CONTENT_OPEN = ''
_CONTENT_CLOSE = '
str:
"""A short, log-safe description of an unexpected page: its
+ which
known interstitial it resembles (bot challenge / age gate / login / captcha)."""
m = re.search(r"<title[^>]*>(.*?)", html, re.IGNORECASE | re.DOTALL)
title = unescape(m.group(1).strip())[:120] if m else "(no )"
low = html.lower()
hits = [name for name, needles in _INTERSTITIAL_MARKERS
if any(n.lower() in low for n in needles)]
return f"title={title!r}; resembles: {'/'.join(hits) if hits else 'unrecognized'}"
class SubscribeStarAPIError(NativeIngestError):
"""Base for native SubscribeStar client failures. status_code / retry_after
are inherited from NativeIngestError."""
class SubscribeStarAuthError(SubscribeStarAPIError, NativeAuthError):
"""Auth/authorization failure — expired cookies, missing age cookie, or an
HTML login/age wall served where the feed was expected. Fix = rotate the
credential, not update the scraper. Maps to error_type 'auth_error'."""
class SubscribeStarDriftError(SubscribeStarAPIError, NativeDriftError):
"""The feed HTML did not match the structure we scrape (no recognizable post
blocks AND no known empty-feed state). The scrape analog of API drift — fail
loud so the import step flags 'SubscribeStar changed its markup' instead of
silently importing nothing."""
def _extr(text: str, start: str, end: str) -> str:
"""Substring between the first `start` and the next `end` after it (gallery-
dl's `text.extr`); '' when either marker is absent."""
i = text.find(start)
if i < 0:
return ""
i += len(start)
j = text.find(end, i)
if j < 0:
return ""
return text[i:j]
def _extract_content(chunk: str) -> str:
"""The post body HTML — gallery-dl's `_data_from_post` content rule: between
the post_content-text wrapper and the youtube-uploads div, with the trix
editor's `<html><body>…</body></html>` wrapper stripped to its inner."""
content = _extr(chunk, _CONTENT_OPEN, _CONTENT_CLOSE)
if "<html><body>" in content:
content = _extr(content, "<body>", "</body>")
return content.strip()
def _attachment_item(
frag: str, base: str, post_id: str, *, kind: str, title_marker: str, url_attr: str
) -> MediaItem | None:
"""One doc/audio attachment from its preview block — gallery-dl's
`_media_from_post` attachment/audio fields. `url_attr` is the URL-bearing
attribute (`href="` for docs, `src="` for audio); `title_marker` precedes the
display name. Returns None when the block carries no URL."""
rel = unescape(_extr(frag, url_attr, '"'))
if not rel:
return None
url = urljoin(base + "/", rel)
upload_id = _extr(frag, 'data-upload-id="', '"')
name = unescape(_extr(frag, title_marker, "<")).strip()
return MediaItem(
url=url,
filename=name or basename_from_url(url),
kind=kind,
filehash=filehash_from_url(url),
post_id=post_id,
media_id=str(upload_id or ""),
)
def _normalize_ss_host(netloc: str) -> str:
"""Rewrite the `subscribestar.art` host to `subscribestar.adult`.
The age wall on the `.art` domain does not clear with the
`18_plus_agreement_generic` cookie (unlike `.com`/`.adult`): a `.art`
creator page keeps 302'ing to `/age_confirmation_warning` even with the
cookie set (Elasid, event #54116). The same creator is reachable on
`.adult`, where the cookie works — so `.art` behaves as an alias that
doesn't honor the age gate. Normalize it to `.adult` at request time (the
stored Source.url is left untouched). `.com`/`.adult` pass through.
"""
host = netloc.lower()
if host == "subscribestar.art" or host.endswith(".subscribestar.art"):
rewritten = netloc[: -len("art")] + "adult"
log.info(
"SubscribeStar: rewrote age-gated .art host %r → %r", netloc, rewritten
)
return rewritten
return netloc
def _split_creator_url(campaign_id: str) -> tuple[str, str]:
"""`campaign_id` is the creator URL → (base, slug).
base = scheme://host (preserving .com vs .adult; .art → .adult, see
_normalize_ss_host); slug = first path segment.
"""
parts = urlsplit(campaign_id)
base = f"{parts.scheme or 'https'}://{_normalize_ss_host(parts.netloc)}"
slug = parts.path.strip("/").split("/")[0] if parts.path else ""
return base, slug
def _parse_ss_datetime(text: str) -> str | None:
"""SubscribeStar renders human dates: 'Jun 17, 2026 03:19 am' (and an
'Updated on <date>' variant). Return ISO-8601 (UTC-naive) or None."""
s = unescape(text or "").strip()
if s.lower().startswith("updated on "):
s = s[len("updated on "):].strip()
for fmt in ("%b %d, %Y %I:%M %p", "%b %d, %Y"):
try:
return datetime.strptime(s, fmt).isoformat()
except ValueError:
continue
return None
_OG_TITLE_RE = re.compile(
r'<meta[^>]+property=["\']og:title["\'][^>]+content=["\']([^"\']+)["\']',
re.IGNORECASE,
)
_TITLE_RE = re.compile(r"<title[^>]*>(.*?)", re.IGNORECASE | re.DOTALL)
# Trailing " | SubscribeStar" / " on SubscribeStar" the profile carries.
_SS_TITLE_SUFFIX_RE = re.compile(
r"\s*[|·]\s*SubscribeStar.*$|\s+on\s+SubscribeStar.*$", re.IGNORECASE
)
def _extract_creator_name(html: str) -> str | None:
"""The creator's display name from a SubscribeStar profile page: prefer the
og:title meta (it's the bare creator name), else the <title> with the
SubscribeStar suffix stripped. None when neither yields anything (#130)."""
m = _OG_TITLE_RE.search(html)
name = unescape(m.group(1)).strip() if m else ""
if not name:
t = _TITLE_RE.search(html)
raw = unescape(t.group(1)).strip() if t else ""
name = _SS_TITLE_SUFFIX_RE.sub("", raw).strip()
return name or None
# -- membership roster (#387 D1) ------------------------------------------
#
# Characterized from a live operator capture of the account's /subscriptions
# page, 2026-09-13 — Scribe note #3989. Read that note before changing any of
# this; each constant below is a finding from it, not a guess.
# The account page is fetched from `.adult`. The `.art` age wall never clears
# with the 18+ cookie for FC's requests (see _normalize_ss_host, issues #1259 /
# #1284). The capture itself came from `.art` only because a human had clicked
# through the gate in the browser.
_ROSTER_BASE = "https://subscribestar.adult"
_ROSTER_URL = f"{_ROSTER_BASE}/subscriptions"
# Two tables, and WHICH table a creator sits in is the only status the page
# gives — there is no per-row status word. Keyed on each card's
# `data-identifier`, the one vocabulary that names a state: the table class
# inside the cancelled card says `for-unsubscribed_users`, a different word for
# the same list (note #3989, CORRECTION 1). The identifier is stored verbatim as
# Membership.status and mapped in native_ingest_common.MEMBERSHIP_STATUS.
_ROSTER_ACTIVE = "active_subscriptions"
_ROSTER_CANCELLED = "cancelled_subscriptions"
_ROSTER_ROW_OPEN = '<td class="for-name">'
# Active rows nest a second `<tr class="for-actions">` INSIDE the row's own
# <tr> — a narrow-screen duplicate of the actions cell. Its <td>s are not
# columns, so every row is cut here before its cells are read.
_ROSTER_NESTED_ROW = '<tr class="for-actions"'
_ROSTER_HREF_RE = re.compile(r'<a href="/([^"/?#]+)"')
_ROSTER_USER_ID_RE = re.compile(r'data-user-id="([^"]*)"')
_ROSTER_NAME_RE = re.compile(r"<img [^>]*>([^<]*)</div>")
_ROSTER_HEAD_RE = re.compile(r'<th class="[^"]*"[^>]*>(.*?)</th>', re.DOTALL)
_ROSTER_CELL_RE = re.compile(r'<td class="[^"]*"[^>]*>(.*?)</td>', re.DOTALL)
_ROSTER_PAGE_LINK_RE = re.compile(r'href="[^"]*[?&]page=\d')
_TAG_RE = re.compile(r"<[^>]+>")
# Columns that hold identity or controls rather than facts about the
# subscription, so they stay out of `details`. Matched on the header's own text,
# lowercased — the page's words, not ours.
_ROSTER_SKIP_COLUMNS = frozenset({"profile", "updates", "actions"})
def _cell_text(fragment: str) -> str:
"""Visible text of a cell: tags dropped, entities decoded, whitespace folded.
Decoding matters here specifically: an active row with no Discord link
renders its cell as the entity `—`, not as an empty cell.
"""
return " ".join(unescape(_TAG_RE.sub(" ", fragment)).split())
def _roster_table(html: str, identifier: str) -> tuple[str, str] | None:
"""One roster card: (its table markup, whatever trails `</table>` inside it).
None when the card is absent. The trailing part is returned rather than
discarded because it is the pagination check: in the characterized page a
card closes the moment its table does.
"""
start = html.find(f'data-identifier="{identifier}"')
if start < 0:
return None
end = html.find("</table>", start)
if end < 0:
raise SubscribeStarDriftError(
f"SubscribeStar roster card {identifier!r} has no table"
)
close = html.find("</div>", end)
trailing = html[end + len("</table>"): close if close >= 0 else len(html)]
return html[start:end], trailing
def _roster_rows(table: str, identifier: str, base: str) -> list[Membership]:
labels = [_cell_text(h).lower() for h in _ROSTER_HEAD_RE.findall(table)]
body = table[table.find("<tbody>"):] if "<tbody>" in table else ""
starts = [m.start() for m in re.finditer(re.escape(_ROSTER_ROW_OPEN), body)]
rows = []
for n, start in enumerate(starts):
row = body[start: starts[n + 1] if n + 1 < len(starts) else len(body)]
row = row.split(_ROSTER_NESTED_ROW, 1)[0]
href = _ROSTER_HREF_RE.search(row)
if href is None:
raise SubscribeStarDriftError(
f"SubscribeStar roster row in {identifier!r} has no creator link"
)
# The creator's numeric id, NOT the slug, is the key (note #3989,
# CORRECTION 2). A slug re-keys when a creator renames; the old row then
# stops appearing, and a disappearance is exactly what reconciliation
# reads as a lapse. The id survives a rename.
user_id = _ROSTER_USER_ID_RE.search(row)
if user_id is None or not user_id.group(1).isdigit():
raise SubscribeStarDriftError(
f"SubscribeStar roster row in {identifier!r} has no numeric "
f"data-user-id — a membership that cannot be attributed to a "
f"creator is not usable"
)
name = _ROSTER_NAME_RE.search(row)
slug = unescape(href.group(1))
cells = _ROSTER_CELL_RE.findall(row)
if len(cells) != len(labels):
# Canary, not a refusal. Identity above does not depend on columns,
# so a shifted column must not fail the whole roster — but it would
# silently mislabel `details` (a price filed under "discord"), so
# say so in the worker log where it is diagnosable.
log.warning(
"SubscribeStar roster %r: %d cells against %d headers — column "
"details may be mislabelled; markup likely changed (note #3989)",
identifier, len(cells), len(labels),
)
rows.append(Membership(
campaign_id=user_id.group(1),
display_name=(_cell_text(name.group(1)) if name else "") or None,
url=f"{base}/{slug}",
vanity=slug,
status=identifier,
# No free-follow concept on this page (#3970 §2: False when a
# platform has none).
is_free_member=False,
# Tier names live behind a per-row modal, not inline. Fetching every
# modal would be N authenticated requests for a field nothing reads.
tier_names=[],
# Deliberately NOT parsed from the price cell: a bare `$` names no
# currency, and a page price is not proven to be the charge (#3970
# finding 4). None keeps "unknown" distinct from zero. The raw text
# is kept in `details`.
amount_cents=None,
currency=None,
details={
# Paired with the header text by POSITION: two columns share the
# `for-date` class, and the updates column's <td> does not carry
# its <th>'s class at all.
"columns": {
label: _cell_text(cell)
for label, cell in zip(labels, cells, strict=False)
if label not in _ROSTER_SKIP_COLUMNS
},
},
))
return rows
def parse_subscriptions_page(html: str, *, base: str = _ROSTER_BASE) -> list[Membership]:
"""Every membership on the account's /subscriptions page.
Refuses rather than guessing, because every conclusion downstream is drawn
from ABSENCE — a roster that comes back short reads as "you cancelled
those". So this raises when:
* the active card is missing — as SubscribeStarAuthError if the page is a
login or age wall (the fix is credentials), otherwise as drift;
* a row has no creator link or no numeric creator id;
* anything renders after a card's table, or the page carries a `page=` link.
Both cards are paginatable (`data-view="app#embed_pagination"`), and the
characterized account was too small to show what pagination looks like —
so possible pagination is treated as a roster FC cannot prove complete.
A missing cancelled card is NOT drift: an account that has never cancelled
plausibly has no such table. A creator present in both tables is reported
once, as active — a current subscription is the fact that matters.
"""
active = _roster_table(html, _ROSTER_ACTIVE)
if active is None:
if any(marker in html for marker in _LOGIN_MARKERS):
raise SubscribeStarAuthError(
"SubscribeStar served a login/age wall instead of the "
"subscriptions page (cookies expired or age cookie missing)"
)
raise SubscribeStarDriftError(
f"SubscribeStar subscriptions page has no {_ROSTER_ACTIVE!r} card "
f"— {_describe_page(html)}"
)
roster_region = html[html.find(f'data-identifier="{_ROSTER_ACTIVE}"'):]
if _ROSTER_PAGE_LINK_RE.search(roster_region):
raise SubscribeStarDriftError(
"SubscribeStar subscriptions page carries a page= link — the roster "
"may be paginated, and FC cannot prove it is complete (note #3989)"
)
memberships: list[Membership] = []
seen: set[str] = set()
for identifier, found in (
(_ROSTER_ACTIVE, active),
(_ROSTER_CANCELLED, _roster_table(html, _ROSTER_CANCELLED)),
):
if found is None:
continue
table, trailing = found
if trailing.strip():
raise SubscribeStarDriftError(
f"SubscribeStar roster card {identifier!r} renders content after "
f"its table — possibly pagination, so the roster cannot be "
f"proven complete (note #3989)"
)
for membership in _roster_rows(table, identifier, base):
if membership.campaign_id in seen:
continue
seen.add(membership.campaign_id)
memberships.append(membership)
return memberships
class SubscribeStarClient:
"""Synchronous SubscribeStar HTML-scrape read client. Construct with a path
to a Netscape cookies.txt (the same file CredentialService.get_cookies_path
materializes, already carrying the age cookie via augment_cookies)."""
def __init__(
self,
cookies_path: str | Path | None,
*,
request_sleep: float = 0.0,
max_retries: int = _MAX_429_RETRIES,
):
self.cookies_path = str(cookies_path) if cookies_path else None
# gallery-dl-parity request profile (Firefox UA, Accept: */*, Accept-
# Language; Referer is stamped per-walk once the creator base is known).
self._session = make_session(
cookies_path, accept="*/*", extra_headers=_GDL_HEADERS
)
self._request_sleep = request_sleep or 0.0
self._max_retries = max_retries
# -- request -----------------------------------------------------------
def _get(self, url: str, *, headers: dict | None = None) -> requests.Response:
if self._request_sleep > 0:
time.sleep(self._request_sleep)
attempt = 0
while True:
try:
resp = self._session.get(url, timeout=_TIMEOUT_SECONDS, headers=headers)
except requests.RequestException as exc:
raise SubscribeStarAPIError(
f"SubscribeStar request failed ({url}): {exc}"
) from exc
if resp.status_code == 429 and attempt < self._max_retries:
attempt += 1
delay = retry_after_seconds(resp, attempt)
log.warning(
"SubscribeStar 429 (%s) — backing off %.1fs (retry %d/%d)",
url, delay, attempt, self._max_retries,
)
time.sleep(delay)
continue
break
# gallery-dl's own gating signal: SubscribeStar 302-redirects an
# unauthenticated / age-unconfirmed request to /verify_subscriber or
# /age_confirmation_warning (lands as a 200 on that URL). Treat it as auth,
# not drift — the fix is to rotate cookies / set the age cookie.
if resp.history and (
"/verify_subscriber" in resp.url
or "/age_confirmation_warning" in resp.url
):
raise SubscribeStarAuthError(
f"SubscribeStar redirected to {resp.url} — auth/age wall "
f"(rotate cookies or set the 18+ age cookie; requested {url})",
status_code=resp.status_code,
)
if resp.status_code in (401, 403):
raise SubscribeStarAuthError(
f"SubscribeStar returned HTTP {resp.status_code} — auth rejected "
f"(cookies expired or tier insufficient; {url})",
status_code=resp.status_code,
)
if resp.status_code != 200:
retry_after = None
if resp.status_code == 429:
hdr = resp.headers.get("Retry-After")
if hdr:
try:
retry_after = float(hdr)
except (TypeError, ValueError):
retry_after = None
raise SubscribeStarAPIError(
f"SubscribeStar returned HTTP {resp.status_code} ({url})",
status_code=resp.status_code,
retry_after=retry_after,
)
return resp
def _feed_html(self, url: str) -> str:
"""Page 1: the creator page (full HTML document)."""
resp = self._get(url)
text = resp.text or ""
if any(m in text for m in _LOGIN_MARKERS) and _POST_OPEN not in text:
raise SubscribeStarAuthError(
f"SubscribeStar served a login/age wall instead of the feed "
f"(cookies expired or age cookie missing; {url})"
)
return text
def _loadmore_html(self, url: str) -> str:
"""Subsequent pages: a plain GET (gallery-dl uses no XHR header) to the
`/posts?...` endpoint, which returns JSON {"html": "..."}."""
resp = self._get(url)
try:
payload = resp.json()
except ValueError as exc:
raise SubscribeStarAuthError(
f"SubscribeStar 'load more' returned non-JSON (session expired?; "
f"{url}): {exc}"
) from exc
html = payload.get("html") if isinstance(payload, dict) else None
return html if isinstance(html, str) else ""
# -- parsing -----------------------------------------------------------
@staticmethod
def _post_chunks(html: str) -> list[str]:
"""Slice the page into one chunk per post (between consecutive wrapper
opens). Regex can't match a post's balanced close, so each chunk runs to
the next post's open (or end of fragment) — enough to scope per-post
field extraction."""
starts = [m.start() for m in re.finditer(re.escape(_POST_OPEN), html)]
chunks = []
for i, start in enumerate(starts):
end = starts[i + 1] if i + 1 < len(starts) else len(html)
chunks.append(html[start:end])
return chunks
def _parse_post(self, chunk: str) -> dict | None:
m = _POST_ID_RE.search(chunk)
if not m:
return None
post_id = m.group(1)
# gallery-dl date method: text up to first '</', then after the last '>'
# — handles both plain and <a>-wrapped dates (see _DATE_OPEN comment).
raw_date = _extr(chunk, _DATE_OPEN, "</").rpartition(">")[2]
published = _parse_ss_datetime(raw_date) if raw_date else None
content = _extract_content(chunk)
return {
"id": post_id,
"attributes": {
# SubscribeStar has no title field; the importer synthesizes a
# display title from the body's first line (sidecar util).
"title": "",
"content": content,
"published_at": published,
"post_type": "subscribestar",
},
# Raw chunk retained so extract_media parses the data-gallery manifest
# and post_is_gated can scan for the locked-teaser marker.
"_html": chunk,
}
def _parse_posts(self, html: str) -> list[dict]:
posts = []
for chunk in self._post_chunks(html):
post = self._parse_post(chunk)
if post is not None:
posts.append(post)
if posts:
dated = sum(1 for p in posts if p["attributes"].get("published_at"))
bodied = sum(1 for p in posts if p["attributes"].get("content"))
log.info(
"SubscribeStar parsed %d posts (%d dated, %d with body)",
len(posts), dated, bodied,
)
# Canary for this exact failure class: posts parsed but NONE got a
# date or a body, while the raw markers ARE present → our extraction
# diverged from the live markup (e.g. the <a>-wrapped date bug). Log
# the marker counts so the cause is diagnosable from the worker log
# alone, without re-fetching the authed page.
if dated == 0 or bodied == 0:
log.warning(
"SubscribeStar parse canary: %d posts but dated=%d bodied=%d; "
"raw markers post-date=%d post_content-text=%d data-gallery=%d "
"— extraction likely diverged from the live markup",
len(posts), dated, bodied,
html.count('class="post-date"'),
html.count("post_content-text"),
html.count("data-gallery"),
)
return posts
@staticmethod
def _next_page_href(html: str) -> str | None:
m = _NEXT_PAGE_RE.search(html)
return unescape(m.group(1)) if m else None
def extract_media(self, post: dict, included_index: dict) -> list[MediaItem]:
"""Resolve downloadable media (gallery-dl's `_media_from_post`): the
per-post `data-gallery` JSON manifest (images/videos), PLUS document
attachments (`uploads-docs`) and audio (`uploads-audios`) — some posts
deliver content only through the latter two. `/previews` (locked teaser)
gallery items are skipped. `included_index` is unused (media is inline)."""
chunk = post.get("_html") or ""
base = post.get("_base") or "https://www.subscribestar.com"
post_id = str(post.get("id") or "")
items: list[MediaItem] = []
for gm in _GALLERY_RE.finditer(chunk):
try:
gallery = json.loads(unescape(gm.group(1)))
except (ValueError, TypeError):
continue
if not isinstance(gallery, list):
continue
for it in gallery:
if not isinstance(it, dict):
continue
rel = it.get("url")
if not isinstance(rel, str) or not rel:
continue
# gallery-dl's _media_from_post: a gallery item whose URL is under
# /previews is a locked/blurred TEASER, not the real file — skip it
# (the SubscribeStar analog of the Patreon gated-preview bug #874).
# This is why a locked post yields no downloadable media.
if "/previews" in rel:
continue
url = urljoin(base + "/", rel)
media_id = str(it.get("id") or "")
name = it.get("original_filename")
filename = name if isinstance(name, str) and name else basename_from_url(url)
items.append(
MediaItem(
url=url,
filename=filename,
kind=str(it.get("type") or "image"),
filehash=filehash_from_url(url),
post_id=post_id,
media_id=media_id,
)
)
# Document attachments (uploads-docs → doc_preview blocks): href URL.
docs = _extr(chunk, 'class="uploads-docs"', 'class="post-edit_form"')
for frag in _DOC_SPLIT_RE.split(docs)[1:]:
item = _attachment_item(
frag, base, post_id, kind="attachment",
title_marker='doc_preview-title">', url_attr='href="',
)
if item is not None:
items.append(item)
# Audio attachments (uploads-audios → audio_preview-data blocks): src URL.
audios = _extr(chunk, 'class="uploads-audios"', 'class="post-edit_form"')
for frag in _AUDIO_SPLIT_RE.split(audios)[1:]:
item = _attachment_item(
frag, base, post_id, kind="audio",
title_marker='audio_preview-title">', url_attr='src="',
)
if item is not None:
items.append(item)
return items
@staticmethod
def post_meta(post: dict) -> dict:
"""Title + date. Title is None — SubscribeStar has no title field, and
the importer synthesizes one from the body.
`date` is the contract the core's REVISIT WINDOW reads (2026-09-23):
ISO-8601 or None. NAIVE here, because `_parse_ss_datetime` renders a
parsed local timestamp with no zone; the core reads a naive date as UTC
rather than discarding it, since a date it refuses to read is a post
the window can never reach.
"""
attrs = post.get("attributes") or {}
return {"title": None, "date": attrs.get("published_at")}
@staticmethod
def post_is_gated(post: dict) -> bool:
"""True when the subscriber cannot view this post (locked teaser). #874
"no stub for gated content": the core skips it ENTIRELY (no media, no
post-record), so we don't replicate a hollow teaser in Curator.
SubscribeStar does NOT expose downloadable preview media for locked posts
(no `data-gallery`), so the Patreon "downloaded blurred junk" failure
can't happen here — this gate only prevents capturing an empty teaser
stub. Detect the locked marker conservatively (default to NOT gated when
absent, so we never over-filter an accessible text post). The exact marker
is being confirmed against a live locked sample; until then we gate only on
the explicit lock-overlay class SubscribeStar renders on a paywalled post.
"""
chunk = post.get("_html") or ""
return 'class="post-content-locked"' in chunk or "for-locked_content" in chunk
@staticmethod
def post_record_key(post: dict) -> tuple[str, str] | None:
"""`(ledger_key, post_id)` for a post's seen-ledger entry (the `post:<id>`
synthetic key gates post-record capture through the same ledger as media),
or None when the post has no id."""
pid = post.get("id")
pid = str(pid) if pid is not None else ""
if not pid:
return None
return (f"post:{pid}", pid)
# -- iteration ---------------------------------------------------------
def iter_posts(
self, campaign_id: str, cursor: str | None = None
) -> Iterator[tuple[dict, dict, str | None]]:
"""Yield (post, {}, page_cursor) for every post in the feed.
`campaign_id` is the creator URL. `cursor` is the relative "load more"
href that fetches a page (None → page 1, the creator page HTML). The
yielded `page_cursor` is the href that FETCHED this post's page, so the
core checkpoints a value that re-fetches the same page on resume (matching
the Patreon cursor contract).
"""
base, slug = _split_creator_url(campaign_id)
if not slug:
raise SubscribeStarDriftError(
f"Could not extract a creator slug from {campaign_id!r}"
)
# Same-site Referer (gallery-dl sends root/ on every request).
self._session.headers["Referer"] = f"{base}/"
current = cursor
first_page = True
while True:
page_cursor = current
if current is None:
html = self._feed_html(f"{base}/{slug}")
else:
html = self._loadmore_html(urljoin(base + "/", current))
posts = self._parse_posts(html)
if first_page and not posts and _POST_OPEN not in html:
# Page 1 with no recognizable post wrappers at all: either a brand
# new creator with zero posts, or our scraper is stale. The core's
# body canary catches systematic emptiness across a populated feed;
# here we only raise if the feed container itself is missing.
if 'data-role="posts_container-list"' not in html:
# Report what we actually got — length + the page's identity —
# so a served interstitial (bot challenge / age gate / login)
# is named instead of mislabeled "markup changed".
looks_json = html.lstrip()[:1] in ("{", "[")
raise SubscribeStarDriftError(
f"SubscribeStar feed for {slug!r} had no posts and no "
f"recognizable feed container ({len(html)} bytes"
f"{', looks like JSON' if looks_json else ''}; "
f"{_describe_page(html)})"
)
for post in posts:
post["_base"] = base
yield post, {}, page_cursor
next_href = self._next_page_href(html)
if not next_href:
return
current = next_href
first_page = False
# -- display name -------------------------------------------------------
def resolve_display_name(self, campaign_id: str) -> str | None:
"""The creator's display name from their profile page, used to name the
Artist at add-time (#130). `campaign_id` is the creator URL. None on any
failure — the caller falls back to the URL handle. Sync: run in an
executor."""
base, slug = _split_creator_url(campaign_id)
if not slug:
return None
self._session.headers["Referer"] = f"{base}/"
try:
html = self._feed_html(f"{base}/{slug}")
except SubscribeStarAPIError:
return None
return _extract_creator_name(html)
# -- membership roster (#387 D1) ----------------------------------------
def iter_memberships(self, user_id: str | None = None) -> Iterator[Membership]:
"""Yield every subscription the account holds (note #3989).
`user_id` exists for the seam's signature (note #3970) and is ignored:
the page is the logged-in account's own, so there is nothing to resolve.
The sweep only resolves an id for a client that exposes
`current_user_id`, which this one does not.
One request, and the whole page is parsed before anything is yielded, so
a drift error can never leave a caller holding part of a roster.
"""
self._session.headers["Referer"] = f"{_ROSTER_BASE}/"
resp = self._get(_ROSTER_URL)
yield from parse_subscriptions_page(resp.text or "", base=_ROSTER_BASE)
# -- verify ------------------------------------------------------------
def verify_auth(self, campaign_id: str) -> tuple[bool | None, str]:
"""Cheap auth probe: fetch the first feed page and report whether the
credential authenticated, without downloading anything."""
base, slug = _split_creator_url(campaign_id)
if not slug:
return None, f"Couldn't parse a SubscribeStar creator from {campaign_id!r}"
self._session.headers["Referer"] = f"{base}/"
try:
html = self._feed_html(f"{base}/{slug}")
except SubscribeStarAuthError as exc:
return False, f"SubscribeStar rejected the credential — {exc}"
except SubscribeStarAPIError as exc:
return None, f"Couldn't verify (network/HTTP issue): {exc}"
if _POST_OPEN in html or 'data-role="posts_container-list"' in html:
return True, "Credentials valid — the SubscribeStar feed loaded."
return None, "Couldn't verify — SubscribeStar feed shape unrecognized."