FabledCurator/backend/app/services/subscribestar_client.py

"""Native SubscribeStar client — the SubscribeStar adapter's read path.

Unlike Patreon (a clean JSON:API), SubscribeStar has NO public API: gallery-dl
scrapes its HTML, and so do we. The platform-agnostic core (`ingest_core`) only
calls `client.iter_posts` / `extract_media` / the optional seams, so the
HTML-scrape divergence is contained entirely to this module.

Feed shape (characterized from a live sample 2026-06-17 — Scribe note
"SubscribeStar HTML characterization"):
  - Page 1 is the creator page HTML: GET <base>/<slug>.
  - Each page embeds `data-role="infinite_scroll-next_page" href="/posts?...&page=N
    &slug=<slug>&sort_by=newest"`. That path returns JSON {"html": "<...posts...>"};
    the fragment carries the NEXT page link, until it runs out.
  - A post is `<div class="post is-shown ..." data-id="<post_id>" ...>`; body lives
    in `.post-content .trix-content`; date in `.post-date`; media in a `data-gallery`
    JSON manifest on `.uploads-images`.

`campaign_id` for SubscribeStar is the full creator URL (e.g.
https://subscribestar.adult/sabu) — the host (.com vs .adult) and slug both come
from it, so no separate resolver is needed.

Drift is loud on purpose (mirrors patreon_client): an HTML login/age-gate where
the feed was expected is AUTH (rotate cookies); a feed whose post structure we
can't parse at all is DRIFT (the scraper needs updating). FC runs on a
plain-HTTP homelab; nothing here uses a secure-context Web API.
"""

from __future__ import annotations

import json
import logging
import re
import time
from collections.abc import Iterator
from dataclasses import dataclass
from datetime import datetime
from html import unescape
from pathlib import Path
from urllib.parse import urljoin, urlsplit

import requests

from ..utils.paths import filehash_from_url
from .native_ingest_common import (
    _MAX_429_RETRIES,
    NativeAuthError,
    NativeDriftError,
    NativeIngestError,
    basename_from_url,
    make_session,
    retry_after_seconds,
)

log = logging.getLogger(__name__)

_TIMEOUT_SECONDS = 30.0
# The INITIAL creator-page GET is a browser-like NAVIGATION (Accept: html, NO
# X-Requested-With). SubscribeStar is Rails: an XHR-flagged request to the full
# page content-negotiates to a non-HTML response with no posts_container — which
# tripped the drift guard on the first live run (cheunart, 2026-06-17). The
# "load more" endpoint IS a real XHR and gets these headers per-request.
_NAV_ACCEPT = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
_LOADMORE_HEADERS = {
    "X-Requested-With": "XMLHttpRequest",
    "Accept": "application/json, text/javascript, */*; q=0.01",
}

# A post block opens with this wrapper; we slice the page between consecutive
# occurrences (regex can't match the balanced close of nested divs, so chunk-
# per-post is the robust approach gallery-dl also uses).
#
# Delimiter is the GENERIC `<div class="post ` (trailing space — matches the post
# container regardless of the classes that follow), exactly what gallery-dl's
# `_pagination` splits on. We previously keyed on `<div class="post is-shown`,
# but `is-shown` is added by SubscribeStar's infinite-scroll JS when a post
# scrolls into view — it's present in a browser-SAVED page but ABSENT from the
# raw server HTML we (and gallery-dl) actually fetch, so the raw feed parsed to
# zero posts → false drift (cheunart, 2026-06-17). The space rules out the
# hyphenated siblings (`post-content`/`post-date`/`post-body`/`post-uploads`).
_POST_OPEN = '<div class="post '
_POST_ID_RE = re.compile(r'data-id="(\d+)"')
_POST_DATE_RE = re.compile(r'<div class="post-date">([^<]+)</div>')
# The body: inner of `.post-content .trix-content`. Non-greedy to the LAST
# </div></div> in the chunk would over-capture; instead capture from trix-content
# to the close of the post-content wrapper, which the post-body block ends with.
_BODY_RE = re.compile(
    r'<div class="post-content"[^>]*>(.*?)</div>\s*</div>\s*</div>', re.DOTALL
)
_GALLERY_RE = re.compile(r'data-gallery="([^"]*)"')
_NEXT_PAGE_RE = re.compile(
    r'data-role="infinite_scroll-next_page"\s+href="([^"]+)"'
)
# Signals an auth/age wall served in place of the feed (cookies expired or the
# age cookie missing) rather than a real — possibly empty — creator feed.
_LOGIN_MARKERS = ("/session/new", 'data-role="sign_in"', "age_confirmation_warning")

# An interstitial served INSTEAD of the feed — characterized so a drift error
# reports the actual cause (bot challenge / age gate / login) rather than a bare
# "markup changed". (name, substrings to look for, case-insensitive.)
_INTERSTITIAL_MARKERS = (
    ("cloudflare/bot-challenge", (
        "just a moment", "cf-challenge", "challenge-platform", "cf_chl",
        "attention required", "enable javascript and cookies",
    )),
    ("age-gate", (
        "18 or older", "adult content", "age_confirmation", "i am over",
        "confirm your age", "must be 18",
    )),
    ("login", ("/session/new", 'data-role="sign_in"', "sign in", "log in")),
    ("captcha", ("g-recaptcha", "hcaptcha", "captcha")),
)


def _describe_page(html: str) -> str:
    """A short, log-safe description of an unexpected page: its <title> + which
    known interstitial it resembles (bot challenge / age gate / login / captcha)."""
    m = re.search(r"<title[^>]*>(.*?)</title>", html, re.IGNORECASE | re.DOTALL)
    title = unescape(m.group(1).strip())[:120] if m else "(no <title>)"
    low = html.lower()
    hits = [name for name, needles in _INTERSTITIAL_MARKERS
            if any(n.lower() in low for n in needles)]
    return f"title={title!r}; resembles: {'/'.join(hits) if hits else 'unrecognized'}"


class SubscribeStarAPIError(NativeIngestError):
    """Base for native SubscribeStar client failures. status_code / retry_after
    are inherited from NativeIngestError."""


class SubscribeStarAuthError(SubscribeStarAPIError, NativeAuthError):
    """Auth/authorization failure — expired cookies, missing age cookie, or an
    HTML login/age wall served where the feed was expected. Fix = rotate the
    credential, not update the scraper. Maps to error_type 'auth_error'."""


class SubscribeStarDriftError(SubscribeStarAPIError, NativeDriftError):
    """The feed HTML did not match the structure we scrape (no recognizable post
    blocks AND no known empty-feed state). The scrape analog of API drift — fail
    loud so the import step flags 'SubscribeStar changed its markup' instead of
    silently importing nothing."""


@dataclass
class MediaItem:
    """One resolved downloadable item belonging to a SubscribeStar post.

    Fields mirror patreon_client.MediaItem (so the downloader is structurally the
    same) plus `media_id` — the stable per-upload gallery id. SubscribeStar's
    full-res URL is an opaque `/post_uploads?payload=...` (not content-addressed),
    so `filehash` is usually None and the ledger keys on `<post_id>:<media_id>`.
    """

    url: str
    filename: str
    kind: str
    filehash: str | None
    post_id: str
    media_id: str


def _split_creator_url(campaign_id: str) -> tuple[str, str]:
    """`campaign_id` is the creator URL → (base, slug).

    base = scheme://host (preserving .com vs .adult); slug = first path segment.
    """
    parts = urlsplit(campaign_id)
    base = f"{parts.scheme or 'https'}://{parts.netloc}"
    slug = parts.path.strip("/").split("/")[0] if parts.path else ""
    return base, slug


def _parse_ss_datetime(text: str) -> str | None:
    """SubscribeStar renders human dates: 'Jun 17, 2026 03:19 am' (and an
    'Updated on <date>' variant). Return ISO-8601 (UTC-naive) or None."""
    s = unescape(text or "").strip()
    if s.lower().startswith("updated on "):
        s = s[len("updated on "):].strip()
    for fmt in ("%b %d, %Y %I:%M %p", "%b %d, %Y"):
        try:
            return datetime.strptime(s, fmt).isoformat()
        except ValueError:
            continue
    return None


class SubscribeStarClient:
    """Synchronous SubscribeStar HTML-scrape read client. Construct with a path
    to a Netscape cookies.txt (the same file CredentialService.get_cookies_path
    materializes, already carrying the age cookie via augment_cookies)."""

    def __init__(
        self,
        cookies_path: str | Path | None,
        *,
        request_sleep: float = 0.0,
        max_retries: int = _MAX_429_RETRIES,
    ):
        self.cookies_path = str(cookies_path) if cookies_path else None
        # Browser-like navigation defaults; the load-more XHR headers are applied
        # per-request (NOT session-wide) so the initial page GET isn't flagged XHR.
        self._session = make_session(cookies_path, accept=_NAV_ACCEPT)
        self._request_sleep = request_sleep or 0.0
        self._max_retries = max_retries

    # -- request -----------------------------------------------------------

    def _get(self, url: str, *, headers: dict | None = None) -> requests.Response:
        if self._request_sleep > 0:
            time.sleep(self._request_sleep)
        attempt = 0
        while True:
            try:
                resp = self._session.get(url, timeout=_TIMEOUT_SECONDS, headers=headers)
            except requests.RequestException as exc:
                raise SubscribeStarAPIError(
                    f"SubscribeStar request failed ({url}): {exc}"
                ) from exc
            if resp.status_code == 429 and attempt < self._max_retries:
                attempt += 1
                delay = retry_after_seconds(resp, attempt)
                log.warning(
                    "SubscribeStar 429 (%s) — backing off %.1fs (retry %d/%d)",
                    url, delay, attempt, self._max_retries,
                )
                time.sleep(delay)
                continue
            break
        # gallery-dl's own gating signal: SubscribeStar 302-redirects an
        # unauthenticated / age-unconfirmed request to /verify_subscriber or
        # /age_confirmation_warning (lands as a 200 on that URL). Treat it as auth,
        # not drift — the fix is to rotate cookies / set the age cookie.
        if resp.history and (
            "/verify_subscriber" in resp.url
            or "/age_confirmation_warning" in resp.url
        ):
            raise SubscribeStarAuthError(
                f"SubscribeStar redirected to {resp.url} — auth/age wall "
                f"(rotate cookies or set the 18+ age cookie; requested {url})",
                status_code=resp.status_code,
            )
        if resp.status_code in (401, 403):
            raise SubscribeStarAuthError(
                f"SubscribeStar returned HTTP {resp.status_code} — auth rejected "
                f"(cookies expired or tier insufficient; {url})",
                status_code=resp.status_code,
            )
        if resp.status_code != 200:
            retry_after = None
            if resp.status_code == 429:
                hdr = resp.headers.get("Retry-After")
                if hdr:
                    try:
                        retry_after = float(hdr)
                    except (TypeError, ValueError):
                        retry_after = None
            raise SubscribeStarAPIError(
                f"SubscribeStar returned HTTP {resp.status_code} ({url})",
                status_code=resp.status_code,
                retry_after=retry_after,
            )
        return resp

    def _feed_html(self, url: str) -> str:
        """Page 1: the creator page (full HTML document)."""
        resp = self._get(url)
        text = resp.text or ""
        if any(m in text for m in _LOGIN_MARKERS) and _POST_OPEN not in text:
            raise SubscribeStarAuthError(
                f"SubscribeStar served a login/age wall instead of the feed "
                f"(cookies expired or age cookie missing; {url})"
            )
        return text

    def _loadmore_html(self, url: str) -> str:
        """Subsequent pages: a real XHR GET that returns JSON {"html": "..."}."""
        resp = self._get(url, headers=_LOADMORE_HEADERS)
        try:
            payload = resp.json()
        except ValueError as exc:
            raise SubscribeStarAuthError(
                f"SubscribeStar 'load more' returned non-JSON (session expired?; "
                f"{url}): {exc}"
            ) from exc
        html = payload.get("html") if isinstance(payload, dict) else None
        return html if isinstance(html, str) else ""

    # -- parsing -----------------------------------------------------------

    @staticmethod
    def _post_chunks(html: str) -> list[str]:
        """Slice the page into one chunk per post (between consecutive wrapper
        opens). Regex can't match a post's balanced close, so each chunk runs to
        the next post's open (or end of fragment) — enough to scope per-post
        field extraction."""
        starts = [m.start() for m in re.finditer(re.escape(_POST_OPEN), html)]
        chunks = []
        for i, start in enumerate(starts):
            end = starts[i + 1] if i + 1 < len(starts) else len(html)
            chunks.append(html[start:end])
        return chunks

    def _parse_post(self, chunk: str) -> dict | None:
        m = _POST_ID_RE.search(chunk)
        if not m:
            return None
        post_id = m.group(1)
        date_m = _POST_DATE_RE.search(chunk)
        published = _parse_ss_datetime(date_m.group(1)) if date_m else None
        body_m = _BODY_RE.search(chunk)
        content = body_m.group(1).strip() if body_m else ""
        return {
            "id": post_id,
            "attributes": {
                # SubscribeStar has no title field; the importer synthesizes a
                # display title from the body's first line (sidecar util).
                "title": "",
                "content": content,
                "published_at": published,
                "post_type": "subscribestar",
            },
            # Raw chunk retained so extract_media parses the data-gallery manifest
            # and post_is_gated can scan for the locked-teaser marker.
            "_html": chunk,
        }

    def _parse_posts(self, html: str) -> list[dict]:
        posts = []
        for chunk in self._post_chunks(html):
            post = self._parse_post(chunk)
            if post is not None:
                posts.append(post)
        return posts

    @staticmethod
    def _next_page_href(html: str) -> str | None:
        m = _NEXT_PAGE_RE.search(html)
        return unescape(m.group(1)) if m else None

    def extract_media(self, post: dict, included_index: dict) -> list[MediaItem]:
        """Resolve downloadable media from the post's `data-gallery` JSON
        manifest. Each item carries a stable `id`, `original_filename`, `type`,
        and a full-res `url` (`/post_uploads?payload=...`, relative — joined to the
        post's base). `included_index` is unused (HTML carries media inline)."""
        chunk = post.get("_html") or ""
        base = post.get("_base") or "https://www.subscribestar.com"
        post_id = str(post.get("id") or "")
        items: list[MediaItem] = []
        for gm in _GALLERY_RE.finditer(chunk):
            try:
                gallery = json.loads(unescape(gm.group(1)))
            except (ValueError, TypeError):
                continue
            if not isinstance(gallery, list):
                continue
            for it in gallery:
                if not isinstance(it, dict):
                    continue
                rel = it.get("url")
                if not isinstance(rel, str) or not rel:
                    continue
                url = urljoin(base + "/", rel)
                media_id = str(it.get("id") or "")
                name = it.get("original_filename")
                filename = name if isinstance(name, str) and name else basename_from_url(url)
                items.append(
                    MediaItem(
                        url=url,
                        filename=filename,
                        kind=str(it.get("type") or "image"),
                        filehash=filehash_from_url(url),
                        post_id=post_id,
                        media_id=media_id,
                    )
                )
        return items

    @staticmethod
    def post_meta(post: dict) -> dict:
        """Title + date for the preview sample. Title is synthesized from the body
        (SubscribeStar has no title field)."""
        attrs = post.get("attributes") or {}
        return {"title": None, "date": attrs.get("published_at")}

    @staticmethod
    def post_is_gated(post: dict) -> bool:
        """True when the subscriber cannot view this post (locked teaser). #874
        "no stub for gated content": the core skips it ENTIRELY (no media, no
        post-record), so we don't replicate a hollow teaser in Curator.

        SubscribeStar does NOT expose downloadable preview media for locked posts
        (no `data-gallery`), so the Patreon "downloaded blurred junk" failure
        can't happen here — this gate only prevents capturing an empty teaser
        stub. Detect the locked marker conservatively (default to NOT gated when
        absent, so we never over-filter an accessible text post). The exact marker
        is being confirmed against a live locked sample; until then we gate only on
        the explicit lock-overlay class SubscribeStar renders on a paywalled post.
        """
        chunk = post.get("_html") or ""
        return 'class="post-content-locked"' in chunk or "for-locked_content" in chunk

    @staticmethod
    def post_record_key(post: dict) -> tuple[str, str] | None:
        """`(ledger_key, post_id)` for a post's seen-ledger entry (the `post:<id>`
        synthetic key gates post-record capture through the same ledger as media),
        or None when the post has no id."""
        pid = post.get("id")
        pid = str(pid) if pid is not None else ""
        if not pid:
            return None
        return (f"post:{pid}", pid)

    # -- iteration ---------------------------------------------------------

    def iter_posts(
        self, campaign_id: str, cursor: str | None = None
    ) -> Iterator[tuple[dict, dict, str | None]]:
        """Yield (post, {}, page_cursor) for every post in the feed.

        `campaign_id` is the creator URL. `cursor` is the relative "load more"
        href that fetches a page (None → page 1, the creator page HTML). The
        yielded `page_cursor` is the href that FETCHED this post's page, so the
        core checkpoints a value that re-fetches the same page on resume (matching
        the Patreon cursor contract).
        """
        base, slug = _split_creator_url(campaign_id)
        if not slug:
            raise SubscribeStarDriftError(
                f"Could not extract a creator slug from {campaign_id!r}"
            )
        current = cursor
        first_page = True
        while True:
            page_cursor = current
            if current is None:
                html = self._feed_html(f"{base}/{slug}")
            else:
                html = self._loadmore_html(urljoin(base + "/", current))
            posts = self._parse_posts(html)
            if first_page and not posts and _POST_OPEN not in html:
                # Page 1 with no recognizable post wrappers at all: either a brand
                # new creator with zero posts, or our scraper is stale. The core's
                # body canary catches systematic emptiness across a populated feed;
                # here we only raise if the feed container itself is missing.
                if 'data-role="posts_container-list"' not in html:
                    # Report what we actually got — length + the page's identity —
                    # so a served interstitial (bot challenge / age gate / login)
                    # is named instead of mislabeled "markup changed".
                    looks_json = html.lstrip()[:1] in ("{", "[")
                    raise SubscribeStarDriftError(
                        f"SubscribeStar feed for {slug!r} had no posts and no "
                        f"recognizable feed container ({len(html)} bytes"
                        f"{', looks like JSON' if looks_json else ''}; "
                        f"{_describe_page(html)})"
                    )
            for post in posts:
                post["_base"] = base
                yield post, {}, page_cursor
            next_href = self._next_page_href(html)
            if not next_href:
                return
            current = next_href
            first_page = False

    # -- verify ------------------------------------------------------------

    def verify_auth(self, campaign_id: str) -> tuple[bool | None, str]:
        """Cheap auth probe: fetch the first feed page and report whether the
        credential authenticated, without downloading anything."""
        base, slug = _split_creator_url(campaign_id)
        if not slug:
            return None, f"Couldn't parse a SubscribeStar creator from {campaign_id!r}"
        try:
            html = self._feed_html(f"{base}/{slug}")
        except SubscribeStarAuthError as exc:
            return False, f"SubscribeStar rejected the credential — {exc}"
        except SubscribeStarAPIError as exc:
            return None, f"Couldn't verify (network/HTTP issue): {exc}"
        if _POST_OPEN in html or 'data-role="posts_container-list"' in html:
            return True, "Credentials valid — the SubscribeStar feed loaded."
        return None, "Couldn't verify — SubscribeStar feed shape unrecognized."