FabledCurator/backend/app/services/subscribestar_downloader.py

"""Native SubscribeStar media downloader — the SubscribeStar counterpart to
patreon_downloader.

Given a SubscribeStar post and its resolved `MediaItem`s
(subscribestar_client.extract_media), download the media to gallery-dl's on-disk
layout (so existing gallery-dl downloads are recognized on disk and not
re-fetched at cutover), write the post-first sidecars the importer consumes, and
report per-media outcomes.

Simpler than the Patreon downloader: SubscribeStar serves every upload as a
direct file via `/post_uploads?payload=...` (plain GET — no Mux/HLS, no yt-dlp),
and the full post body is already present in the feed HTML (no detail-endpoint
enrichment). PURE: no DB; the seen-skip is an injected predicate.

On-disk layout (matches gallery-dl's subscribestar config
`{date:%Y-%m-%d}_{id}_{title[:40]}` / `{num:>02}_{filename}`): SubscribeStar posts
have no title, so the directory is `<YYYY-MM-DD>_<post_id>_` — the display title is
synthesized by the importer from the body, so the bare dir name is on-disk only.

FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
"""

from __future__ import annotations

import contextlib
import json
import logging
import os
import time
from collections.abc import Callable
from datetime import datetime
from pathlib import Path

import requests

from .file_validator import is_validatable, quarantine_file, validate_file
from .patreon_downloader import (
    MediaOutcome,
    PostRecordOutcome,
    _sanitize,
)
from .subscribestar_client import _MAX_429_RETRIES, _load_session, _retry_after_seconds

log = logging.getLogger(__name__)

_TITLE_MAX = 40
_TIMEOUT_SECONDS = 120.0
_CHUNK = 1 << 16
_MAX_MEDIA_RETRIES = 3
_BACKOFF_CAP_SECONDS = 30.0
_TRANSIENT_TRANSPORT_EXC = (
    requests.ConnectionError,
    requests.Timeout,
    requests.exceptions.ChunkedEncodingError,
)


def _post_dir_name(post: dict) -> str:
    """`<YYYY-MM-DD>_<post_id>_<title40>` matching gallery-dl's subscribestar
    layout (title is empty for SubscribeStar → trailing underscore). Date prefix
    omitted when published_at is missing/unparseable."""
    post_id = str(post.get("id") or "")
    attrs = post.get("attributes") or {}
    title = attrs.get("title")
    title40 = (title if isinstance(title, str) else "")[:_TITLE_MAX]
    published = attrs.get("published_at")
    date_prefix = None
    if isinstance(published, str) and published:
        try:
            date_prefix = f"{datetime.fromisoformat(published):%Y-%m-%d}"
        except ValueError:
            date_prefix = None
    raw = f"{date_prefix}_{post_id}_{title40}" if date_prefix else f"{post_id}_{title40}"
    return _sanitize(raw)


class SubscribeStarDownloader:
    """Download resolved SubscribeStar media to gallery-dl's on-disk layout. PURE:
    no DB. The HTTP session is an injectable seam (pass `session=` or monkeypatch
    `_fetch_to_file`) so tests run without network."""

    def __init__(
        self,
        images_root: Path,
        cookies_path: str | None = None,
        *,
        validate: bool = True,
        rate_limit: float = 0.0,
        session: requests.Session | None = None,
        max_retries: int = _MAX_429_RETRIES,
    ):
        self.images_root = Path(images_root)
        self.cookies_path = str(cookies_path) if cookies_path else None
        self._validate = validate
        self._rate_limit = rate_limit or 0.0
        self._max_retries = max_retries
        self.session = session if session is not None else _load_session(cookies_path)

    # -- public ------------------------------------------------------------

    def download_post(
        self,
        post: dict,
        media_items: list,
        artist_slug: str,
        *,
        is_seen: Callable[[object], bool] = lambda m: False,
        should_stop: Callable[[], bool] = lambda: False,
        recapture: bool = False,
    ) -> list[MediaOutcome]:
        """Download every media item of one post; return per-item outcomes.
        Mirrors PatreonDownloader.download_post (two-tier skip, mid-post time-box,
        recapture surfacing) minus the video branch."""
        post_dir = self.images_root / artist_slug / "subscribestar" / _post_dir_name(post)
        outcomes: list[MediaOutcome] = []
        for i, media in enumerate(media_items, start=1):
            if should_stop():
                break
            try:
                outcomes.append(
                    self._download_one(
                        post, media, post_dir, artist_slug, i, is_seen,
                        recapture=recapture,
                    )
                )
            except Exception as exc:  # resilient: isolate one item's failure
                log.warning(
                    "SubscribeStar media failed (post %s, item %d): %s",
                    post.get("id"), i, exc,
                )
                outcomes.append(
                    MediaOutcome(media=media, status="error", path=None, error=str(exc))
                )
        return outcomes

    # -- per-item ----------------------------------------------------------

    def _download_one(
        self,
        post: dict,
        media,
        post_dir: Path,
        artist_slug: str,
        index: int,
        is_seen: Callable[[object], bool],
        *,
        recapture: bool = False,
    ) -> MediaOutcome:
        seen = is_seen(media)
        if seen and not recapture:
            return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)

        nn = f"{index:02d}"
        media_path = post_dir / _sanitize(f"{nn}_{media.filename}")

        if media_path.exists():  # tier-2: already on disk
            return MediaOutcome(
                media=media, status="skipped_disk", path=media_path, error=None
            )
        # recapture: a seen item not on disk is NOT re-downloaded (recovery's job).
        if seen:
            return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)

        post_dir.mkdir(parents=True, exist_ok=True)
        if self._rate_limit > 0:
            time.sleep(self._rate_limit)

        out_path = self._fetch_get(media.url, media_path)
        reason, quarantine_dest = self._validate_path(out_path, artist_slug, media.url)
        if reason is not None:
            return MediaOutcome(
                media=media, status="quarantined", path=quarantine_dest, error=reason,
            )
        self._write_sidecar(post, out_path, source_url=media.url)
        return MediaOutcome(media=media, status="downloaded", path=out_path, error=None)

    # -- download seams ----------------------------------------------------

    def _fetch_get(self, url: str, dest: Path) -> Path:
        """Stream `url` to a .part file then atomic-rename to `dest`."""
        part = dest.with_name(dest.name + ".part")
        try:
            self._fetch_to_file(url, part)
        except Exception:
            with contextlib.suppress(OSError):
                part.unlink()
            raise
        os.replace(part, dest)
        return dest

    def _fetch_to_file(self, url: str, dest: Path) -> None:
        """Stream a URL to `dest`, retrying TRANSIENT failures (transport blips,
        429, 5xx) with backoff + resume-from-disk (Range), failing fast on
        permanent 4xx. Mirrors PatreonDownloader._fetch_to_file."""
        attempt = 0
        while True:
            have = dest.stat().st_size if dest.exists() else 0
            headers = {"Range": f"bytes={have}-"} if have > 0 else None
            try:
                resp = self.session.get(
                    url, stream=True, timeout=_TIMEOUT_SECONDS, headers=headers,
                )
                if (resp.status_code == 429 or resp.status_code >= 500) \
                        and attempt < _MAX_MEDIA_RETRIES:
                    attempt += 1
                    delay = _retry_after_seconds(resp, attempt)
                    log.warning(
                        "SubscribeStar media transient HTTP %d (%s) — backing off "
                        "%.1fs (retry %d/%d)",
                        resp.status_code, url, delay, attempt, _MAX_MEDIA_RETRIES,
                    )
                    time.sleep(delay)
                    continue
                if have > 0 and resp.status_code == 416:
                    return  # Range past EOF — we already have the whole file
                resp.raise_for_status()
                mode = "ab" if (have > 0 and resp.status_code == 206) else "wb"
                with open(dest, mode) as fh:
                    for chunk in resp.iter_content(chunk_size=_CHUNK):
                        if chunk:
                            fh.write(chunk)
                return
            except _TRANSIENT_TRANSPORT_EXC as exc:
                if attempt >= _MAX_MEDIA_RETRIES:
                    raise
                attempt += 1
                delay = min(2.0 * (2 ** (attempt - 1)), _BACKOFF_CAP_SECONDS)
                log.warning(
                    "SubscribeStar media transport error (%s) — backing off %.1fs "
                    "(retry %d/%d): %s",
                    url, delay, attempt, _MAX_MEDIA_RETRIES, exc,
                )
                time.sleep(delay)

    # -- validation --------------------------------------------------------

    def _validate_path(
        self, path: Path, artist_slug: str, source_url: str | None = None
    ) -> tuple[str | None, Path | None]:
        """Validate a freshly-written file; quarantine if bad. Mirrors the Patreon
        path (shared file_validator)."""
        if not self._validate or not is_validatable(path):
            return None, None
        try:
            result = validate_file(path)
        except Exception as exc:
            log.warning("Validator raised on %s: %s", path, exc)
            return None, None
        if result.ok:
            return None, None
        dest = quarantine_file(
            self.images_root, path, artist_slug, "subscribestar",
            url=source_url, result=result,
        )
        return (result.reason or "validation failed"), (dest or path)

    # -- sidecar -----------------------------------------------------------

    def _write_sidecar(
        self, post: dict, media_path: Path, *, source_url: str | None = None
    ) -> Path:
        """Per-media sidecar — post-first (#856): image identity ONLY
        (category/id/source_url). The post body/links live solely in _post.json."""
        return self._write_sidecar_data(
            post, media_path.with_suffix(".json"), source_url=source_url, minimal=True,
        )

    def _write_sidecar_data(
        self, post: dict, sidecar_path: Path, *, source_url: str | None = None,
        minimal: bool = False,
    ) -> Path:
        """Serialize the post's metadata. minimal=True → per-media sidecar (image
        identity only); else the full post record (body/title/date/url)."""
        if minimal:
            data = {"category": "subscribestar", "id": str(post.get("id") or "")}
            if source_url:
                data["source_url"] = source_url
            sidecar_path.write_text(json.dumps(data, indent=2))
            return sidecar_path
        attrs = post.get("attributes") or {}
        content = attrs.get("content")
        # SubscribeStar synthesizes the post permalink from the id (matches the
        # platforms/subscribestar.py derive_post_url helper).
        pid = str(post.get("id") or "")
        data = {
            "category": "subscribestar",
            "id": pid,
            "post_id": pid,  # ensures derive_post_url has its key on the native path
            "title": attrs.get("title") if isinstance(attrs.get("title"), str) else "",
            "content": content if isinstance(content, str) else "",
            "published_at": attrs.get("published_at"),
        }
        if source_url:
            data["source_url"] = source_url
        sidecar_path.write_text(json.dumps(data, indent=2))
        return sidecar_path

    def write_post_record(self, post: dict, artist_slug: str) -> PostRecordOutcome:
        """Write the post-first `_post.json` (body/links/metadata) — the sole
        writer of the post record on the native path. SubscribeStar's body is
        already in the feed HTML, so no detail-fetch is needed."""
        attrs = post.get("attributes") or {}
        title = attrs.get("title") if isinstance(attrs.get("title"), str) else None
        post_type = attrs.get("post_type") if isinstance(attrs.get("post_type"), str) else None
        pid = str(post.get("id") or "")
        if not pid:
            return PostRecordOutcome(
                path=None, post_type=post_type, title=title, body_chars=0,
            )
        post_dir = self.images_root / artist_slug / "subscribestar" / _post_dir_name(post)
        post_dir.mkdir(parents=True, exist_ok=True)
        path = self._write_sidecar_data(post, post_dir / "_post.json")
        body = attrs.get("content")
        body_chars = len(body) if isinstance(body, str) else 0
        return PostRecordOutcome(
            path=path, post_type=post_type, title=title, body_chars=body_chars,
        )