FabledCurator/backend/app/services/patreon_downloader.py

"""Native Patreon media downloader (build step 2b of the native ingester).

Given a Patreon post and its already-resolved `MediaItem`s (from
patreon_client.extract_media), download the media to the EXACT on-disk layout
gallery-dl produces, write a sidecar JSON the existing importer consumes, and
report per-media outcomes.

This module is PURE: no DB. The cross-run seen-ledger and the iter_posts
orchestration are a LATER step. The tier-1 (seen) skip is an INJECTED predicate
(`is_seen`), so this module needs no DB and is unit-testable without network.

On-disk layout (matches gallery-dl):
  <images_root>/<artist_slug>/patreon/<DIR>/<NN>_<filename>
  where <DIR> = "<YYYY-MM-DD>_<post_id>_<title40>" (date prefix omitted when the
  post's published_at is unparseable), <NN> is the 1-based index of the item in
  the post zero-padded to 2, and <filename> is MediaItem.filename. The sidecar
  is written next to the media as <NN>_<stem>.json (media_path.with_suffix).

Video: a MediaItem whose URL is a Mux/HLS stream (host stream.mux.com or path
endswith .m3u8) is fetched with yt-dlp (subprocess), passing the same
Referer/Origin headers gallery-dl forwards for Mux playback (see gallery_dl.py
_get_default_config). yt-dlp may remux to a container of its own choosing, so we
accept the actual output extension and record the real path.

FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
"""

from __future__ import annotations

import json
import logging
import os
import subprocess
import time
from collections.abc import Callable
from pathlib import Path
from urllib.parse import urlsplit

import requests

from ..utils.prosemirror import post_body_html
from .native_ingest_common import (
    _BACKOFF_CAP_SECONDS,
    _MAX_MEDIA_RETRIES,
    BaseNativeDownloader,
    MediaItem,
    MediaOutcome,
    PostRecordOutcome,
    post_dir_name,
)

log = logging.getLogger(__name__)

# yt-dlp subprocess wall-clock per attempt (video only; the shared HTTP fetch
# budgets live in BaseNativeDownloader).
_TIMEOUT_SECONDS = 120.0

# Referer/Origin yt-dlp must send for Mux-hosted Patreon video. Mux's JWT
# playback policy checks Referer/Origin on every request, so yt-dlp must send
# Patreon's, not its own default. (gallery-dl forwarded the same headers before
# the #697 cutover removed its Patreon path; this is now the only place they
# live.)
_VIDEO_HEADERS = {
    "Referer": "https://www.patreon.com/",
    "Origin": "https://www.patreon.com",
}


def _is_video_url(url: str) -> bool:
    parts = urlsplit(url)
    if parts.hostname and parts.hostname.lower() == "stream.mux.com":
        return True
    return parts.path.lower().endswith(".m3u8")


class PatreonDownloader(BaseNativeDownloader):
    """Download resolved Patreon media to gallery-dl's on-disk layout. Subclasses
    BaseNativeDownloader for the shared streaming GET (transient-retry +
    Range-resume) and validation/quarantine; adds the Mux/HLS yt-dlp video branch
    and the detail-fetch body enrichment. PURE: no DB. `_run_ytdlp` is
    monkeypatchable and the HTTP session is the injectable `session=` seam.
    """

    def __init__(
        self,
        images_root: Path,
        cookies_path: str | None = None,
        *,
        validate: bool = True,
        rate_limit: float = 0.0,
        session: requests.Session | None = None,
        content_fetcher: Callable[[str], str | None] | None = None,
    ):
        super().__init__(
            images_root, cookies_path, platform="patreon",
            validate=validate, rate_limit=rate_limit, session=session,
        )
        # Best-effort enrichment seam: (post_id) -> full HTML body, or None. The
        # feed endpoint often omits `content`; the adapter wires this to
        # PatreonClient.fetch_post_detail_content so the sidecar captures the
        # real body (formatting + inline <img> + external <a href> links).
        # None in unit tests / when enrichment isn't wanted.
        self._content_fetcher = content_fetcher

    # The download loop, layout and per-file sidecar are BaseNativeDownloader's.
    # Patreon adds one branch: a Mux/HLS video is fetched with yt-dlp, which may
    # remux it to a container of its own choosing.

    def _existing_output(self, path: Path, media: MediaItem) -> Path | None:
        if path.exists():
            return path
        if _is_video_url(media.url):
            # A remux lands at a different extension; honour one already there.
            return self._existing_video_output(path)
        return None

    def _fetch_media(
        self, post: dict, media: MediaItem, path: Path, artist_slug: str,
    ) -> MediaOutcome:
        if not _is_video_url(media.url):
            return super()._fetch_media(post, media, path, artist_slug)
        out_path = self._run_ytdlp(media.url, path, _VIDEO_HEADERS)
        if out_path is None or not Path(out_path).exists():
            return MediaOutcome(
                media=media, status="error", path=None, error="yt-dlp produced no output",
            )
        out_path = Path(out_path)
        self._write_media_sidecar(post, out_path, media)
        return MediaOutcome(media=media, status="downloaded", path=out_path, error=None)

    # -- video (Mux/HLS via yt-dlp) ----------------------------------------

    def _run_ytdlp(self, url: str, dest: Path, headers: dict) -> Path | None:
        """Invoke yt-dlp to fetch a Mux/HLS stream to (around) `dest`.

        Returns the actual output path (yt-dlp may remux to a different
        container, so we resolve the real file afterward). Overridden/
        monkeypatched in tests to avoid spawning a real subprocess.

        The output template uses `dest` without its extension; yt-dlp appends
        the chosen container extension. We pass Referer/Origin (Mux JWT policy)
        and the cookies file.

        Mirrors the GET path's transient/permanent split (plan #705 #8): a hung
        fetch (TimeoutExpired) or a spawn failure (OSError) is TRANSIENT — back
        off and retry. A non-zero yt-dlp exit (CalledProcessError) is treated as
        PERMANENT for this pass — yt-dlp already does its OWN internal network
        retries, so a non-zero exit is effectively a real failure (private/gone/
        geo-blocked), like a 4xx on the GET path: fail fast to the per-item error
        → dead-letter path.
        """
        dest = Path(dest)
        out_template = str(dest.with_suffix("")) + ".%(ext)s"
        cmd = ["yt-dlp", "--no-progress", "-o", out_template]
        for key, value in headers.items():
            cmd += ["--add-header", f"{key}:{value}"]
        if self.cookies_path and os.path.isfile(self.cookies_path):
            cmd += ["--cookies", self.cookies_path]
        cmd.append(url)
        attempt = 0
        while True:
            try:
                subprocess.run(
                    cmd,
                    check=True,
                    capture_output=True,
                    text=True,
                    timeout=_TIMEOUT_SECONDS,
                )
                break
            except subprocess.CalledProcessError as exc:
                # Permanent for this pass — fail fast (no retry).
                log.warning(
                    "yt-dlp failed (exit %s) for %s: %s",
                    exc.returncode, url, (exc.stderr or "").strip() or exc,
                )
                return None
            except (OSError, subprocess.TimeoutExpired) as exc:
                # Transient — back off and retry, like a transport blip on a GET.
                if attempt >= _MAX_MEDIA_RETRIES:
                    log.warning(
                        "yt-dlp transient failure exhausted for %s: %s", url, exc
                    )
                    return None
                attempt += 1
                delay = min(2.0 * (2 ** (attempt - 1)), _BACKOFF_CAP_SECONDS)
                log.warning(
                    "yt-dlp transient failure (%s) — backing off %.1fs "
                    "(retry %d/%d): %s",
                    url, delay, attempt, _MAX_MEDIA_RETRIES, exc,
                )
                time.sleep(delay)
        return self._existing_video_output(dest)

    def _existing_video_output(self, dest: Path) -> Path | None:
        """Find a yt-dlp output for `dest` regardless of chosen extension.

        Returns `dest` itself if present, else any sibling sharing the same
        stem (the remuxed container). None if nothing matched.
        """
        dest = Path(dest)
        if dest.exists():
            return dest
        stem = dest.with_suffix("").name
        parent = dest.parent
        if not parent.is_dir():
            return None
        for cand in sorted(parent.iterdir()):
            if cand.is_file() and cand.stem == stem and cand.name != dest.name:
                return cand
        return None

    def _write_post_record_file(
        self, post: dict, sidecar_path: Path, *, detail_fetch: bool = True,
    ) -> Path:
        """Serialize the full post record (body, title, date, url) to
        `sidecar_path`. The per-file sidecars carry only the file's identity
        (BaseNativeDownloader._media_sidecar)."""
        attrs = post.get("attributes") or {}
        title = attrs.get("title")
        # Resolve the body HTML from the feed attrs: legacy flat `content`, else
        # convert the current `content_json_string` ProseMirror doc (#842).
        content = post_body_html(attrs)
        # The feed/list endpoint frequently returns an empty body; the full body
        # only comes from the per-post detail endpoint. Enrich on first write for
        # this post and MEMOIZE the RESOLVED HTML by mutating the shared `post`
        # dict — so a multi-image post fetches detail at most once, the post-record
        # body-length read reuses it, and a fully-seen post (no fresh download → no
        # sidecar write) never pays the extra GET.
        # `detail_fetch=False` on a REVISIT (a post inside the tick's revisit
        # window that we already captured): re-read the body from the feed
        # response we are holding and pay nothing. Without this a 30-day window
        # would buy one detail GET per body-less post per tick, forever — a
        # per-creator cost that grows with how prolific they are, to re-fetch a
        # body we already stored.
        if (not content or not content.strip()) and self._content_fetcher and detail_fetch:
            fetched = self._content_fetcher(str(post.get("id") or ""))
            if fetched:
                content = fetched
        if isinstance(content, str) and content.strip():
            attrs["content"] = content
            post["attributes"] = attrs
        published = attrs.get("published_at")
        url = attrs.get("url")
        data = {
            "category": "patreon",
            "id": str(post.get("id") or ""),
            "title": title if isinstance(title, str) else "",
            "content": content if isinstance(content, str) else "",
            "published_at": published if isinstance(published, str) else None,
            "url": url if isinstance(url, str) else None,
        }
        sidecar_path.write_text(json.dumps(data, indent=2))
        return sidecar_path

    def write_post_record(
        self, post: dict, artist_slug: str, *, revisit: bool = False,
    ) -> PostRecordOutcome:
        """Write a post-ONLY sidecar (no media file) for a media-less post, so
        the importer can still upsert the Post + its body — text posts often hold
        the only copy of an external <a href> link. Named `_post.json`: the
        leading underscore keeps it from colliding with a media sidecar
        (`<NN>_<stem>.json`) and from being resolved as some media file's sidecar
        by find_sidecar.

        Returns a PostRecordOutcome (path None when the post has no id) carrying
        the captured body's shape — post_type + final char count — so the engine
        can log per-post handling without re-reading the post itself.

        `revisit=True` is the tick re-reading a post it already captured
        (ingest_core's revisit window, #...). Two differences, both about not
        making an update cost more than it is worth:

        * no detail-fetch — the body comes from the feed response already in
          hand, so a revisit costs zero requests;
        * a body that comes back empty writes NOTHING and returns `path=None`.
          On a first capture an empty body is the truth about the post; on a
          revisit it usually just means this post's body only ever came from
          the detail endpoint we just declined to call, and writing it would
          blank a stored body to say something we never learned.
        """
        attrs = post.get("attributes") or {}
        title = attrs.get("title") if isinstance(attrs.get("title"), str) else None
        post_type = attrs.get("post_type") if isinstance(attrs.get("post_type"), str) else None
        pid = str(post.get("id") or "")
        if not pid:
            return PostRecordOutcome(
                path=None, post_type=post_type, title=title, body_chars=0,
            )
        if revisit:
            feed_body = post_body_html(attrs)
            if not (isinstance(feed_body, str) and feed_body.strip()):
                return PostRecordOutcome(
                    path=None, post_type=post_type, title=title, body_chars=0,
                )
        post_dir = self.images_root / artist_slug / "patreon" / post_dir_name(post)
        post_dir.mkdir(parents=True, exist_ok=True)
        path = self._write_post_record_file(
            post, post_dir / "_post.json", detail_fetch=not revisit,
        )
        # _write_post_record_file has by now memoized any detail-fetched body onto
        # post["attributes"]["content"], so re-read it for the FINAL char count.
        body = (post.get("attributes") or {}).get("content")
        body_chars = len(body) if isinstance(body, str) else 0
        return PostRecordOutcome(
            path=path, post_type=post_type, title=title, body_chars=body_chars,
        )