FabledCurator/backend/app/services/patreon_ingester.py

"""Native Patreon ingester — the Patreon ADAPTER over the platform-agnostic core.

The orchestration that drives a native subscription walk (page a feed → extract
media → tiered skip → download → mark-seen / record-failures / checkpoint-cursor
→ return a gallery-dl-shaped `DownloadResult`, across tick/backfill/recovery)
now lives in `ingest_core.Ingester` — it's identical for every platform. This
module is the thin Patreon adapter: it wires the Patreon `client`/`downloader`/
ledger models/constraints/key into the core and supplies the Patreon-specific
failure mapping. `download_service.download_source` calls `PatreonIngester.run`
exactly as before; the public surface (this class, `_ledger_key`,
`DEAD_LETTER_THRESHOLD`, `verify_patreon_credential`) is unchanged.

Three modes (selected by `download_service` from `config_overrides` state):
  - tick      — newest→oldest, skip seen (tier-1 ledger + tier-2 disk), early-out
                after N contiguous already-have-it items (the cheap native
                equivalent of gallery-dl's `exit:20`, now free of per-file HEADs).
  - backfill  — full-history walk in a time-boxed chunk, resuming from the
                pagination cursor checkpoint; reaches the bottom → "complete".
  - recovery  — like backfill but BYPASSES the tier-1 seen-ledger AND the
                dead-letter ledger, so deliberately-dropped-and-deleted near-dups
                get re-fetched and re-evaluated under the current pHash threshold
                (tier-2 disk skip still spares files we kept).

The seen/dead-letter ledgers live in Postgres (`patreon_seen_media` /
`patreon_failed_media`); the core opens SHORT-LIVED sync sessions per page batch
— never held across a network fetch ([[db-connection-held-across-subprocess]]).
FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
"""

from __future__ import annotations

import asyncio
import logging
from collections.abc import Callable
from pathlib import Path

from ..models import PatreonFailedMedia, PatreonSeenMedia
from .gallery_dl import DownloadResult, ErrorType
from .ingest_core import DEAD_LETTER_THRESHOLD, Ingester
from .patreon_client import (
    MediaItem,
    PatreonAPIError,
    PatreonAuthError,
    PatreonClient,
    PatreonDriftError,
)
from .patreon_downloader import PatreonDownloader
from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source

__all__ = [
    "DEAD_LETTER_THRESHOLD",
    "PatreonIngester",
    "_ledger_key",
    "verify_patreon_credential",
]

log = logging.getLogger(__name__)

# Ledger keys are stored in patreon_seen_media.filehash VARCHAR(128); bound any
# synthesized key so a pathologically long file_name can't overflow the column.
_LEDGER_KEY_MAX = 128


def _ledger_key(media: MediaItem) -> str:
    """Stable per-media identity for the cross-run seen-ledger.

    A Patreon CDN URL carries a 32-char MD5 (`media.filehash`) — that is the
    natural key. Some media have none: Mux/HLS video (`stream.mux.com`, no
    content hash at discovery) and the odd inline-content `<img>` pointing at a
    hashless URL. The plan calls the video case the ``video:<post_id>:<media_id>``
    sentinel; `MediaItem` carries no media_id, so the post-scoped filename is the
    stable proxy. Bounded to the column width.
    """
    if media.filehash:
        return media.filehash
    return f"{media.post_id}:{media.filename}"[:_LEDGER_KEY_MAX]


class PatreonIngester(Ingester):
    """Walk a Patreon campaign's posts, download unseen media, return a
    `DownloadResult`. A thin adapter over `ingest_core.Ingester`.

    Construct with the per-source `cookies_path` and a sync sessionmaker for the
    ledgers. `client` / `downloader` are injectable seams so unit tests run
    without network, subprocess, or a real CDN.
    """

    def __init__(
        self,
        images_root: Path,
        cookies_path: str | None,
        session_factory: Callable[[], object],
        *,
        validate: bool = True,
        rate_limit: float = 0.0,
        request_sleep: float = 0.0,
        client: PatreonClient | None = None,
        downloader: PatreonDownloader | None = None,
    ):
        self.images_root = Path(images_root)
        self.cookies_path = str(cookies_path) if cookies_path else None
        # Pacing (plan #703): request_sleep paces the API page fetches,
        # rate_limit paces the media downloads. Injected client/downloader (in
        # tests) already carry their own pacing, so these only apply to the
        # default-constructed ones.
        resolved_client = (
            client
            if client is not None
            else PatreonClient(cookies_path, request_sleep=request_sleep)
        )
        resolved_downloader = (
            downloader
            if downloader is not None
            else PatreonDownloader(
                self.images_root, cookies_path, validate=validate, rate_limit=rate_limit,
                # Enrich empty feed bodies from the per-post detail endpoint, via
                # the SAME client (shares its cookie session + request pacing).
                content_fetcher=resolved_client.fetch_post_detail_content,
            )
        )
        super().__init__(
            client=resolved_client,
            downloader=resolved_downloader,
            session_factory=session_factory,
            seen_model=PatreonSeenMedia,
            failed_model=PatreonFailedMedia,
            seen_constraint="uq_patreon_seen_media_source_id",
            failed_constraint="uq_patreon_failed_media_source_id",
            ledger_key=_ledger_key,
            platform="patreon",
            error_base=PatreonAPIError,
        )

    # -- failure mapping (Patreon exception taxonomy) ----------------------

    def _failure_result(self, exc: Exception, _result) -> DownloadResult:
        """Map a client-level exception to a loud, typed failed DownloadResult.

        We NEVER return a silent zero-download "success" — the whole point of the
        native ingester is to fail RED when Patreon's API shape or our auth
        changes. The typed mapping lets FailingSourcesCard render the right chip
        and tells the operator what to do:
          - PatreonAuthError  → AUTH_ERROR  (rotate cookies)
          - PatreonDriftError → API_DRIFT   (ingester field-set/parser needs update)
          - HTTP 429 / 404    → RATE_LIMITED / NOT_FOUND
          - other HTTP status → HTTP_ERROR;  transport failure → NETWORK_ERROR

        PatreonAuthError and PatreonDriftError both subclass PatreonAPIError, so
        they must be matched before the generic HTTP/transport fallthrough.
        """
        message = str(exc)
        if isinstance(exc, PatreonAuthError):
            error_type = ErrorType.AUTH_ERROR
        elif isinstance(exc, PatreonDriftError):
            error_type = ErrorType.API_DRIFT
            message = f"Patreon API changed — ingester needs update: {message}"
        else:  # generic PatreonAPIError: HTTP non-2xx (status_code set) or transport
            status = getattr(exc, "status_code", None)
            if status == 429:
                error_type = ErrorType.RATE_LIMITED
            elif status == 404:
                error_type = ErrorType.NOT_FOUND
            elif status is not None:
                error_type = ErrorType.HTTP_ERROR
            else:
                error_type = ErrorType.NETWORK_ERROR
        log.warning("Patreon ingest failed (%s): %s", error_type.value, message)
        result = _result(
            success=False, return_code=1,
            error_type=error_type, error_message=message,
        )
        # plan #708 B1: carry the server's Retry-After up to the cooldown.
        if error_type == ErrorType.RATE_LIMITED:
            result.retry_after_seconds = getattr(exc, "retry_after", None)
        return result


async def verify_patreon_credential(
    url: str,
    cookies_path: str | None,
    overrides: dict | None,
) -> tuple[bool | None, str]:
    """Native Patreon credential probe — the verify counterpart to the ingester's
    download path, sharing its campaign-id resolution. Resolves the campaign id
    (override / id: URL / vanity) then does ONE authenticated `/api/posts` fetch
    via PatreonClient.verify_auth. Returns the uniform `(ok, message)` contract
    (True / False / None) so download_backends.verify_credential can treat it
    interchangeably with the gallery-dl probe. No download, no DB.
    """
    campaign_id, _ = await resolve_campaign_id_for_source(url, cookies_path, overrides)
    if not campaign_id:
        vanity = extract_vanity(url)
        return None, (
            f"Couldn't resolve the Patreon campaign id — can't verify. "
            f"source_url={url!r}; vanity={vanity!r} "
            "(cookies expired, or the creator moved/renamed?)."
        )
    client = PatreonClient(cookies_path)
    loop = asyncio.get_running_loop()
    return await loop.run_in_executor(None, client.verify_auth, campaign_id)