"""Native Patreon ingester — the Patreon ADAPTER over the platform-agnostic core.
The orchestration that drives a native subscription walk (page a feed → extract
media → tiered skip → download → mark-seen / record-failures / checkpoint-cursor
→ return a gallery-dl-shaped `DownloadResult`, across tick/backfill/recovery)
now lives in `ingest_core.Ingester` — it's identical for every platform. This
module is the thin Patreon adapter: it wires the Patreon `client`/`downloader`/
ledger models/constraints/key into the core and supplies the Patreon-specific
failure mapping. `download_service.download_source` calls `PatreonIngester.run`
exactly as before; the public surface (this class, `_ledger_key`,
`DEAD_LETTER_THRESHOLD`, `verify_patreon_credential`) is unchanged.
Three modes (selected by `download_service` from `config_overrides` state):
- tick — newest→oldest, skip seen (tier-1 ledger + tier-2 disk), early-out
after N contiguous already-have-it items (the cheap native
equivalent of gallery-dl's `exit:20`, now free of per-file HEADs).
- backfill — full-history walk in a time-boxed chunk, resuming from the
pagination cursor checkpoint; reaches the bottom → "complete".
- recovery — like backfill but BYPASSES the tier-1 seen-ledger AND the
dead-letter ledger, so deliberately-dropped-and-deleted near-dups
get re-fetched and re-evaluated under the current pHash threshold
(tier-2 disk skip still spares files we kept).
The seen/dead-letter ledgers live in Postgres (`patreon_seen_media` /
`patreon_failed_media`); the core opens SHORT-LIVED sync sessions per page batch
— never held across a network fetch ([[db-connection-held-across-subprocess]]).
FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
"""
from __future__ import annotations
import asyncio
import logging
from collections.abc import Callable
from pathlib import Path
from ..models import PatreonFailedMedia, PatreonSeenMedia
from .gallery_dl import DownloadResult, ErrorType
from .ingest_core import DEAD_LETTER_THRESHOLD, Ingester
from .patreon_client import (
MediaItem,
PatreonAPIError,
PatreonAuthError,
PatreonClient,
PatreonDriftError,
)
from .patreon_downloader import PatreonDownloader
from .patreon_resolver import resolve_campaign_id_for_source
__all__ = [
"DEAD_LETTER_THRESHOLD",
"PatreonIngester",
"_ledger_key",
"verify_patreon_credential",
]
log = logging.getLogger(__name__)
# Ledger keys are stored in patreon_seen_media.filehash VARCHAR(128); bound any
# synthesized key so a pathologically long file_name can't overflow the column.
_LEDGER_KEY_MAX = 128
def _ledger_key(media: MediaItem) -> str:
"""Stable per-media identity for the cross-run seen-ledger.
A Patreon CDN URL carries a 32-char MD5 (`media.filehash`) — that is the
natural key. Some media have none: Mux/HLS video (`stream.mux.com`, no
content hash at discovery) and the odd inline-content `![]()
` pointing at a
hashless URL. The plan calls the video case the ``video::``
sentinel; `MediaItem` carries no media_id, so the post-scoped filename is the
stable proxy. Bounded to the column width.
"""
if media.filehash:
return media.filehash
return f"{media.post_id}:{media.filename}"[:_LEDGER_KEY_MAX]
class PatreonIngester(Ingester):
"""Walk a Patreon campaign's posts, download unseen media, return a
`DownloadResult`. A thin adapter over `ingest_core.Ingester`.
Construct with the per-source `cookies_path` and a sync sessionmaker for the
ledgers. `client` / `downloader` are injectable seams so unit tests run
without network, subprocess, or a real CDN.
"""
def __init__(
self,
images_root: Path,
cookies_path: str | None,
session_factory: Callable[[], object],
*,
validate: bool = True,
rate_limit: float = 0.0,
request_sleep: float = 0.0,
client: PatreonClient | None = None,
downloader: PatreonDownloader | None = None,
):
self.images_root = Path(images_root)
self.cookies_path = str(cookies_path) if cookies_path else None
# Pacing (plan #703): request_sleep paces the API page fetches,
# rate_limit paces the media downloads. Injected client/downloader (in
# tests) already carry their own pacing, so these only apply to the
# default-constructed ones.
resolved_client = (
client
if client is not None
else PatreonClient(cookies_path, request_sleep=request_sleep)
)
resolved_downloader = (
downloader
if downloader is not None
else PatreonDownloader(
self.images_root, cookies_path, validate=validate, rate_limit=rate_limit
)
)
super().__init__(
client=resolved_client,
downloader=resolved_downloader,
session_factory=session_factory,
seen_model=PatreonSeenMedia,
failed_model=PatreonFailedMedia,
seen_constraint="uq_patreon_seen_media_source_id",
failed_constraint="uq_patreon_failed_media_source_id",
ledger_key=_ledger_key,
platform="patreon",
error_base=PatreonAPIError,
)
# -- failure mapping (Patreon exception taxonomy) ----------------------
def _failure_result(self, exc: Exception, _result) -> DownloadResult:
"""Map a client-level exception to a loud, typed failed DownloadResult.
We NEVER return a silent zero-download "success" — the whole point of the
native ingester is to fail RED when Patreon's API shape or our auth
changes. The typed mapping lets FailingSourcesCard render the right chip
and tells the operator what to do:
- PatreonAuthError → AUTH_ERROR (rotate cookies)
- PatreonDriftError → API_DRIFT (ingester field-set/parser needs update)
- HTTP 429 / 404 → RATE_LIMITED / NOT_FOUND
- other HTTP status → HTTP_ERROR; transport failure → NETWORK_ERROR
PatreonAuthError and PatreonDriftError both subclass PatreonAPIError, so
they must be matched before the generic HTTP/transport fallthrough.
"""
message = str(exc)
if isinstance(exc, PatreonAuthError):
error_type = ErrorType.AUTH_ERROR
elif isinstance(exc, PatreonDriftError):
error_type = ErrorType.API_DRIFT
message = f"Patreon API changed — ingester needs update: {message}"
else: # generic PatreonAPIError: HTTP non-2xx (status_code set) or transport
status = getattr(exc, "status_code", None)
if status == 429:
error_type = ErrorType.RATE_LIMITED
elif status == 404:
error_type = ErrorType.NOT_FOUND
elif status is not None:
error_type = ErrorType.HTTP_ERROR
else:
error_type = ErrorType.NETWORK_ERROR
log.warning("Patreon ingest failed (%s): %s", error_type.value, message)
result = _result(
success=False, return_code=1,
error_type=error_type, error_message=message,
)
# plan #708 B1: carry the server's Retry-After up to the cooldown.
if error_type == ErrorType.RATE_LIMITED:
result.retry_after_seconds = getattr(exc, "retry_after", None)
return result
async def verify_patreon_credential(
url: str,
cookies_path: str | None,
overrides: dict | None,
) -> tuple[bool | None, str]:
"""Native Patreon credential probe — the verify counterpart to the ingester's
download path, sharing its campaign-id resolution. Resolves the campaign id
(override / id: URL / vanity) then does ONE authenticated `/api/posts` fetch
via PatreonClient.verify_auth. Returns the uniform `(ok, message)` contract
(True / False / None) so download_backends.verify_credential can treat it
interchangeably with the gallery-dl probe. No download, no DB.
"""
campaign_id, _ = await resolve_campaign_id_for_source(url, cookies_path, overrides)
if not campaign_id:
return None, (
"Couldn't resolve the Patreon campaign id from the source URL — "
"can't verify (cookies expired, or the creator moved/renamed?)."
)
client = PatreonClient(cookies_path)
loop = asyncio.get_running_loop()
return await loop.run_in_executor(None, client.verify_auth, campaign_id)