"""Native Patreon ingester — the Patreon ADAPTER over the platform-agnostic core. The orchestration that drives a native subscription walk (page a feed → extract media → tiered skip → download → mark-seen / record-failures / checkpoint-cursor → return a gallery-dl-shaped `DownloadResult`, across tick/backfill/recovery) now lives in `ingest_core.Ingester` — it's identical for every platform. This module is the thin Patreon adapter: it wires the Patreon `client`/`downloader`/ ledger models/constraints/key into the core and supplies the Patreon-specific failure mapping. `download_service.download_source` calls `PatreonIngester.run` exactly as before; the public surface (this class, `_ledger_key`, `DEAD_LETTER_THRESHOLD`, `verify_patreon_credential`) is unchanged. Three modes (selected by `download_service` from `config_overrides` state): - tick — newest→oldest, skip seen (tier-1 ledger + tier-2 disk), early-out after N contiguous already-have-it items (the cheap native equivalent of gallery-dl's `exit:20`, now free of per-file HEADs). - backfill — full-history walk in a time-boxed chunk, resuming from the pagination cursor checkpoint; reaches the bottom → "complete". - recovery — like backfill but BYPASSES the tier-1 seen-ledger AND the dead-letter ledger, so deliberately-dropped-and-deleted near-dups get re-fetched and re-evaluated under the current pHash threshold (tier-2 disk skip still spares files we kept). The seen/dead-letter ledgers live in Postgres (`patreon_seen_media` / `patreon_failed_media`); the core opens SHORT-LIVED sync sessions per page batch — never held across a network fetch ([[db-connection-held-across-subprocess]]). FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API. """ from __future__ import annotations import asyncio import logging from collections.abc import Callable from pathlib import Path from ..models import PatreonFailedMedia, PatreonSeenMedia from .ingest_core import DEAD_LETTER_THRESHOLD, Ingester from .patreon_client import MediaItem, PatreonAPIError, PatreonClient from .patreon_downloader import PatreonDownloader from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source __all__ = [ "DEAD_LETTER_THRESHOLD", "PatreonIngester", "_ledger_key", "verify_patreon_credential", ] log = logging.getLogger(__name__) # Ledger keys are stored in patreon_seen_media.filehash VARCHAR(128); bound any # synthesized key so a pathologically long file_name can't overflow the column. _LEDGER_KEY_MAX = 128 def _ledger_key(media: MediaItem) -> str: """Stable per-media identity for the cross-run seen-ledger. A Patreon CDN URL carries a 32-char MD5 (`media.filehash`) — that is the natural key. Some media have none: Mux/HLS video (`stream.mux.com`, no content hash at discovery) and the odd inline-content `` pointing at a hashless URL. The plan calls the video case the ``video::`` sentinel; `MediaItem` carries no media_id, so the post-scoped filename is the stable proxy. Bounded to the column width. """ if media.filehash: return media.filehash return f"{media.post_id}:{media.filename}"[:_LEDGER_KEY_MAX] class PatreonIngester(Ingester): """Walk a Patreon campaign's posts, download unseen media, return a `DownloadResult`. A thin adapter over `ingest_core.Ingester`. Construct with the per-source `cookies_path` and a sync sessionmaker for the ledgers. `client` / `downloader` are injectable seams so unit tests run without network, subprocess, or a real CDN. """ def __init__( self, images_root: Path, cookies_path: str | None, session_factory: Callable[[], object], *, validate: bool = True, rate_limit: float = 0.0, request_sleep: float = 0.0, auth_token: str | None = None, client: PatreonClient | None = None, downloader: PatreonDownloader | None = None, ): # auth_token: accepted for the uniform native-ingester construction # (download_backends passes it to every adapter); Patreon # authenticates by cookies, so it's unused here. del auth_token self.images_root = Path(images_root) self.cookies_path = str(cookies_path) if cookies_path else None # Pacing (plan #703): request_sleep paces the API page fetches, # rate_limit paces the media downloads. Injected client/downloader (in # tests) already carry their own pacing, so these only apply to the # default-constructed ones. resolved_client = ( client if client is not None else PatreonClient(cookies_path, request_sleep=request_sleep) ) resolved_downloader = ( downloader if downloader is not None else PatreonDownloader( self.images_root, cookies_path, validate=validate, rate_limit=rate_limit, # Enrich empty feed bodies from the per-post detail endpoint, via # the SAME client (shares its cookie session + request pacing). content_fetcher=resolved_client.fetch_post_detail_content, ) ) super().__init__( client=resolved_client, downloader=resolved_downloader, session_factory=session_factory, seen_model=PatreonSeenMedia, failed_model=PatreonFailedMedia, seen_constraint="uq_patreon_seen_media_source_id", failed_constraint="uq_patreon_failed_media_source_id", ledger_key=_ledger_key, platform="patreon", error_base=PatreonAPIError, # API_DRIFT message phrasing; the base Ingester._failure_result owns # the auth/drift/HTTP→error_type mapping now (shared across platforms). drift_label="Patreon API", ) async def verify_patreon_credential( url: str, cookies_path: str | None, overrides: dict | None, ) -> tuple[bool | None, str]: """Native Patreon credential probe — the verify counterpart to the ingester's download path, sharing its campaign-id resolution. Resolves the campaign id (override / id: URL / vanity) then does ONE authenticated `/api/posts` fetch via PatreonClient.verify_auth. Returns the uniform `(ok, message)` contract (True / False / None) so download_backends.verify_credential can treat it interchangeably with the gallery-dl probe. No download, no DB. """ campaign_id, _ = await resolve_campaign_id_for_source(url, cookies_path, overrides) if not campaign_id: vanity = extract_vanity(url) return None, ( f"Couldn't resolve the Patreon campaign id — can't verify. " f"source_url={url!r}; vanity={vanity!r} " "(cookies expired, or the creator moved/renamed?)." ) client = PatreonClient(cookies_path) loop = asyncio.get_running_loop() return await loop.run_in_executor(None, client.verify_auth, campaign_id)