"""Platform → download-backend dispatch (one place that knows which platforms are served by the native FC ingester vs. the gallery-dl subprocess). gallery-dl wasn't built to be driven by an automated scheduler — no native checkpoint/resume, no structured logs, per-file HEADs that dominate wall-clock. The native ingester (services/patreon_ingester.py, plan #697) replaces it for Patreon and is the path we grow as more platforms migrate. To keep that migration DRY, every caller that has to behave differently per backend — download routing, the credential-verify probe, cursor handling — asks THIS module instead of testing ``platform == "patreon"`` inline. When a platform gets a native ingester, it moves into ``NATIVE_INGESTER_PLATFORMS`` here and both the download path and verify switch over together. The backend surfaces share a UNIFORM signature so a caller invokes the same function regardless of platform: - verify_credential(...) → (ok: bool|None, message: str) - (download stays in download_service for now; uses_native_ingester() is the shared predicate it routes on, so the decision lives here too.) """ from __future__ import annotations import asyncio from pathlib import Path from .gallery_dl import DownloadResult, ErrorType from .patreon_ingester import PatreonIngester from .patreon_resolver import resolve_campaign_id_for_source # Platforms whose download + verify go through the native ingester rather than # gallery-dl. gallery-dl still serves every other platform (subscribestar, # hentaifoundry, discord, pixiv, deviantart) unchanged. NATIVE_INGESTER_PLATFORMS = frozenset({"patreon"}) def uses_native_ingester(platform: str) -> bool: """True when `platform` is served by the native ingester (not gallery-dl). The single predicate the download path and verify both route on.""" return platform in NATIVE_INGESTER_PLATFORMS async def run_download( *, ctx: dict, source_config, skip_value: bool | str, mode: str | None, gdl, sync_session_factory, ) -> tuple[DownloadResult, str | None]: """Uniform download across backends — the download counterpart to `verify_source_credential`, so this module is the ONE place that knows how each platform both downloads AND verifies (the seam that makes adding a platform a bounded job). Returns `(DownloadResult, resolved_campaign_id)`; `resolved_campaign_id` is non-None only when a native vanity lookup ran this call (so phase 3 caches it). Native platforms route through their ingester in `mode` (tick/backfill/recovery); gallery-dl platforms run the subprocess. The caller (download_service) prepares `source_config`/`skip_value`/`mode` from the backfill state machine and owns phase 3. """ platform = ctx["platform"] if uses_native_ingester(platform): return await _run_native_ingester( ctx, source_config, mode, gdl, sync_session_factory ) result = await gdl.download( url=ctx["url"], artist_slug=ctx["artist_slug"], platform=platform, source_config=source_config, cookies_path=ctx["cookies_path"], auth_token=ctx["auth_token"], skip_value=skip_value, ) return result, None async def _run_native_ingester( ctx: dict, source_config, mode: str | None, gdl, sync_session_factory, ) -> tuple[DownloadResult, str | None]: """Patreon (today the only native platform): resolve the campaign id, then run the native ingester in a worker thread (it is sync requests/subprocess). `resolved_campaign_id` is non-None only when we had to look it up from the vanity URL this run, so phase 3 caches it the way the old gallery-dl retry did. A campaign id we cannot resolve is a loud NOT_FOUND — never a silent empty success. """ overrides = ctx["config_overrides"] or {} campaign_id, resolved_campaign_id = await resolve_campaign_id_for_source( ctx["url"], ctx["cookies_path"], overrides ) if not campaign_id: return ( DownloadResult( success=False, url=ctx["url"], artist_slug=ctx["artist_slug"], platform="patreon", error_type=ErrorType.NOT_FOUND, error_message=( "Could not resolve Patreon campaign id from the source URL " "(vanity lookup failed — cookies expired or creator moved?)" ), ), None, ) # Honor the operator's existing rate-limit knobs on the native path (plan # #703): the global download_rate_limit_seconds (gallery-dl's `rate_limit`, # here on the gdl service) paces media downloads; page fetches use the # per-source sleep_request override, else `max(0.5, rate_limit/4)` — the same # API-pacing default gallery-dl applied as its `sleep-request`. rate_limit = gdl._rate_limit request_sleep = ( source_config.sleep_request if source_config.sleep_request is not None else max(0.5, rate_limit / 4) ) ingester = PatreonIngester( images_root=gdl.images_root, cookies_path=ctx["cookies_path"], session_factory=sync_session_factory, validate=gdl._validate_files, rate_limit=rate_limit, request_sleep=request_sleep, ) loop = asyncio.get_running_loop() dl_result = await loop.run_in_executor( None, lambda: ingester.run( source_id=ctx["source_id"], campaign_id=campaign_id, artist_slug=ctx["artist_slug"], url=ctx["url"], mode=mode, resume_cursor=source_config.resume_cursor, time_budget_seconds=source_config.timeout, posts_base=int(overrides.get("_backfill_posts", 0)), ), ) return dl_result, resolved_campaign_id async def preview_source( *, platform: str, url: str, source_id: int, config_overrides: dict | None, cookies_path: str | None, images_root: Path, sync_session_factory, page_limit: int = 3, ) -> dict: """Dry-run preview for a native platform (plan #708 B4): resolve the campaign id, then walk a few pages counting media not already seen/dead — no download. Returns the preview dict (total_new / posts_scanned / pages_scanned / has_more / sample), or `{"error": msg}` on a resolve / auth / drift failure. Native-only — the caller gates on `uses_native_ingester`. """ import asyncio from .patreon_client import PatreonAPIError campaign_id, _ = await resolve_campaign_id_for_source( url, cookies_path, config_overrides or {} ) if not campaign_id: return { "error": ( "Couldn't resolve the campaign id from the source URL " "(cookies expired, or the creator moved/renamed?)." ) } ingester = PatreonIngester( images_root=images_root, cookies_path=cookies_path, session_factory=sync_session_factory, ) loop = asyncio.get_running_loop() try: result = await loop.run_in_executor( None, lambda: ingester.preview(source_id, campaign_id, page_limit=page_limit), ) except PatreonAPIError as exc: return {"error": f"Couldn't preview: {exc}"} return result async def verify_source_credential( *, platform: str, url: str, artist_slug: str, config_overrides: dict | None, cookies_path: str | None, auth_token: str | None, images_root: Path, ) -> tuple[bool | None, str]: """Uniform credential probe across backends. Returns `(ok, message)`: True = authenticated, False = rejected, None = inconclusive (drift / network / nothing to test). Callers don't branch on platform — they call this and render the result. """ if uses_native_ingester(platform): # Native ingester platforms verify via their own lightweight auth probe # (resolve campaign id + one authenticated API page). Patreon today. from .patreon_ingester import verify_patreon_credential return await verify_patreon_credential(url, cookies_path, config_overrides) # gallery-dl platforms: --simulate one item; the extractor errors before it # can list if auth is bad. from .gallery_dl import GalleryDLService, SourceConfig gdl = GalleryDLService(images_root=images_root) return await gdl.verify( url=url, artist_slug=artist_slug, platform=platform, source_config=SourceConfig.from_dict(config_overrides or {}), cookies_path=cookies_path, auth_token=auth_token, )