"""Platform → download-backend dispatch (one place that knows which platforms are served by the native FC ingester vs. the gallery-dl subprocess). gallery-dl wasn't built to be driven by an automated scheduler — no native checkpoint/resume, no structured logs, per-file HEADs that dominate wall-clock. The native ingester (services/patreon_ingester.py, plan #697) replaces it for Patreon and is the path we grow as more platforms migrate. To keep that migration DRY, every caller that has to behave differently per backend — download routing, the credential-verify probe, cursor handling — asks THIS module instead of testing ``platform == "patreon"`` inline. When a platform gets a native ingester, it moves into ``NATIVE_INGESTER_PLATFORMS`` here and both the download path and verify switch over together. The backend surfaces share a UNIFORM signature so a caller invokes the same function regardless of platform: - verify_credential(...) → (ok: bool|None, message: str) - (download stays in download_service for now; uses_native_ingester() is the shared predicate it routes on, so the decision lives here too.) """ from __future__ import annotations import asyncio from pathlib import Path from .gallery_dl import DownloadResult, ErrorType from .native_ingest_common import NativeIngestError from .patreon_ingester import PatreonIngester from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source from .subscribestar_ingester import SubscribeStarIngester # Platforms whose download + verify go through the native ingester rather than # gallery-dl. gallery-dl still serves the rest (hentaifoundry, discord, pixiv, # deviantart) until they migrate too. NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar"}) # Native ingester classes keyed by platform (uniform constructor signature, so # _run_native_ingester / preview build whichever the source needs). _NATIVE_INGESTERS = { "patreon": PatreonIngester, "subscribestar": SubscribeStarIngester, } # Mirrors patreon_resolver._CAMPAIGNS_URL — surfaced in resolution-failure # messages so the operator sees the exact lookup endpoint that was hit. _CAMPAIGNS_API = "https://www.patreon.com/api/campaigns" def uses_native_ingester(platform: str) -> bool: """True when `platform` is served by the native ingester (not gallery-dl). The single predicate the download path and verify both route on.""" return platform in NATIVE_INGESTER_PLATFORMS async def run_download( *, ctx: dict, source_config, skip_value: bool | str, mode: str | None, gdl, sync_session_factory, ) -> tuple[DownloadResult, str | None]: """Uniform download across backends — the download counterpart to `verify_source_credential`, so this module is the ONE place that knows how each platform both downloads AND verifies (the seam that makes adding a platform a bounded job). Returns `(DownloadResult, resolved_campaign_id)`; `resolved_campaign_id` is non-None only when a native vanity lookup ran this call (so phase 3 caches it). Native platforms route through their ingester in `mode` (tick/backfill/recovery); gallery-dl platforms run the subprocess. The caller (download_service) prepares `source_config`/`skip_value`/`mode` from the backfill state machine and owns phase 3. """ platform = ctx["platform"] if uses_native_ingester(platform): return await _run_native_ingester( ctx, source_config, mode, gdl, sync_session_factory ) result = await gdl.download( url=ctx["url"], artist_slug=ctx["artist_slug"], platform=platform, source_config=source_config, cookies_path=ctx["cookies_path"], auth_token=ctx["auth_token"], skip_value=skip_value, ) return result, None async def _resolve_native_campaign_id( platform: str, url: str, cookies_path: str | None, overrides: dict, ) -> tuple[str | None, str | None]: """`(campaign_id, resolved_campaign_id)` for a native source. SubscribeStar's feed id IS the creator URL (no lookup → resolved None). Patreon resolves the campaign id from the vanity URL (resolved non-None when a lookup actually ran, so phase 3 caches it).""" if platform == "subscribestar": return url, None return await resolve_campaign_id_for_source(url, cookies_path, overrides) async def _run_native_ingester( ctx: dict, source_config, mode: str | None, gdl, sync_session_factory, ) -> tuple[DownloadResult, str | None]: """Run the native ingester for a native platform in a worker thread (sync requests/subprocess). Patreon resolves a campaign id from the vanity URL; SubscribeStar's feed id is the creator URL itself. A campaign id we cannot resolve is a loud NOT_FOUND — never a silent empty success. `resolved_campaign_id` is non-None only when a lookup ran this call, so phase 3 caches it the way the old gallery-dl retry did. """ platform = ctx["platform"] overrides = ctx["config_overrides"] or {} campaign_id, resolved_campaign_id = await _resolve_native_campaign_id( platform, ctx["url"], ctx["cookies_path"], overrides ) if not campaign_id: # Only reachable for Patreon (SubscribeStar's campaign id is the URL). url = ctx["url"] vanity = extract_vanity(url) return ( DownloadResult( success=False, url=url, artist_slug=ctx["artist_slug"], platform=platform, error_type=ErrorType.NOT_FOUND, error_message=( f"Could not resolve Patreon campaign id. source_url={url!r}; " f"vanity={vanity!r}; " f"lookup=GET {_CAMPAIGNS_API}?filter[vanity]={vanity or ''} " "(vanity lookup failed — cookies expired or creator moved?)" ), ), None, ) # Honor the operator's existing rate-limit knobs on the native path (plan # #703): the global download_rate_limit_seconds (gallery-dl's `rate_limit`, # here on the gdl service) paces media downloads; page fetches use the # per-source sleep_request override, else `max(0.5, rate_limit/4)` — the same # API-pacing default gallery-dl applied as its `sleep-request`. rate_limit = gdl._rate_limit request_sleep = ( source_config.sleep_request if source_config.sleep_request is not None else max(0.5, rate_limit / 4) ) ingester = _NATIVE_INGESTERS[platform]( images_root=gdl.images_root, cookies_path=ctx["cookies_path"], session_factory=sync_session_factory, validate=gdl._validate_files, rate_limit=rate_limit, request_sleep=request_sleep, ) loop = asyncio.get_running_loop() dl_result = await loop.run_in_executor( None, lambda: ingester.run( source_id=ctx["source_id"], campaign_id=campaign_id, artist_slug=ctx["artist_slug"], url=ctx["url"], mode=mode, resume_cursor=source_config.resume_cursor, time_budget_seconds=source_config.timeout, posts_base=int(overrides.get("_backfill_posts", 0)), # plan #709: live progress writes to this running event mid-walk. event_id=ctx.get("event_id"), ), ) return dl_result, resolved_campaign_id async def preview_source( *, platform: str, url: str, source_id: int, config_overrides: dict | None, cookies_path: str | None, images_root: Path, sync_session_factory, page_limit: int = 3, ) -> dict: """Dry-run preview for a native platform (plan #708 B4): resolve the campaign id, then walk a few pages counting media not already seen/dead — no download. Returns the preview dict (total_new / posts_scanned / pages_scanned / has_more / sample), or `{"error": msg}` on a resolve / auth / drift failure. Native-only — the caller gates on `uses_native_ingester`. """ import asyncio campaign_id, _ = await _resolve_native_campaign_id( platform, url, cookies_path, config_overrides or {} ) if not campaign_id: vanity = extract_vanity(url) return { "error": ( f"Couldn't resolve the campaign id. source_url={url!r}; " f"vanity={vanity!r}; lookup=GET {_CAMPAIGNS_API}?filter[vanity]={vanity or ''} " "(cookies expired, or the creator moved/renamed?)." ) } ingester = _NATIVE_INGESTERS[platform]( images_root=images_root, cookies_path=cookies_path, session_factory=sync_session_factory, ) loop = asyncio.get_running_loop() try: result = await loop.run_in_executor( None, lambda: ingester.preview(source_id, campaign_id, page_limit=page_limit), ) except NativeIngestError as exc: return {"error": f"Couldn't preview: {exc}"} return result async def verify_source_credential( *, platform: str, url: str, artist_slug: str, config_overrides: dict | None, cookies_path: str | None, auth_token: str | None, images_root: Path, ) -> tuple[bool | None, str]: """Uniform credential probe across backends. Returns `(ok, message)`: True = authenticated, False = rejected, None = inconclusive (drift / network / nothing to test). Callers don't branch on platform — they call this and render the result. """ if uses_native_ingester(platform): # Native ingester platforms verify via their own lightweight auth probe # (one authenticated feed fetch). SubscribeStar's probe takes the creator # URL directly; Patreon's resolves the campaign id first. if platform == "subscribestar": from .subscribestar_ingester import verify_subscribestar_credential return await verify_subscribestar_credential(url, cookies_path, config_overrides) from .patreon_ingester import verify_patreon_credential return await verify_patreon_credential(url, cookies_path, config_overrides) # gallery-dl platforms: --simulate one item; the extractor errors before it # can list if auth is bad. from .gallery_dl import GalleryDLService, SourceConfig gdl = GalleryDLService(images_root=images_root) return await gdl.verify( url=url, artist_slug=artist_slug, platform=platform, source_config=SourceConfig.from_dict(config_overrides or {}), cookies_path=cookies_path, auth_token=auth_token, )