"""Platform → download-backend dispatch (one place that knows which platforms are served by the native FC ingester vs. the gallery-dl subprocess). gallery-dl wasn't built to be driven by an automated scheduler — no native checkpoint/resume, no structured logs, per-file HEADs that dominate wall-clock. The native ingester (services/patreon_ingester.py, plan #697) replaces it for Patreon and is the path we grow as more platforms migrate. To keep that migration DRY, every caller that has to behave differently per backend — download routing, the credential-verify probe, cursor handling — asks THIS module instead of testing ``platform == "patreon"`` inline. When a platform gets a native ingester, it moves into ``NATIVE_INGESTER_PLATFORMS`` here and both the download path and verify switch over together. The backend surfaces share a UNIFORM signature so a caller invokes the same function regardless of platform: - verify_credential(...) → (ok: bool|None, message: str) - (download stays in download_service for now; uses_native_ingester() is the shared predicate it routes on, so the decision lives here too.) """ from __future__ import annotations import asyncio from pathlib import Path from .gallery_dl import DownloadResult, ErrorType from .patreon_ingester import PatreonIngester from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source from .pixiv_client import user_id_from_url from .pixiv_ingester import PixivIngester from .subscribestar_ingester import SubscribeStarIngester # Platforms whose download + verify go through the native ingester rather than # gallery-dl. gallery-dl still serves the rest (hentaifoundry, discord, # deviantart — the latter slated for retirement, not migration) until they # migrate too. NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar", "pixiv"}) # Mirrors patreon_resolver._CAMPAIGNS_URL — surfaced in resolution-failure # messages so the operator sees the exact lookup endpoint that was hit. _CAMPAIGNS_API = "https://www.patreon.com/api/campaigns" def _native_ingester_cls(platform: str): """The native ingester class for `platform` (uniform constructor signature). A call-time lookup (not a module-level dict captured at import) so tests can monkeypatch db_mod.PatreonIngester / SubscribeStarIngester and have the dispatch pick up the replacement.""" return { "patreon": PatreonIngester, "pixiv": PixivIngester, "subscribestar": SubscribeStarIngester, }[platform] def uses_native_ingester(platform: str) -> bool: """True when `platform` is served by the native ingester (not gallery-dl). The single predicate the download path and verify both route on.""" return platform in NATIVE_INGESTER_PLATFORMS async def run_download( *, ctx: dict, source_config, skip_value: bool | str, mode: str | None, gdl, sync_session_factory, ) -> tuple[DownloadResult, str | None]: """Uniform download across backends — the download counterpart to `verify_source_credential`, so this module is the ONE place that knows how each platform both downloads AND verifies (the seam that makes adding a platform a bounded job). Returns `(DownloadResult, resolved_campaign_id)`; `resolved_campaign_id` is non-None only when a native vanity lookup ran this call (so phase 3 caches it). Native platforms route through their ingester in `mode` (tick/backfill/recovery); gallery-dl platforms run the subprocess. The caller (download_service) prepares `source_config`/`skip_value`/`mode` from the backfill state machine and owns phase 3. """ platform = ctx["platform"] if uses_native_ingester(platform): return await _run_native_ingester( ctx, source_config, mode, gdl, sync_session_factory ) result = await gdl.download( url=ctx["url"], artist_slug=ctx["artist_slug"], platform=platform, source_config=source_config, cookies_path=ctx["cookies_path"], auth_token=ctx["auth_token"], skip_value=skip_value, ) return result, None async def _resolve_native_campaign_id( platform: str, url: str, cookies_path: str | None, overrides: dict, ) -> tuple[str | None, str | None]: """`(campaign_id, resolved_campaign_id)` for a native source. SubscribeStar's feed id IS the creator URL; Pixiv's is the numeric user id parsed straight from it (no lookup → resolved None either way). Patreon resolves the campaign id from the vanity URL (resolved non-None when a lookup actually ran, so phase 3 caches it).""" if platform == "subscribestar": return url, None if platform == "pixiv": return user_id_from_url(url), None return await resolve_campaign_id_for_source(url, cookies_path, overrides) def _campaign_resolution_error(platform: str, url: str) -> str: """Operator-facing message for a native source whose campaign id could not be resolved — names the platform's own lookup mechanism.""" if platform == "pixiv": return ( f"Could not extract a pixiv user id. source_url={url!r} — expected " "a URL like https://www.pixiv.net/users/." ) vanity = extract_vanity(url) return ( f"Could not resolve Patreon campaign id. source_url={url!r}; " f"vanity={vanity!r}; " f"lookup=GET {_CAMPAIGNS_API}?filter[vanity]={vanity or ''} " "(vanity lookup failed — cookies expired or creator moved?)" ) async def _run_native_ingester( ctx: dict, source_config, mode: str | None, gdl, sync_session_factory, ) -> tuple[DownloadResult, str | None]: """Run the native ingester for a native platform in a worker thread (sync requests/subprocess). Patreon resolves a campaign id from the vanity URL; SubscribeStar's feed id is the creator URL itself. A campaign id we cannot resolve is a loud NOT_FOUND — never a silent empty success. `resolved_campaign_id` is non-None only when a lookup ran this call, so phase 3 caches it the way the old gallery-dl retry did. """ platform = ctx["platform"] overrides = ctx["config_overrides"] or {} campaign_id, resolved_campaign_id = await _resolve_native_campaign_id( platform, ctx["url"], ctx["cookies_path"], overrides ) if not campaign_id: # Patreon: vanity lookup failed. Pixiv: no numeric user id in the URL. # (SubscribeStar's campaign id is the URL itself — never lands here.) url = ctx["url"] return ( DownloadResult( success=False, url=url, artist_slug=ctx["artist_slug"], platform=platform, error_type=ErrorType.NOT_FOUND, error_message=_campaign_resolution_error(platform, url), ), None, ) # Honor the operator's existing rate-limit knobs on the native path (plan # #703): the global download_rate_limit_seconds (gallery-dl's `rate_limit`, # here on the gdl service) paces media downloads; page fetches use the # per-source sleep_request override, else `max(0.5, rate_limit/4)` — the same # API-pacing default gallery-dl applied as its `sleep-request`. rate_limit = gdl._rate_limit request_sleep = ( source_config.sleep_request if source_config.sleep_request is not None else max(0.5, rate_limit / 4) ) ingester = _native_ingester_cls(platform)( images_root=gdl.images_root, cookies_path=ctx["cookies_path"], session_factory=sync_session_factory, validate=gdl._validate_files, rate_limit=rate_limit, request_sleep=request_sleep, # Uniform across adapters: token platforms (pixiv) authenticate with # it, cookie platforms accept-and-ignore — so this construction stays # platform-agnostic. auth_token=ctx["auth_token"], ) loop = asyncio.get_running_loop() dl_result = await loop.run_in_executor( None, lambda: ingester.run( source_id=ctx["source_id"], campaign_id=campaign_id, artist_slug=ctx["artist_slug"], url=ctx["url"], mode=mode, resume_cursor=source_config.resume_cursor, time_budget_seconds=source_config.timeout, posts_base=int(overrides.get("_backfill_posts", 0)), # plan #709: live progress writes to this running event mid-walk. event_id=ctx.get("event_id"), ), ) return dl_result, resolved_campaign_id async def verify_source_credential( *, platform: str, url: str, artist_slug: str, config_overrides: dict | None, cookies_path: str | None, auth_token: str | None, images_root: Path, ) -> tuple[bool | None, str]: """Uniform credential probe across backends. Returns `(ok, message)`: True = authenticated, False = rejected, None = inconclusive (drift / network / nothing to test). Callers don't branch on platform — they call this and render the result. """ if uses_native_ingester(platform): # Native ingester platforms verify via their own lightweight auth probe. # SubscribeStar's probe takes the creator URL directly; Patreon's # resolves the campaign id first; Pixiv's is one OAuth refresh (the # exact call that fails when the token is bad — no feed walk). if platform == "subscribestar": from .subscribestar_ingester import verify_subscribestar_credential return await verify_subscribestar_credential(url, cookies_path, config_overrides) if platform == "pixiv": from .pixiv_ingester import verify_pixiv_credential return await verify_pixiv_credential(auth_token) from .patreon_ingester import verify_patreon_credential return await verify_patreon_credential(url, cookies_path, config_overrides) # gallery-dl platforms: --simulate one item; the extractor errors before it # can list if auth is bad. from .gallery_dl import GalleryDLService, SourceConfig gdl = GalleryDLService(images_root=images_root) return await gdl.verify( url=url, artist_slug=artist_slug, platform=platform, source_config=SourceConfig.from_dict(config_overrides or {}), cookies_path=cookies_path, auth_token=auth_token, )