FabledCurator/backend/app/services/download_backends.py

"""Platform → download-backend dispatch (one place that knows which platforms
are served by the native FC ingester vs. the gallery-dl subprocess).

gallery-dl wasn't built to be driven by an automated scheduler — no native
checkpoint/resume, no structured logs, per-file HEADs that dominate wall-clock.
The native ingester (services/patreon_ingester.py, plan #697) replaces it for
Patreon and is the path we grow as more platforms migrate. To keep that
migration DRY, every caller that has to behave differently per backend —
download routing, the credential-verify probe, cursor handling — asks THIS
module instead of testing ``platform == "patreon"`` inline. When a platform gets
a native ingester, it moves into ``NATIVE_INGESTER_PLATFORMS`` here and both the
download path and verify switch over together.

The backend surfaces share a UNIFORM signature so a caller invokes the same
function regardless of platform:
  - verify_credential(...)  → (ok: bool|None, message: str)
  - (download stays in download_service for now; uses_native_ingester() is the
    shared predicate it routes on, so the decision lives here too.)
"""

from __future__ import annotations

import asyncio
from pathlib import Path

from .gallery_dl import DownloadResult, ErrorType
from .patreon_ingester import PatreonIngester
from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source
from .pixiv_client import user_id_from_url
from .pixiv_ingester import PixivIngester
from .platforms import known_platform_keys
from .subscribestar_ingester import SubscribeStarIngester

# Platforms whose download + verify go through the native ingester rather than
# gallery-dl. gallery-dl still serves the rest (hentaifoundry, discord) until
# they migrate too. pixiv left this set when it was retired (milestone #406).
NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar"})


def _unsupported_platform_message(platform: str) -> str | None:
    """Why `platform` may not be downloaded or verified, or None if it may.

    A source can outlive its platform. Retiring one (DeviantArt #3069, pixiv
    #406) unregisters it, but its `Source` rows — and the `enabled` flag on
    them — are data, and data survives a deploy. So this refuses at the two
    functions every download and every credential probe pass through, instead
    of trusting the scheduler's `enabled` filter and every future caller to
    agree.

    Without it a retired platform does not fail: it falls through to the
    gallery-dl branch, which is precisely where a platform lands once it is no
    longer native — and gallery-dl still has an extractor for it.
    """
    if platform in known_platform_keys():
        return None
    return f"{platform!r} is not a supported platform (retired or unknown)"

# Mirrors patreon_resolver._CAMPAIGNS_URL — surfaced in resolution-failure
# messages so the operator sees the exact lookup endpoint that was hit.
_CAMPAIGNS_API = "https://www.patreon.com/api/campaigns"


def _native_ingester_cls(platform: str):
    """The native ingester class for `platform` (uniform constructor signature).
    A call-time lookup (not a module-level dict captured at import) so tests can
    monkeypatch db_mod.PatreonIngester / SubscribeStarIngester and have the
    dispatch pick up the replacement."""
    return {
        "patreon": PatreonIngester,
        "pixiv": PixivIngester,
        "subscribestar": SubscribeStarIngester,
    }[platform]


def uses_native_ingester(platform: str) -> bool:
    """True when `platform` is served by the native ingester (not gallery-dl).
    The single predicate the download path and verify both route on."""
    return platform in NATIVE_INGESTER_PLATFORMS


async def run_download(
    *,
    ctx: dict,
    source_config,
    skip_value: bool | str,
    mode: str | None,
    gdl,
    sync_session_factory,
) -> tuple[DownloadResult, str | None]:
    """Uniform download across backends — the download counterpart to
    `verify_source_credential`, so this module is the ONE place that knows how
    each platform both downloads AND verifies (the seam that makes adding a
    platform a bounded job).

    Returns `(DownloadResult, resolved_campaign_id)`; `resolved_campaign_id` is
    non-None only when a native vanity lookup ran this call (so phase 3 caches
    it). Native platforms route through their ingester in `mode`
    (tick/backfill/recovery); gallery-dl platforms run the subprocess. The caller
    (download_service) prepares `source_config`/`skip_value`/`mode` from the
    backfill state machine and owns phase 3.
    """
    platform = ctx["platform"]
    refusal = _unsupported_platform_message(platform)
    if refusal is not None:
        return DownloadResult(
            success=False, url=ctx["url"], artist_slug=ctx["artist_slug"],
            platform=platform,
            error_type=ErrorType.UNSUPPORTED_URL, error_message=refusal,
        ), None
    if uses_native_ingester(platform):
        return await _run_native_ingester(
            ctx, source_config, mode, gdl, sync_session_factory
        )
    result = await gdl.download(
        url=ctx["url"],
        artist_slug=ctx["artist_slug"],
        platform=platform,
        source_config=source_config,
        cookies_path=ctx["cookies_path"],
        auth_token=ctx["auth_token"],
        skip_value=skip_value,
    )
    return result, None


async def _resolve_native_campaign_id(
    platform: str, url: str, cookies_path: str | None, overrides: dict,
) -> tuple[str | None, str | None]:
    """`(campaign_id, resolved_campaign_id)` for a native source. SubscribeStar's
    feed id IS the creator URL; Pixiv's is the numeric user id parsed straight
    from it (no lookup → resolved None either way). Patreon resolves the
    campaign id from the vanity URL (resolved non-None when a lookup actually ran,
    so phase 3 caches it)."""
    if platform == "subscribestar":
        return url, None
    if platform == "pixiv":
        return user_id_from_url(url), None
    return await resolve_campaign_id_for_source(url, cookies_path, overrides)


def _campaign_resolution_error(platform: str, url: str) -> str:
    """Operator-facing message for a native source whose campaign id could not
    be resolved — names the platform's own lookup mechanism."""
    if platform == "pixiv":
        return (
            f"Could not extract a pixiv user id. source_url={url!r} — expected "
            "a URL like https://www.pixiv.net/users/<id>."
        )
    vanity = extract_vanity(url)
    return (
        f"Could not resolve Patreon campaign id. source_url={url!r}; "
        f"vanity={vanity!r}; "
        f"lookup=GET {_CAMPAIGNS_API}?filter[vanity]={vanity or ''} "
        "(vanity lookup failed — cookies expired or creator moved?)"
    )


async def _run_native_ingester(
    ctx: dict, source_config, mode: str | None, gdl, sync_session_factory,
) -> tuple[DownloadResult, str | None]:
    """Run the native ingester for a native platform in a worker thread (sync
    requests/subprocess). Patreon resolves a campaign id from the vanity URL;
    SubscribeStar's feed id is the creator URL itself. A campaign id we cannot
    resolve is a loud NOT_FOUND — never a silent empty success.

    `resolved_campaign_id` is non-None only when a lookup ran this call, so phase
    3 caches it the way the old gallery-dl retry did.
    """
    platform = ctx["platform"]
    overrides = ctx["config_overrides"] or {}
    campaign_id, resolved_campaign_id = await _resolve_native_campaign_id(
        platform, ctx["url"], ctx["cookies_path"], overrides
    )
    if not campaign_id:
        # Patreon: vanity lookup failed. Pixiv: no numeric user id in the URL.
        # (SubscribeStar's campaign id is the URL itself — never lands here.)
        url = ctx["url"]
        return (
            DownloadResult(
                success=False,
                url=url,
                artist_slug=ctx["artist_slug"],
                platform=platform,
                error_type=ErrorType.NOT_FOUND,
                error_message=_campaign_resolution_error(platform, url),
            ),
            None,
        )

    # Honor the operator's existing rate-limit knobs on the native path (plan
    # #703): the global download_rate_limit_seconds (gallery-dl's `rate_limit`,
    # here on the gdl service) paces media downloads; page fetches use the
    # per-source sleep_request override, else `max(0.5, rate_limit/4)` — the same
    # API-pacing default gallery-dl applied as its `sleep-request`.
    rate_limit = gdl._rate_limit
    request_sleep = (
        source_config.sleep_request
        if source_config.sleep_request is not None
        else max(0.5, rate_limit / 4)
    )
    ingester = _native_ingester_cls(platform)(
        images_root=gdl.images_root,
        cookies_path=ctx["cookies_path"],
        session_factory=sync_session_factory,
        validate=gdl._validate_files,
        rate_limit=rate_limit,
        request_sleep=request_sleep,
        # Uniform across adapters: token platforms (pixiv) authenticate with
        # it, cookie platforms accept-and-ignore — so this construction stays
        # platform-agnostic.
        auth_token=ctx["auth_token"],
    )
    loop = asyncio.get_running_loop()
    dl_result = await loop.run_in_executor(
        None,
        lambda: ingester.run(
            source_id=ctx["source_id"],
            campaign_id=campaign_id,
            artist_slug=ctx["artist_slug"],
            url=ctx["url"],
            mode=mode,
            resume_cursor=source_config.resume_cursor,
            time_budget_seconds=source_config.timeout,
            posts_base=int(overrides.get("_backfill_posts", 0)),
            # plan #709: live progress writes to this running event mid-walk.
            event_id=ctx.get("event_id"),
        ),
    )
    return dl_result, resolved_campaign_id


async def verify_source_credential(
    *,
    platform: str,
    url: str,
    artist_slug: str,
    config_overrides: dict | None,
    cookies_path: str | None,
    auth_token: str | None,
    images_root: Path,
) -> tuple[bool | None, str]:
    """Uniform credential probe across backends. Returns `(ok, message)`:
    True = authenticated, False = rejected, None = inconclusive (drift /
    network / nothing to test). Callers don't branch on platform — they call
    this and render the result.
    """
    refusal = _unsupported_platform_message(platform)
    if refusal is not None:
        # Inconclusive rather than False: nothing was probed, so nothing was
        # rejected. False would tell the operator their credential is bad.
        return None, refusal
    if uses_native_ingester(platform):
        # Native ingester platforms verify via their own lightweight auth probe.
        # SubscribeStar's probe takes the creator URL directly; Patreon's
        # resolves the campaign id first; Pixiv's is one OAuth refresh (the
        # exact call that fails when the token is bad — no feed walk).
        if platform == "subscribestar":
            from .subscribestar_ingester import verify_subscribestar_credential

            return await verify_subscribestar_credential(url, cookies_path, config_overrides)
        if platform == "pixiv":
            from .pixiv_ingester import verify_pixiv_credential

            return await verify_pixiv_credential(auth_token)
        from .patreon_ingester import verify_patreon_credential

        return await verify_patreon_credential(url, cookies_path, config_overrides)

    # gallery-dl platforms: --simulate one item; the extractor errors before it
    # can list if auth is bad.
    from .gallery_dl import GalleryDLService, SourceConfig

    gdl = GalleryDLService(images_root=images_root)
    return await gdl.verify(
        url=url,
        artist_slug=artist_slug,
        platform=platform,
        source_config=SourceConfig.from_dict(config_overrides or {}),
        cookies_path=cookies_path,
        auth_token=auth_token,
    )