FabledCurator/backend/app/services/download_backends.py

"""Platform → download-backend dispatch (one place that knows which platforms
are served by the native FC ingester vs. the gallery-dl subprocess).

gallery-dl wasn't built to be driven by an automated scheduler — no native
checkpoint/resume, no structured logs, per-file HEADs that dominate wall-clock.
The native ingester (services/patreon_ingester.py, plan #697) replaces it for
Patreon and is the path we grow as more platforms migrate. To keep that
migration DRY, every caller that has to behave differently per backend —
download routing, the credential-verify probe, cursor handling — asks THIS
module instead of testing ``platform == "patreon"`` inline. When a platform gets
a native ingester, it moves into ``NATIVE_INGESTER_PLATFORMS`` here and both the
download path and verify switch over together.

The backend surfaces share a UNIFORM signature so a caller invokes the same
function regardless of platform:
  - verify_credential(...)  → (ok: bool|None, message: str)
  - (download stays in download_service for now; uses_native_ingester() is the
    shared predicate it routes on, so the decision lives here too.)
"""

from __future__ import annotations

import asyncio
from pathlib import Path

from .gallery_dl import DownloadResult, ErrorType
from .patreon_ingester import PatreonIngester
from .patreon_resolver import resolve_campaign_id_for_source

# Platforms whose download + verify go through the native ingester rather than
# gallery-dl. gallery-dl still serves every other platform (subscribestar,
# hentaifoundry, discord, pixiv, deviantart) unchanged.
NATIVE_INGESTER_PLATFORMS = frozenset({"patreon"})


def uses_native_ingester(platform: str) -> bool:
    """True when `platform` is served by the native ingester (not gallery-dl).
    The single predicate the download path and verify both route on."""
    return platform in NATIVE_INGESTER_PLATFORMS


async def run_download(
    *,
    ctx: dict,
    source_config,
    skip_value: bool | str,
    mode: str | None,
    gdl,
    sync_session_factory,
) -> tuple[DownloadResult, str | None]:
    """Uniform download across backends — the download counterpart to
    `verify_source_credential`, so this module is the ONE place that knows how
    each platform both downloads AND verifies (the seam that makes adding a
    platform a bounded job).

    Returns `(DownloadResult, resolved_campaign_id)`; `resolved_campaign_id` is
    non-None only when a native vanity lookup ran this call (so phase 3 caches
    it). Native platforms route through their ingester in `mode`
    (tick/backfill/recovery); gallery-dl platforms run the subprocess. The caller
    (download_service) prepares `source_config`/`skip_value`/`mode` from the
    backfill state machine and owns phase 3.
    """
    platform = ctx["platform"]
    if uses_native_ingester(platform):
        return await _run_native_ingester(
            ctx, source_config, mode, gdl, sync_session_factory
        )
    result = await gdl.download(
        url=ctx["url"],
        artist_slug=ctx["artist_slug"],
        platform=platform,
        source_config=source_config,
        cookies_path=ctx["cookies_path"],
        auth_token=ctx["auth_token"],
        skip_value=skip_value,
    )
    return result, None


async def _run_native_ingester(
    ctx: dict, source_config, mode: str | None, gdl, sync_session_factory,
) -> tuple[DownloadResult, str | None]:
    """Patreon (today the only native platform): resolve the campaign id, then run
    the native ingester in a worker thread (it is sync requests/subprocess).

    `resolved_campaign_id` is non-None only when we had to look it up from the
    vanity URL this run, so phase 3 caches it the way the old gallery-dl retry
    did. A campaign id we cannot resolve is a loud NOT_FOUND — never a silent
    empty success.
    """
    overrides = ctx["config_overrides"] or {}
    campaign_id, resolved_campaign_id = await resolve_campaign_id_for_source(
        ctx["url"], ctx["cookies_path"], overrides
    )
    if not campaign_id:
        return (
            DownloadResult(
                success=False,
                url=ctx["url"],
                artist_slug=ctx["artist_slug"],
                platform="patreon",
                error_type=ErrorType.NOT_FOUND,
                error_message=(
                    "Could not resolve Patreon campaign id from the source URL "
                    "(vanity lookup failed — cookies expired or creator moved?)"
                ),
            ),
            None,
        )

    # Honor the operator's existing rate-limit knobs on the native path (plan
    # #703): the global download_rate_limit_seconds (gallery-dl's `rate_limit`,
    # here on the gdl service) paces media downloads; page fetches use the
    # per-source sleep_request override, else `max(0.5, rate_limit/4)` — the same
    # API-pacing default gallery-dl applied as its `sleep-request`.
    rate_limit = gdl._rate_limit
    request_sleep = (
        source_config.sleep_request
        if source_config.sleep_request is not None
        else max(0.5, rate_limit / 4)
    )
    ingester = PatreonIngester(
        images_root=gdl.images_root,
        cookies_path=ctx["cookies_path"],
        session_factory=sync_session_factory,
        validate=gdl._validate_files,
        rate_limit=rate_limit,
        request_sleep=request_sleep,
    )
    loop = asyncio.get_running_loop()
    dl_result = await loop.run_in_executor(
        None,
        lambda: ingester.run(
            source_id=ctx["source_id"],
            campaign_id=campaign_id,
            artist_slug=ctx["artist_slug"],
            url=ctx["url"],
            mode=mode,
            resume_cursor=source_config.resume_cursor,
            time_budget_seconds=source_config.timeout,
            posts_base=int(overrides.get("_backfill_posts", 0)),
        ),
    )
    return dl_result, resolved_campaign_id


async def preview_source(
    *,
    platform: str,
    url: str,
    source_id: int,
    config_overrides: dict | None,
    cookies_path: str | None,
    images_root: Path,
    sync_session_factory,
    page_limit: int = 3,
) -> dict:
    """Dry-run preview for a native platform (plan #708 B4): resolve the campaign
    id, then walk a few pages counting media not already seen/dead — no download.

    Returns the preview dict (total_new / posts_scanned / pages_scanned /
    has_more / sample), or `{"error": msg}` on a resolve / auth / drift failure.
    Native-only — the caller gates on `uses_native_ingester`.
    """
    import asyncio

    from .patreon_client import PatreonAPIError

    campaign_id, _ = await resolve_campaign_id_for_source(
        url, cookies_path, config_overrides or {}
    )
    if not campaign_id:
        return {
            "error": (
                "Couldn't resolve the campaign id from the source URL "
                "(cookies expired, or the creator moved/renamed?)."
            )
        }
    ingester = PatreonIngester(
        images_root=images_root,
        cookies_path=cookies_path,
        session_factory=sync_session_factory,
    )
    loop = asyncio.get_running_loop()
    try:
        result = await loop.run_in_executor(
            None,
            lambda: ingester.preview(source_id, campaign_id, page_limit=page_limit),
        )
    except PatreonAPIError as exc:
        return {"error": f"Couldn't preview: {exc}"}
    return result


async def verify_source_credential(
    *,
    platform: str,
    url: str,
    artist_slug: str,
    config_overrides: dict | None,
    cookies_path: str | None,
    auth_token: str | None,
    images_root: Path,
) -> tuple[bool | None, str]:
    """Uniform credential probe across backends. Returns `(ok, message)`:
    True = authenticated, False = rejected, None = inconclusive (drift /
    network / nothing to test). Callers don't branch on platform — they call
    this and render the result.
    """
    if uses_native_ingester(platform):
        # Native ingester platforms verify via their own lightweight auth probe
        # (resolve campaign id + one authenticated API page). Patreon today.
        from .patreon_ingester import verify_patreon_credential

        return await verify_patreon_credential(url, cookies_path, config_overrides)

    # gallery-dl platforms: --simulate one item; the extractor errors before it
    # can list if auth is bad.
    from .gallery_dl import GalleryDLService, SourceConfig

    gdl = GalleryDLService(images_root=images_root)
    return await gdl.verify(
        url=url,
        artist_slug=artist_slug,
        platform=platform,
        source_config=SourceConfig.from_dict(config_overrides or {}),
        cookies_path=cookies_path,
        auth_token=auth_token,
    )