FabledCurator/backend/app/services/download_backends.py

"""Platform → download-backend dispatch (one place that knows which platforms
are served by the native FC ingester vs. the gallery-dl subprocess).

gallery-dl wasn't built to be driven by an automated scheduler — no native
checkpoint/resume, no structured logs, per-file HEADs that dominate wall-clock.
The native ingester (services/patreon_ingester.py, plan #697) replaces it for
Patreon and is the path we grow as more platforms migrate. To keep that
migration DRY, every caller that has to behave differently per backend —
download routing, the credential-verify probe, cursor handling — asks THIS
module instead of testing ``platform == "patreon"`` inline. When a platform gets
a native ingester, it moves into ``NATIVE_INGESTER_PLATFORMS`` here and both the
download path and verify switch over together.

The backend surfaces share a UNIFORM signature so a caller invokes the same
function regardless of platform:
  - verify_credential(...)  → (ok: bool|None, message: str)
  - (download stays in download_service for now; uses_native_ingester() is the
    shared predicate it routes on, so the decision lives here too.)
"""

from __future__ import annotations

import asyncio
from pathlib import Path

from .gallery_dl import DownloadResult, ErrorType
from .native_ingest_common import NativeIngestError
from .patreon_ingester import PatreonIngester
from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source
from .subscribestar_ingester import SubscribeStarIngester

# Platforms whose download + verify go through the native ingester rather than
# gallery-dl. gallery-dl still serves the rest (hentaifoundry, discord, pixiv,
# deviantart) until they migrate too.
NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar"})

# Native ingester classes keyed by platform (uniform constructor signature, so
# _run_native_ingester / preview build whichever the source needs).
_NATIVE_INGESTERS = {
    "patreon": PatreonIngester,
    "subscribestar": SubscribeStarIngester,
}

# Mirrors patreon_resolver._CAMPAIGNS_URL — surfaced in resolution-failure
# messages so the operator sees the exact lookup endpoint that was hit.
_CAMPAIGNS_API = "https://www.patreon.com/api/campaigns"


def uses_native_ingester(platform: str) -> bool:
    """True when `platform` is served by the native ingester (not gallery-dl).
    The single predicate the download path and verify both route on."""
    return platform in NATIVE_INGESTER_PLATFORMS


async def run_download(
    *,
    ctx: dict,
    source_config,
    skip_value: bool | str,
    mode: str | None,
    gdl,
    sync_session_factory,
) -> tuple[DownloadResult, str | None]:
    """Uniform download across backends — the download counterpart to
    `verify_source_credential`, so this module is the ONE place that knows how
    each platform both downloads AND verifies (the seam that makes adding a
    platform a bounded job).

    Returns `(DownloadResult, resolved_campaign_id)`; `resolved_campaign_id` is
    non-None only when a native vanity lookup ran this call (so phase 3 caches
    it). Native platforms route through their ingester in `mode`
    (tick/backfill/recovery); gallery-dl platforms run the subprocess. The caller
    (download_service) prepares `source_config`/`skip_value`/`mode` from the
    backfill state machine and owns phase 3.
    """
    platform = ctx["platform"]
    if uses_native_ingester(platform):
        return await _run_native_ingester(
            ctx, source_config, mode, gdl, sync_session_factory
        )
    result = await gdl.download(
        url=ctx["url"],
        artist_slug=ctx["artist_slug"],
        platform=platform,
        source_config=source_config,
        cookies_path=ctx["cookies_path"],
        auth_token=ctx["auth_token"],
        skip_value=skip_value,
    )
    return result, None


async def _resolve_native_campaign_id(
    platform: str, url: str, cookies_path: str | None, overrides: dict,
) -> tuple[str | None, str | None]:
    """`(campaign_id, resolved_campaign_id)` for a native source. SubscribeStar's
    feed id IS the creator URL (no lookup → resolved None). Patreon resolves the
    campaign id from the vanity URL (resolved non-None when a lookup actually ran,
    so phase 3 caches it)."""
    if platform == "subscribestar":
        return url, None
    return await resolve_campaign_id_for_source(url, cookies_path, overrides)


async def _run_native_ingester(
    ctx: dict, source_config, mode: str | None, gdl, sync_session_factory,
) -> tuple[DownloadResult, str | None]:
    """Run the native ingester for a native platform in a worker thread (sync
    requests/subprocess). Patreon resolves a campaign id from the vanity URL;
    SubscribeStar's feed id is the creator URL itself. A campaign id we cannot
    resolve is a loud NOT_FOUND — never a silent empty success.

    `resolved_campaign_id` is non-None only when a lookup ran this call, so phase
    3 caches it the way the old gallery-dl retry did.
    """
    platform = ctx["platform"]
    overrides = ctx["config_overrides"] or {}
    campaign_id, resolved_campaign_id = await _resolve_native_campaign_id(
        platform, ctx["url"], ctx["cookies_path"], overrides
    )
    if not campaign_id:
        # Only reachable for Patreon (SubscribeStar's campaign id is the URL).
        url = ctx["url"]
        vanity = extract_vanity(url)
        return (
            DownloadResult(
                success=False,
                url=url,
                artist_slug=ctx["artist_slug"],
                platform=platform,
                error_type=ErrorType.NOT_FOUND,
                error_message=(
                    f"Could not resolve Patreon campaign id. source_url={url!r}; "
                    f"vanity={vanity!r}; "
                    f"lookup=GET {_CAMPAIGNS_API}?filter[vanity]={vanity or ''} "
                    "(vanity lookup failed — cookies expired or creator moved?)"
                ),
            ),
            None,
        )

    # Honor the operator's existing rate-limit knobs on the native path (plan
    # #703): the global download_rate_limit_seconds (gallery-dl's `rate_limit`,
    # here on the gdl service) paces media downloads; page fetches use the
    # per-source sleep_request override, else `max(0.5, rate_limit/4)` — the same
    # API-pacing default gallery-dl applied as its `sleep-request`.
    rate_limit = gdl._rate_limit
    request_sleep = (
        source_config.sleep_request
        if source_config.sleep_request is not None
        else max(0.5, rate_limit / 4)
    )
    ingester = _NATIVE_INGESTERS[platform](
        images_root=gdl.images_root,
        cookies_path=ctx["cookies_path"],
        session_factory=sync_session_factory,
        validate=gdl._validate_files,
        rate_limit=rate_limit,
        request_sleep=request_sleep,
    )
    loop = asyncio.get_running_loop()
    dl_result = await loop.run_in_executor(
        None,
        lambda: ingester.run(
            source_id=ctx["source_id"],
            campaign_id=campaign_id,
            artist_slug=ctx["artist_slug"],
            url=ctx["url"],
            mode=mode,
            resume_cursor=source_config.resume_cursor,
            time_budget_seconds=source_config.timeout,
            posts_base=int(overrides.get("_backfill_posts", 0)),
            # plan #709: live progress writes to this running event mid-walk.
            event_id=ctx.get("event_id"),
        ),
    )
    return dl_result, resolved_campaign_id


async def preview_source(
    *,
    platform: str,
    url: str,
    source_id: int,
    config_overrides: dict | None,
    cookies_path: str | None,
    images_root: Path,
    sync_session_factory,
    page_limit: int = 3,
) -> dict:
    """Dry-run preview for a native platform (plan #708 B4): resolve the campaign
    id, then walk a few pages counting media not already seen/dead — no download.

    Returns the preview dict (total_new / posts_scanned / pages_scanned /
    has_more / sample), or `{"error": msg}` on a resolve / auth / drift failure.
    Native-only — the caller gates on `uses_native_ingester`.
    """
    import asyncio

    campaign_id, _ = await _resolve_native_campaign_id(
        platform, url, cookies_path, config_overrides or {}
    )
    if not campaign_id:
        vanity = extract_vanity(url)
        return {
            "error": (
                f"Couldn't resolve the campaign id. source_url={url!r}; "
                f"vanity={vanity!r}; lookup=GET {_CAMPAIGNS_API}?filter[vanity]={vanity or ''} "
                "(cookies expired, or the creator moved/renamed?)."
            )
        }
    ingester = _NATIVE_INGESTERS[platform](
        images_root=images_root,
        cookies_path=cookies_path,
        session_factory=sync_session_factory,
    )
    loop = asyncio.get_running_loop()
    try:
        result = await loop.run_in_executor(
            None,
            lambda: ingester.preview(source_id, campaign_id, page_limit=page_limit),
        )
    except NativeIngestError as exc:
        return {"error": f"Couldn't preview: {exc}"}
    return result


async def verify_source_credential(
    *,
    platform: str,
    url: str,
    artist_slug: str,
    config_overrides: dict | None,
    cookies_path: str | None,
    auth_token: str | None,
    images_root: Path,
) -> tuple[bool | None, str]:
    """Uniform credential probe across backends. Returns `(ok, message)`:
    True = authenticated, False = rejected, None = inconclusive (drift /
    network / nothing to test). Callers don't branch on platform — they call
    this and render the result.
    """
    if uses_native_ingester(platform):
        # Native ingester platforms verify via their own lightweight auth probe
        # (one authenticated feed fetch). SubscribeStar's probe takes the creator
        # URL directly; Patreon's resolves the campaign id first.
        if platform == "subscribestar":
            from .subscribestar_ingester import verify_subscribestar_credential

            return await verify_subscribestar_credential(url, cookies_path, config_overrides)
        from .patreon_ingester import verify_patreon_credential

        return await verify_patreon_credential(url, cookies_path, config_overrides)

    # gallery-dl platforms: --simulate one item; the extractor errors before it
    # can list if auth is bad.
    from .gallery_dl import GalleryDLService, SourceConfig

    gdl = GalleryDLService(images_root=images_root)
    return await gdl.verify(
        url=url,
        artist_slug=artist_slug,
        platform=platform,
        source_config=SourceConfig.from_dict(config_overrides or {}),
        cookies_path=cookies_path,
        auth_token=auth_token,
    )