242 lines
9.2 KiB
Python
242 lines
9.2 KiB
Python
"""Platform → download-backend dispatch (one place that knows which platforms
|
|
are served by the native FC ingester vs. the gallery-dl subprocess).
|
|
|
|
gallery-dl wasn't built to be driven by an automated scheduler — no native
|
|
checkpoint/resume, no structured logs, per-file HEADs that dominate wall-clock.
|
|
The native ingester (services/patreon_ingester.py, plan #697) replaces it for
|
|
Patreon and is the path we grow as more platforms migrate. To keep that
|
|
migration DRY, every caller that has to behave differently per backend —
|
|
download routing, the credential-verify probe, cursor handling — asks THIS
|
|
module instead of testing ``platform == "patreon"`` inline. When a platform gets
|
|
a native ingester, it moves into ``NATIVE_INGESTER_PLATFORMS`` here and both the
|
|
download path and verify switch over together.
|
|
|
|
The backend surfaces share a UNIFORM signature so a caller invokes the same
|
|
function regardless of platform:
|
|
- verify_credential(...) → (ok: bool|None, message: str)
|
|
- (download stays in download_service for now; uses_native_ingester() is the
|
|
shared predicate it routes on, so the decision lives here too.)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
from pathlib import Path
|
|
|
|
from .gallery_dl import DownloadResult, ErrorType
|
|
from .patreon_ingester import PatreonIngester
|
|
from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source
|
|
|
|
# Platforms whose download + verify go through the native ingester rather than
|
|
# gallery-dl. gallery-dl still serves every other platform (subscribestar,
|
|
# hentaifoundry, discord, pixiv, deviantart) unchanged.
|
|
NATIVE_INGESTER_PLATFORMS = frozenset({"patreon"})
|
|
|
|
# Mirrors patreon_resolver._CAMPAIGNS_URL — surfaced in resolution-failure
|
|
# messages so the operator sees the exact lookup endpoint that was hit.
|
|
_CAMPAIGNS_API = "https://www.patreon.com/api/campaigns"
|
|
|
|
|
|
def uses_native_ingester(platform: str) -> bool:
|
|
"""True when `platform` is served by the native ingester (not gallery-dl).
|
|
The single predicate the download path and verify both route on."""
|
|
return platform in NATIVE_INGESTER_PLATFORMS
|
|
|
|
|
|
async def run_download(
|
|
*,
|
|
ctx: dict,
|
|
source_config,
|
|
skip_value: bool | str,
|
|
mode: str | None,
|
|
gdl,
|
|
sync_session_factory,
|
|
) -> tuple[DownloadResult, str | None]:
|
|
"""Uniform download across backends — the download counterpart to
|
|
`verify_source_credential`, so this module is the ONE place that knows how
|
|
each platform both downloads AND verifies (the seam that makes adding a
|
|
platform a bounded job).
|
|
|
|
Returns `(DownloadResult, resolved_campaign_id)`; `resolved_campaign_id` is
|
|
non-None only when a native vanity lookup ran this call (so phase 3 caches
|
|
it). Native platforms route through their ingester in `mode`
|
|
(tick/backfill/recovery); gallery-dl platforms run the subprocess. The caller
|
|
(download_service) prepares `source_config`/`skip_value`/`mode` from the
|
|
backfill state machine and owns phase 3.
|
|
"""
|
|
platform = ctx["platform"]
|
|
if uses_native_ingester(platform):
|
|
return await _run_native_ingester(
|
|
ctx, source_config, mode, gdl, sync_session_factory
|
|
)
|
|
result = await gdl.download(
|
|
url=ctx["url"],
|
|
artist_slug=ctx["artist_slug"],
|
|
platform=platform,
|
|
source_config=source_config,
|
|
cookies_path=ctx["cookies_path"],
|
|
auth_token=ctx["auth_token"],
|
|
skip_value=skip_value,
|
|
)
|
|
return result, None
|
|
|
|
|
|
async def _run_native_ingester(
|
|
ctx: dict, source_config, mode: str | None, gdl, sync_session_factory,
|
|
) -> tuple[DownloadResult, str | None]:
|
|
"""Patreon (today the only native platform): resolve the campaign id, then run
|
|
the native ingester in a worker thread (it is sync requests/subprocess).
|
|
|
|
`resolved_campaign_id` is non-None only when we had to look it up from the
|
|
vanity URL this run, so phase 3 caches it the way the old gallery-dl retry
|
|
did. A campaign id we cannot resolve is a loud NOT_FOUND — never a silent
|
|
empty success.
|
|
"""
|
|
overrides = ctx["config_overrides"] or {}
|
|
campaign_id, resolved_campaign_id = await resolve_campaign_id_for_source(
|
|
ctx["url"], ctx["cookies_path"], overrides
|
|
)
|
|
if not campaign_id:
|
|
url = ctx["url"]
|
|
vanity = extract_vanity(url)
|
|
return (
|
|
DownloadResult(
|
|
success=False,
|
|
url=url,
|
|
artist_slug=ctx["artist_slug"],
|
|
platform="patreon",
|
|
error_type=ErrorType.NOT_FOUND,
|
|
error_message=(
|
|
f"Could not resolve Patreon campaign id. source_url={url!r}; "
|
|
f"vanity={vanity!r}; "
|
|
f"lookup=GET {_CAMPAIGNS_API}?filter[vanity]={vanity or ''} "
|
|
"(vanity lookup failed — cookies expired or creator moved?)"
|
|
),
|
|
),
|
|
None,
|
|
)
|
|
|
|
# Honor the operator's existing rate-limit knobs on the native path (plan
|
|
# #703): the global download_rate_limit_seconds (gallery-dl's `rate_limit`,
|
|
# here on the gdl service) paces media downloads; page fetches use the
|
|
# per-source sleep_request override, else `max(0.5, rate_limit/4)` — the same
|
|
# API-pacing default gallery-dl applied as its `sleep-request`.
|
|
rate_limit = gdl._rate_limit
|
|
request_sleep = (
|
|
source_config.sleep_request
|
|
if source_config.sleep_request is not None
|
|
else max(0.5, rate_limit / 4)
|
|
)
|
|
ingester = PatreonIngester(
|
|
images_root=gdl.images_root,
|
|
cookies_path=ctx["cookies_path"],
|
|
session_factory=sync_session_factory,
|
|
validate=gdl._validate_files,
|
|
rate_limit=rate_limit,
|
|
request_sleep=request_sleep,
|
|
)
|
|
loop = asyncio.get_running_loop()
|
|
dl_result = await loop.run_in_executor(
|
|
None,
|
|
lambda: ingester.run(
|
|
source_id=ctx["source_id"],
|
|
campaign_id=campaign_id,
|
|
artist_slug=ctx["artist_slug"],
|
|
url=ctx["url"],
|
|
mode=mode,
|
|
resume_cursor=source_config.resume_cursor,
|
|
time_budget_seconds=source_config.timeout,
|
|
posts_base=int(overrides.get("_backfill_posts", 0)),
|
|
# plan #709: live progress writes to this running event mid-walk.
|
|
event_id=ctx.get("event_id"),
|
|
),
|
|
)
|
|
return dl_result, resolved_campaign_id
|
|
|
|
|
|
async def preview_source(
|
|
*,
|
|
platform: str,
|
|
url: str,
|
|
source_id: int,
|
|
config_overrides: dict | None,
|
|
cookies_path: str | None,
|
|
images_root: Path,
|
|
sync_session_factory,
|
|
page_limit: int = 3,
|
|
) -> dict:
|
|
"""Dry-run preview for a native platform (plan #708 B4): resolve the campaign
|
|
id, then walk a few pages counting media not already seen/dead — no download.
|
|
|
|
Returns the preview dict (total_new / posts_scanned / pages_scanned /
|
|
has_more / sample), or `{"error": msg}` on a resolve / auth / drift failure.
|
|
Native-only — the caller gates on `uses_native_ingester`.
|
|
"""
|
|
import asyncio
|
|
|
|
from .patreon_client import PatreonAPIError
|
|
|
|
campaign_id, _ = await resolve_campaign_id_for_source(
|
|
url, cookies_path, config_overrides or {}
|
|
)
|
|
if not campaign_id:
|
|
vanity = extract_vanity(url)
|
|
return {
|
|
"error": (
|
|
f"Couldn't resolve the campaign id. source_url={url!r}; "
|
|
f"vanity={vanity!r}; lookup=GET {_CAMPAIGNS_API}?filter[vanity]={vanity or ''} "
|
|
"(cookies expired, or the creator moved/renamed?)."
|
|
)
|
|
}
|
|
ingester = PatreonIngester(
|
|
images_root=images_root,
|
|
cookies_path=cookies_path,
|
|
session_factory=sync_session_factory,
|
|
)
|
|
loop = asyncio.get_running_loop()
|
|
try:
|
|
result = await loop.run_in_executor(
|
|
None,
|
|
lambda: ingester.preview(source_id, campaign_id, page_limit=page_limit),
|
|
)
|
|
except PatreonAPIError as exc:
|
|
return {"error": f"Couldn't preview: {exc}"}
|
|
return result
|
|
|
|
|
|
async def verify_source_credential(
|
|
*,
|
|
platform: str,
|
|
url: str,
|
|
artist_slug: str,
|
|
config_overrides: dict | None,
|
|
cookies_path: str | None,
|
|
auth_token: str | None,
|
|
images_root: Path,
|
|
) -> tuple[bool | None, str]:
|
|
"""Uniform credential probe across backends. Returns `(ok, message)`:
|
|
True = authenticated, False = rejected, None = inconclusive (drift /
|
|
network / nothing to test). Callers don't branch on platform — they call
|
|
this and render the result.
|
|
"""
|
|
if uses_native_ingester(platform):
|
|
# Native ingester platforms verify via their own lightweight auth probe
|
|
# (resolve campaign id + one authenticated API page). Patreon today.
|
|
from .patreon_ingester import verify_patreon_credential
|
|
|
|
return await verify_patreon_credential(url, cookies_path, config_overrides)
|
|
|
|
# gallery-dl platforms: --simulate one item; the extractor errors before it
|
|
# can list if auth is bad.
|
|
from .gallery_dl import GalleryDLService, SourceConfig
|
|
|
|
gdl = GalleryDLService(images_root=images_root)
|
|
return await gdl.verify(
|
|
url=url,
|
|
artist_slug=artist_slug,
|
|
platform=platform,
|
|
source_config=SourceConfig.from_dict(config_overrides or {}),
|
|
cookies_path=cookies_path,
|
|
auth_token=auth_token,
|
|
)
|