refactor(downloads): unify phase-2 dispatch in download_backends — A5 (plan #707)
Mirror verify_source_credential: download_backends.run_download is now the single download entry, so that module is the ONE registry of how each platform both downloads AND verifies — the seam that makes adding a platform a bounded job (write its adapter construction next to its verify). The native-ingester construction + campaign-id resolution moves out of download_service into download_backends._run_native_ingester. download_service.download_source drops its `if uses_native_ingester ... else gdl.download` branch and calls one `self._run_download(...)` (a thin delegate to run_download passing the service's gdl + sync sessionmaker). mode (tick/backfill/ recovery) is still chosen there from the backfill state machine and threaded through. Removed the now-unused PatreonIngester / resolve_campaign_id_for_source imports from download_service. Tests: the phase-2 stub seam moves from svc._run_patreon_ingester to svc._run_download (helper + the db-release test); the two native-construction tests repoint to download_backends.run_download (patching download_backends.resolve_campaign_id_for_source / PatreonIngester). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -20,8 +20,13 @@ function regardless of platform:
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
from .gallery_dl import DownloadResult, ErrorType
|
||||
from .patreon_ingester import PatreonIngester
|
||||
from .patreon_resolver import resolve_campaign_id_for_source
|
||||
|
||||
# Platforms whose download + verify go through the native ingester rather than
|
||||
# gallery-dl. gallery-dl still serves every other platform (subscribestar,
|
||||
# hentaifoundry, discord, pixiv, deviantart) unchanged.
|
||||
@@ -34,6 +39,111 @@ def uses_native_ingester(platform: str) -> bool:
|
||||
return platform in NATIVE_INGESTER_PLATFORMS
|
||||
|
||||
|
||||
async def run_download(
|
||||
*,
|
||||
ctx: dict,
|
||||
source_config,
|
||||
skip_value: bool | str,
|
||||
mode: str | None,
|
||||
gdl,
|
||||
sync_session_factory,
|
||||
) -> tuple[DownloadResult, str | None]:
|
||||
"""Uniform download across backends — the download counterpart to
|
||||
`verify_source_credential`, so this module is the ONE place that knows how
|
||||
each platform both downloads AND verifies (the seam that makes adding a
|
||||
platform a bounded job).
|
||||
|
||||
Returns `(DownloadResult, resolved_campaign_id)`; `resolved_campaign_id` is
|
||||
non-None only when a native vanity lookup ran this call (so phase 3 caches
|
||||
it). Native platforms route through their ingester in `mode`
|
||||
(tick/backfill/recovery); gallery-dl platforms run the subprocess. The caller
|
||||
(download_service) prepares `source_config`/`skip_value`/`mode` from the
|
||||
backfill state machine and owns phase 3.
|
||||
"""
|
||||
platform = ctx["platform"]
|
||||
if uses_native_ingester(platform):
|
||||
return await _run_native_ingester(
|
||||
ctx, source_config, mode, gdl, sync_session_factory
|
||||
)
|
||||
result = await gdl.download(
|
||||
url=ctx["url"],
|
||||
artist_slug=ctx["artist_slug"],
|
||||
platform=platform,
|
||||
source_config=source_config,
|
||||
cookies_path=ctx["cookies_path"],
|
||||
auth_token=ctx["auth_token"],
|
||||
skip_value=skip_value,
|
||||
)
|
||||
return result, None
|
||||
|
||||
|
||||
async def _run_native_ingester(
|
||||
ctx: dict, source_config, mode: str | None, gdl, sync_session_factory,
|
||||
) -> tuple[DownloadResult, str | None]:
|
||||
"""Patreon (today the only native platform): resolve the campaign id, then run
|
||||
the native ingester in a worker thread (it is sync requests/subprocess).
|
||||
|
||||
`resolved_campaign_id` is non-None only when we had to look it up from the
|
||||
vanity URL this run, so phase 3 caches it the way the old gallery-dl retry
|
||||
did. A campaign id we cannot resolve is a loud NOT_FOUND — never a silent
|
||||
empty success.
|
||||
"""
|
||||
overrides = ctx["config_overrides"] or {}
|
||||
campaign_id, resolved_campaign_id = await resolve_campaign_id_for_source(
|
||||
ctx["url"], ctx["cookies_path"], overrides
|
||||
)
|
||||
if not campaign_id:
|
||||
return (
|
||||
DownloadResult(
|
||||
success=False,
|
||||
url=ctx["url"],
|
||||
artist_slug=ctx["artist_slug"],
|
||||
platform="patreon",
|
||||
error_type=ErrorType.NOT_FOUND,
|
||||
error_message=(
|
||||
"Could not resolve Patreon campaign id from the source URL "
|
||||
"(vanity lookup failed — cookies expired or creator moved?)"
|
||||
),
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
# Honor the operator's existing rate-limit knobs on the native path (plan
|
||||
# #703): the global download_rate_limit_seconds (gallery-dl's `rate_limit`,
|
||||
# here on the gdl service) paces media downloads; page fetches use the
|
||||
# per-source sleep_request override, else `max(0.5, rate_limit/4)` — the same
|
||||
# API-pacing default gallery-dl applied as its `sleep-request`.
|
||||
rate_limit = gdl._rate_limit
|
||||
request_sleep = (
|
||||
source_config.sleep_request
|
||||
if source_config.sleep_request is not None
|
||||
else max(0.5, rate_limit / 4)
|
||||
)
|
||||
ingester = PatreonIngester(
|
||||
images_root=gdl.images_root,
|
||||
cookies_path=ctx["cookies_path"],
|
||||
session_factory=sync_session_factory,
|
||||
validate=gdl._validate_files,
|
||||
rate_limit=rate_limit,
|
||||
request_sleep=request_sleep,
|
||||
)
|
||||
loop = asyncio.get_running_loop()
|
||||
dl_result = await loop.run_in_executor(
|
||||
None,
|
||||
lambda: ingester.run(
|
||||
source_id=ctx["source_id"],
|
||||
campaign_id=campaign_id,
|
||||
artist_slug=ctx["artist_slug"],
|
||||
url=ctx["url"],
|
||||
mode=mode,
|
||||
resume_cursor=source_config.resume_cursor,
|
||||
time_budget_seconds=source_config.timeout,
|
||||
posts_base=int(overrides.get("_backfill_posts", 0)),
|
||||
),
|
||||
)
|
||||
return dl_result, resolved_campaign_id
|
||||
|
||||
|
||||
async def verify_source_credential(
|
||||
*,
|
||||
platform: str,
|
||||
|
||||
Reference in New Issue
Block a user