feat(subscribestar): flip dispatch to the native ingester (#893, Step 5)
SubscribeStar now downloads + verifies through the native core ingester instead of gallery-dl — the go-live switch for milestone #71. - download_backends: subscribestar added to NATIVE_INGESTER_PLATFORMS; a _NATIVE_INGESTERS registry + _resolve_native_campaign_id make _run_native_ingester / preview_source / verify_source_credential platform-aware. SubscribeStar's campaign_id IS the creator URL (no resolver); Patreon still resolves the vanity. preview now catches the shared NativeIngestError (covers both platforms). - platform_lock: subscribestar serialized (one paced walk at a time). - gallery_dl: subscribestar entry removed from PLATFORM_DEFAULTS (rule 22 — no fallback once native works). - frontend SourceActions: isPatreon → isNative (patreon|subscribestar) so the recover/recapture actions show for subscribestar; download_service's cursor/mode/post_first + the preview endpoint already key on uses_native_ingester, so backfill/recovery/recapture/preview light up for free. - tests: download_backends (subscribestar native), platform_lock (serialized), and three gallery-dl-sample tests repointed to hentaifoundry (api_credentials verify, gallery_dl_service skip-value, api_sources arm-no-preflight). post_is_gated stays best-effort (can't cause junk downloads); not gating this. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -24,13 +24,22 @@ import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
from .gallery_dl import DownloadResult, ErrorType
|
||||
from .native_ingest_common import NativeIngestError
|
||||
from .patreon_ingester import PatreonIngester
|
||||
from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source
|
||||
from .subscribestar_ingester import SubscribeStarIngester
|
||||
|
||||
# Platforms whose download + verify go through the native ingester rather than
|
||||
# gallery-dl. gallery-dl still serves every other platform (subscribestar,
|
||||
# hentaifoundry, discord, pixiv, deviantart) unchanged.
|
||||
NATIVE_INGESTER_PLATFORMS = frozenset({"patreon"})
|
||||
# gallery-dl. gallery-dl still serves the rest (hentaifoundry, discord, pixiv,
|
||||
# deviantart) until they migrate too.
|
||||
NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar"})
|
||||
|
||||
# Native ingester classes keyed by platform (uniform constructor signature, so
|
||||
# _run_native_ingester / preview build whichever the source needs).
|
||||
_NATIVE_INGESTERS = {
|
||||
"patreon": PatreonIngester,
|
||||
"subscribestar": SubscribeStarIngester,
|
||||
}
|
||||
|
||||
# Mirrors patreon_resolver._CAMPAIGNS_URL — surfaced in resolution-failure
|
||||
# messages so the operator sees the exact lookup endpoint that was hit.
|
||||
@@ -81,22 +90,36 @@ async def run_download(
|
||||
return result, None
|
||||
|
||||
|
||||
async def _resolve_native_campaign_id(
|
||||
platform: str, url: str, cookies_path: str | None, overrides: dict,
|
||||
) -> tuple[str | None, str | None]:
|
||||
"""`(campaign_id, resolved_campaign_id)` for a native source. SubscribeStar's
|
||||
feed id IS the creator URL (no lookup → resolved None). Patreon resolves the
|
||||
campaign id from the vanity URL (resolved non-None when a lookup actually ran,
|
||||
so phase 3 caches it)."""
|
||||
if platform == "subscribestar":
|
||||
return url, None
|
||||
return await resolve_campaign_id_for_source(url, cookies_path, overrides)
|
||||
|
||||
|
||||
async def _run_native_ingester(
|
||||
ctx: dict, source_config, mode: str | None, gdl, sync_session_factory,
|
||||
) -> tuple[DownloadResult, str | None]:
|
||||
"""Patreon (today the only native platform): resolve the campaign id, then run
|
||||
the native ingester in a worker thread (it is sync requests/subprocess).
|
||||
"""Run the native ingester for a native platform in a worker thread (sync
|
||||
requests/subprocess). Patreon resolves a campaign id from the vanity URL;
|
||||
SubscribeStar's feed id is the creator URL itself. A campaign id we cannot
|
||||
resolve is a loud NOT_FOUND — never a silent empty success.
|
||||
|
||||
`resolved_campaign_id` is non-None only when we had to look it up from the
|
||||
vanity URL this run, so phase 3 caches it the way the old gallery-dl retry
|
||||
did. A campaign id we cannot resolve is a loud NOT_FOUND — never a silent
|
||||
empty success.
|
||||
`resolved_campaign_id` is non-None only when a lookup ran this call, so phase
|
||||
3 caches it the way the old gallery-dl retry did.
|
||||
"""
|
||||
platform = ctx["platform"]
|
||||
overrides = ctx["config_overrides"] or {}
|
||||
campaign_id, resolved_campaign_id = await resolve_campaign_id_for_source(
|
||||
ctx["url"], ctx["cookies_path"], overrides
|
||||
campaign_id, resolved_campaign_id = await _resolve_native_campaign_id(
|
||||
platform, ctx["url"], ctx["cookies_path"], overrides
|
||||
)
|
||||
if not campaign_id:
|
||||
# Only reachable for Patreon (SubscribeStar's campaign id is the URL).
|
||||
url = ctx["url"]
|
||||
vanity = extract_vanity(url)
|
||||
return (
|
||||
@@ -104,7 +127,7 @@ async def _run_native_ingester(
|
||||
success=False,
|
||||
url=url,
|
||||
artist_slug=ctx["artist_slug"],
|
||||
platform="patreon",
|
||||
platform=platform,
|
||||
error_type=ErrorType.NOT_FOUND,
|
||||
error_message=(
|
||||
f"Could not resolve Patreon campaign id. source_url={url!r}; "
|
||||
@@ -127,7 +150,7 @@ async def _run_native_ingester(
|
||||
if source_config.sleep_request is not None
|
||||
else max(0.5, rate_limit / 4)
|
||||
)
|
||||
ingester = PatreonIngester(
|
||||
ingester = _NATIVE_INGESTERS[platform](
|
||||
images_root=gdl.images_root,
|
||||
cookies_path=ctx["cookies_path"],
|
||||
session_factory=sync_session_factory,
|
||||
@@ -174,10 +197,8 @@ async def preview_source(
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
from .patreon_client import PatreonAPIError
|
||||
|
||||
campaign_id, _ = await resolve_campaign_id_for_source(
|
||||
url, cookies_path, config_overrides or {}
|
||||
campaign_id, _ = await _resolve_native_campaign_id(
|
||||
platform, url, cookies_path, config_overrides or {}
|
||||
)
|
||||
if not campaign_id:
|
||||
vanity = extract_vanity(url)
|
||||
@@ -188,7 +209,7 @@ async def preview_source(
|
||||
"(cookies expired, or the creator moved/renamed?)."
|
||||
)
|
||||
}
|
||||
ingester = PatreonIngester(
|
||||
ingester = _NATIVE_INGESTERS[platform](
|
||||
images_root=images_root,
|
||||
cookies_path=cookies_path,
|
||||
session_factory=sync_session_factory,
|
||||
@@ -199,7 +220,7 @@ async def preview_source(
|
||||
None,
|
||||
lambda: ingester.preview(source_id, campaign_id, page_limit=page_limit),
|
||||
)
|
||||
except PatreonAPIError as exc:
|
||||
except NativeIngestError as exc:
|
||||
return {"error": f"Couldn't preview: {exc}"}
|
||||
return result
|
||||
|
||||
@@ -221,7 +242,12 @@ async def verify_source_credential(
|
||||
"""
|
||||
if uses_native_ingester(platform):
|
||||
# Native ingester platforms verify via their own lightweight auth probe
|
||||
# (resolve campaign id + one authenticated API page). Patreon today.
|
||||
# (one authenticated feed fetch). SubscribeStar's probe takes the creator
|
||||
# URL directly; Patreon's resolves the campaign id first.
|
||||
if platform == "subscribestar":
|
||||
from .subscribestar_ingester import verify_subscribestar_credential
|
||||
|
||||
return await verify_subscribestar_credential(url, cookies_path, config_overrides)
|
||||
from .patreon_ingester import verify_patreon_credential
|
||||
|
||||
return await verify_patreon_credential(url, cookies_path, config_overrides)
|
||||
|
||||
@@ -298,11 +298,8 @@ class GalleryDLService:
|
||||
# removed at the plan-#697 cutover — it now uses the native ingester
|
||||
# (services/patreon_ingester.py), not gallery-dl.
|
||||
PLATFORM_DEFAULTS = {
|
||||
"subscribestar": {
|
||||
"content_types": ["all"],
|
||||
"directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"],
|
||||
"filename": "{num:>02}_{filename}.{extension}",
|
||||
},
|
||||
# subscribestar removed — it's a native-ingester platform now (#71); the
|
||||
# remaining entries are the gallery-dl platforms not yet migrated.
|
||||
"hentaifoundry": {
|
||||
"content_types": ["all"],
|
||||
"directory": [],
|
||||
|
||||
@@ -21,9 +21,10 @@ from ..config import get_config
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Platforms walked one-at-a-time. gallery-dl platforms are intentionally NOT
|
||||
# here: each runs as a self-pacing subprocess and they're lower-volume. Add a
|
||||
# platform here to cap it to a single concurrent walk.
|
||||
SERIALIZED_PLATFORMS = frozenset({"patreon"})
|
||||
# here: each runs as a self-pacing subprocess and they're lower-volume. The
|
||||
# native-ingester platforms are serialized (one paced scrape/API walk at a time).
|
||||
# Add a platform here to cap it to a single concurrent walk.
|
||||
SERIALIZED_PLATFORMS = frozenset({"patreon", "subscribestar"})
|
||||
|
||||
_LOCK_PREFIX = "fc:download_lock:"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user