feat(subscribestar): flip dispatch to the native ingester (#893, Step 5)
CI / lint (push) Successful in 2s
CI / frontend-build (push) Successful in 21s
CI / backend-lint-and-test (push) Successful in 28s
CI / integration (push) Failing after 3m18s

SubscribeStar now downloads + verifies through the native core ingester instead
of gallery-dl — the go-live switch for milestone #71.

- download_backends: subscribestar added to NATIVE_INGESTER_PLATFORMS; a
  _NATIVE_INGESTERS registry + _resolve_native_campaign_id make _run_native_ingester
  / preview_source / verify_source_credential platform-aware. SubscribeStar's
  campaign_id IS the creator URL (no resolver); Patreon still resolves the vanity.
  preview now catches the shared NativeIngestError (covers both platforms).
- platform_lock: subscribestar serialized (one paced walk at a time).
- gallery_dl: subscribestar entry removed from PLATFORM_DEFAULTS (rule 22 — no
  fallback once native works).
- frontend SourceActions: isPatreon → isNative (patreon|subscribestar) so the
  recover/recapture actions show for subscribestar; download_service's
  cursor/mode/post_first + the preview endpoint already key on
  uses_native_ingester, so backfill/recovery/recapture/preview light up for free.
- tests: download_backends (subscribestar native), platform_lock (serialized),
  and three gallery-dl-sample tests repointed to hentaifoundry (api_credentials
  verify, gallery_dl_service skip-value, api_sources arm-no-preflight).

post_is_gated stays best-effort (can't cause junk downloads); not gating this.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-06-17 12:03:48 -04:00
parent 82551a89d1
commit d8d8ecd78f
9 changed files with 83 additions and 48 deletions
+46 -20
View File
@@ -24,13 +24,22 @@ import asyncio
from pathlib import Path
from .gallery_dl import DownloadResult, ErrorType
from .native_ingest_common import NativeIngestError
from .patreon_ingester import PatreonIngester
from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source
from .subscribestar_ingester import SubscribeStarIngester
# Platforms whose download + verify go through the native ingester rather than
# gallery-dl. gallery-dl still serves every other platform (subscribestar,
# hentaifoundry, discord, pixiv, deviantart) unchanged.
NATIVE_INGESTER_PLATFORMS = frozenset({"patreon"})
# gallery-dl. gallery-dl still serves the rest (hentaifoundry, discord, pixiv,
# deviantart) until they migrate too.
NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar"})
# Native ingester classes keyed by platform (uniform constructor signature, so
# _run_native_ingester / preview build whichever the source needs).
_NATIVE_INGESTERS = {
"patreon": PatreonIngester,
"subscribestar": SubscribeStarIngester,
}
# Mirrors patreon_resolver._CAMPAIGNS_URL — surfaced in resolution-failure
# messages so the operator sees the exact lookup endpoint that was hit.
@@ -81,22 +90,36 @@ async def run_download(
return result, None
async def _resolve_native_campaign_id(
platform: str, url: str, cookies_path: str | None, overrides: dict,
) -> tuple[str | None, str | None]:
"""`(campaign_id, resolved_campaign_id)` for a native source. SubscribeStar's
feed id IS the creator URL (no lookup → resolved None). Patreon resolves the
campaign id from the vanity URL (resolved non-None when a lookup actually ran,
so phase 3 caches it)."""
if platform == "subscribestar":
return url, None
return await resolve_campaign_id_for_source(url, cookies_path, overrides)
async def _run_native_ingester(
ctx: dict, source_config, mode: str | None, gdl, sync_session_factory,
) -> tuple[DownloadResult, str | None]:
"""Patreon (today the only native platform): resolve the campaign id, then run
the native ingester in a worker thread (it is sync requests/subprocess).
"""Run the native ingester for a native platform in a worker thread (sync
requests/subprocess). Patreon resolves a campaign id from the vanity URL;
SubscribeStar's feed id is the creator URL itself. A campaign id we cannot
resolve is a loud NOT_FOUND — never a silent empty success.
`resolved_campaign_id` is non-None only when we had to look it up from the
vanity URL this run, so phase 3 caches it the way the old gallery-dl retry
did. A campaign id we cannot resolve is a loud NOT_FOUND — never a silent
empty success.
`resolved_campaign_id` is non-None only when a lookup ran this call, so phase
3 caches it the way the old gallery-dl retry did.
"""
platform = ctx["platform"]
overrides = ctx["config_overrides"] or {}
campaign_id, resolved_campaign_id = await resolve_campaign_id_for_source(
ctx["url"], ctx["cookies_path"], overrides
campaign_id, resolved_campaign_id = await _resolve_native_campaign_id(
platform, ctx["url"], ctx["cookies_path"], overrides
)
if not campaign_id:
# Only reachable for Patreon (SubscribeStar's campaign id is the URL).
url = ctx["url"]
vanity = extract_vanity(url)
return (
@@ -104,7 +127,7 @@ async def _run_native_ingester(
success=False,
url=url,
artist_slug=ctx["artist_slug"],
platform="patreon",
platform=platform,
error_type=ErrorType.NOT_FOUND,
error_message=(
f"Could not resolve Patreon campaign id. source_url={url!r}; "
@@ -127,7 +150,7 @@ async def _run_native_ingester(
if source_config.sleep_request is not None
else max(0.5, rate_limit / 4)
)
ingester = PatreonIngester(
ingester = _NATIVE_INGESTERS[platform](
images_root=gdl.images_root,
cookies_path=ctx["cookies_path"],
session_factory=sync_session_factory,
@@ -174,10 +197,8 @@ async def preview_source(
"""
import asyncio
from .patreon_client import PatreonAPIError
campaign_id, _ = await resolve_campaign_id_for_source(
url, cookies_path, config_overrides or {}
campaign_id, _ = await _resolve_native_campaign_id(
platform, url, cookies_path, config_overrides or {}
)
if not campaign_id:
vanity = extract_vanity(url)
@@ -188,7 +209,7 @@ async def preview_source(
"(cookies expired, or the creator moved/renamed?)."
)
}
ingester = PatreonIngester(
ingester = _NATIVE_INGESTERS[platform](
images_root=images_root,
cookies_path=cookies_path,
session_factory=sync_session_factory,
@@ -199,7 +220,7 @@ async def preview_source(
None,
lambda: ingester.preview(source_id, campaign_id, page_limit=page_limit),
)
except PatreonAPIError as exc:
except NativeIngestError as exc:
return {"error": f"Couldn't preview: {exc}"}
return result
@@ -221,7 +242,12 @@ async def verify_source_credential(
"""
if uses_native_ingester(platform):
# Native ingester platforms verify via their own lightweight auth probe
# (resolve campaign id + one authenticated API page). Patreon today.
# (one authenticated feed fetch). SubscribeStar's probe takes the creator
# URL directly; Patreon's resolves the campaign id first.
if platform == "subscribestar":
from .subscribestar_ingester import verify_subscribestar_credential
return await verify_subscribestar_credential(url, cookies_path, config_overrides)
from .patreon_ingester import verify_patreon_credential
return await verify_patreon_credential(url, cookies_path, config_overrides)
+2 -5
View File
@@ -298,11 +298,8 @@ class GalleryDLService:
# removed at the plan-#697 cutover — it now uses the native ingester
# (services/patreon_ingester.py), not gallery-dl.
PLATFORM_DEFAULTS = {
"subscribestar": {
"content_types": ["all"],
"directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"],
"filename": "{num:>02}_{filename}.{extension}",
},
# subscribestar removed — it's a native-ingester platform now (#71); the
# remaining entries are the gallery-dl platforms not yet migrated.
"hentaifoundry": {
"content_types": ["all"],
"directory": [],
+4 -3
View File
@@ -21,9 +21,10 @@ from ..config import get_config
log = logging.getLogger(__name__)
# Platforms walked one-at-a-time. gallery-dl platforms are intentionally NOT
# here: each runs as a self-pacing subprocess and they're lower-volume. Add a
# platform here to cap it to a single concurrent walk.
SERIALIZED_PLATFORMS = frozenset({"patreon"})
# here: each runs as a self-pacing subprocess and they're lower-volume. The
# native-ingester platforms are serialized (one paced scrape/API walk at a time).
# Add a platform here to cap it to a single concurrent walk.
SERIALIZED_PLATFORMS = frozenset({"patreon", "subscribestar"})
_LOCK_PREFIX = "fc:download_lock:"