From d8d8ecd78fea48fb537729735ff15382f3d0770a Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 17 Jun 2026 12:03:48 -0400 Subject: [PATCH] feat(subscribestar): flip dispatch to the native ingester (#893, Step 5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SubscribeStar now downloads + verifies through the native core ingester instead of gallery-dl — the go-live switch for milestone #71. - download_backends: subscribestar added to NATIVE_INGESTER_PLATFORMS; a _NATIVE_INGESTERS registry + _resolve_native_campaign_id make _run_native_ingester / preview_source / verify_source_credential platform-aware. SubscribeStar's campaign_id IS the creator URL (no resolver); Patreon still resolves the vanity. preview now catches the shared NativeIngestError (covers both platforms). - platform_lock: subscribestar serialized (one paced walk at a time). - gallery_dl: subscribestar entry removed from PLATFORM_DEFAULTS (rule 22 — no fallback once native works). - frontend SourceActions: isPatreon → isNative (patreon|subscribestar) so the recover/recapture actions show for subscribestar; download_service's cursor/mode/post_first + the preview endpoint already key on uses_native_ingester, so backfill/recovery/recapture/preview light up for free. - tests: download_backends (subscribestar native), platform_lock (serialized), and three gallery-dl-sample tests repointed to hentaifoundry (api_credentials verify, gallery_dl_service skip-value, api_sources arm-no-preflight). post_is_gated stays best-effort (can't cause junk downloads); not gating this. Co-Authored-By: Claude Opus 4.8 --- backend/app/services/download_backends.py | 66 +++++++++++++------ backend/app/services/gallery_dl.py | 7 +- backend/app/services/platform_lock.py | 7 +- .../subscriptions/SourceActions.vue | 11 ++-- tests/test_api_credentials.py | 11 ++-- tests/test_api_sources.py | 4 +- tests/test_download_backends.py | 13 ++-- tests/test_gallery_dl_service.py | 4 +- tests/test_platform_lock.py | 8 ++- 9 files changed, 83 insertions(+), 48 deletions(-) diff --git a/backend/app/services/download_backends.py b/backend/app/services/download_backends.py index fab1bb5..350f009 100644 --- a/backend/app/services/download_backends.py +++ b/backend/app/services/download_backends.py @@ -24,13 +24,22 @@ import asyncio from pathlib import Path from .gallery_dl import DownloadResult, ErrorType +from .native_ingest_common import NativeIngestError from .patreon_ingester import PatreonIngester from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source +from .subscribestar_ingester import SubscribeStarIngester # Platforms whose download + verify go through the native ingester rather than -# gallery-dl. gallery-dl still serves every other platform (subscribestar, -# hentaifoundry, discord, pixiv, deviantart) unchanged. -NATIVE_INGESTER_PLATFORMS = frozenset({"patreon"}) +# gallery-dl. gallery-dl still serves the rest (hentaifoundry, discord, pixiv, +# deviantart) until they migrate too. +NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar"}) + +# Native ingester classes keyed by platform (uniform constructor signature, so +# _run_native_ingester / preview build whichever the source needs). +_NATIVE_INGESTERS = { + "patreon": PatreonIngester, + "subscribestar": SubscribeStarIngester, +} # Mirrors patreon_resolver._CAMPAIGNS_URL — surfaced in resolution-failure # messages so the operator sees the exact lookup endpoint that was hit. @@ -81,22 +90,36 @@ async def run_download( return result, None +async def _resolve_native_campaign_id( + platform: str, url: str, cookies_path: str | None, overrides: dict, +) -> tuple[str | None, str | None]: + """`(campaign_id, resolved_campaign_id)` for a native source. SubscribeStar's + feed id IS the creator URL (no lookup → resolved None). Patreon resolves the + campaign id from the vanity URL (resolved non-None when a lookup actually ran, + so phase 3 caches it).""" + if platform == "subscribestar": + return url, None + return await resolve_campaign_id_for_source(url, cookies_path, overrides) + + async def _run_native_ingester( ctx: dict, source_config, mode: str | None, gdl, sync_session_factory, ) -> tuple[DownloadResult, str | None]: - """Patreon (today the only native platform): resolve the campaign id, then run - the native ingester in a worker thread (it is sync requests/subprocess). + """Run the native ingester for a native platform in a worker thread (sync + requests/subprocess). Patreon resolves a campaign id from the vanity URL; + SubscribeStar's feed id is the creator URL itself. A campaign id we cannot + resolve is a loud NOT_FOUND — never a silent empty success. - `resolved_campaign_id` is non-None only when we had to look it up from the - vanity URL this run, so phase 3 caches it the way the old gallery-dl retry - did. A campaign id we cannot resolve is a loud NOT_FOUND — never a silent - empty success. + `resolved_campaign_id` is non-None only when a lookup ran this call, so phase + 3 caches it the way the old gallery-dl retry did. """ + platform = ctx["platform"] overrides = ctx["config_overrides"] or {} - campaign_id, resolved_campaign_id = await resolve_campaign_id_for_source( - ctx["url"], ctx["cookies_path"], overrides + campaign_id, resolved_campaign_id = await _resolve_native_campaign_id( + platform, ctx["url"], ctx["cookies_path"], overrides ) if not campaign_id: + # Only reachable for Patreon (SubscribeStar's campaign id is the URL). url = ctx["url"] vanity = extract_vanity(url) return ( @@ -104,7 +127,7 @@ async def _run_native_ingester( success=False, url=url, artist_slug=ctx["artist_slug"], - platform="patreon", + platform=platform, error_type=ErrorType.NOT_FOUND, error_message=( f"Could not resolve Patreon campaign id. source_url={url!r}; " @@ -127,7 +150,7 @@ async def _run_native_ingester( if source_config.sleep_request is not None else max(0.5, rate_limit / 4) ) - ingester = PatreonIngester( + ingester = _NATIVE_INGESTERS[platform]( images_root=gdl.images_root, cookies_path=ctx["cookies_path"], session_factory=sync_session_factory, @@ -174,10 +197,8 @@ async def preview_source( """ import asyncio - from .patreon_client import PatreonAPIError - - campaign_id, _ = await resolve_campaign_id_for_source( - url, cookies_path, config_overrides or {} + campaign_id, _ = await _resolve_native_campaign_id( + platform, url, cookies_path, config_overrides or {} ) if not campaign_id: vanity = extract_vanity(url) @@ -188,7 +209,7 @@ async def preview_source( "(cookies expired, or the creator moved/renamed?)." ) } - ingester = PatreonIngester( + ingester = _NATIVE_INGESTERS[platform]( images_root=images_root, cookies_path=cookies_path, session_factory=sync_session_factory, @@ -199,7 +220,7 @@ async def preview_source( None, lambda: ingester.preview(source_id, campaign_id, page_limit=page_limit), ) - except PatreonAPIError as exc: + except NativeIngestError as exc: return {"error": f"Couldn't preview: {exc}"} return result @@ -221,7 +242,12 @@ async def verify_source_credential( """ if uses_native_ingester(platform): # Native ingester platforms verify via their own lightweight auth probe - # (resolve campaign id + one authenticated API page). Patreon today. + # (one authenticated feed fetch). SubscribeStar's probe takes the creator + # URL directly; Patreon's resolves the campaign id first. + if platform == "subscribestar": + from .subscribestar_ingester import verify_subscribestar_credential + + return await verify_subscribestar_credential(url, cookies_path, config_overrides) from .patreon_ingester import verify_patreon_credential return await verify_patreon_credential(url, cookies_path, config_overrides) diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py index 6797b89..dea0057 100644 --- a/backend/app/services/gallery_dl.py +++ b/backend/app/services/gallery_dl.py @@ -298,11 +298,8 @@ class GalleryDLService: # removed at the plan-#697 cutover — it now uses the native ingester # (services/patreon_ingester.py), not gallery-dl. PLATFORM_DEFAULTS = { - "subscribestar": { - "content_types": ["all"], - "directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"], - "filename": "{num:>02}_{filename}.{extension}", - }, + # subscribestar removed — it's a native-ingester platform now (#71); the + # remaining entries are the gallery-dl platforms not yet migrated. "hentaifoundry": { "content_types": ["all"], "directory": [], diff --git a/backend/app/services/platform_lock.py b/backend/app/services/platform_lock.py index c70e1e8..27d029e 100644 --- a/backend/app/services/platform_lock.py +++ b/backend/app/services/platform_lock.py @@ -21,9 +21,10 @@ from ..config import get_config log = logging.getLogger(__name__) # Platforms walked one-at-a-time. gallery-dl platforms are intentionally NOT -# here: each runs as a self-pacing subprocess and they're lower-volume. Add a -# platform here to cap it to a single concurrent walk. -SERIALIZED_PLATFORMS = frozenset({"patreon"}) +# here: each runs as a self-pacing subprocess and they're lower-volume. The +# native-ingester platforms are serialized (one paced scrape/API walk at a time). +# Add a platform here to cap it to a single concurrent walk. +SERIALIZED_PLATFORMS = frozenset({"patreon", "subscribestar"}) _LOCK_PREFIX = "fc:download_lock:" diff --git a/frontend/src/components/subscriptions/SourceActions.vue b/frontend/src/components/subscriptions/SourceActions.vue index 7d23dc9..06a21d6 100644 --- a/frontend/src/components/subscriptions/SourceActions.vue +++ b/frontend/src/components/subscriptions/SourceActions.vue @@ -28,7 +28,7 @@ @@ -39,7 +39,7 @@ @@ -49,7 +49,7 @@ already on disk — without re-downloading media - + props.source.backfill_state === 'running') const recovering = computed(() => !!props.source.backfill_bypass_seen) const recapturing = computed(() => !!props.source.backfill_recapture) -const isPatreon = computed(() => props.source.platform === 'patreon') +// Recover / recapture are native-ingester features (ledger-bypass re-walk and +// post-text re-grab), available to every native platform — not just Patreon. +const NATIVE_PLATFORMS = ['patreon', 'subscribestar'] +const isNative = computed(() => NATIVE_PLATFORMS.includes(props.source.platform))