From 0bbcdee3bde153ac538b540b5cbea7d11d2c09fd Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 3 Jul 2026 09:54:18 -0400 Subject: [PATCH] feat(pixiv): flip dispatch to the native ingester (#129 step 4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pixiv joins NATIVE_INGESTER_PLATFORMS: download/verify/preview and the recover/recapture UI actions now route through PixivIngester. Campaign id is parsed straight from the source URL (numeric user id — no network resolver), with a platform-aware resolution-failure message. auth_token now rides the uniform adapter construction (token platforms use it, cookie platforms accept-and-ignore), and the preview endpoint fetches/threads it. The legacy gallery-dl pixiv path is fully removed (PLATFORM_DEFAULTS entry + the refresh-token config branches in download/verify) per no-legacy policy; gallery-dl keeps hentaifoundry/discord/deviantart until they migrate/retire. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM --- backend/app/api/sources.py | 2 + backend/app/services/download_backends.py | 69 ++++++++++++------- backend/app/services/gallery_dl.py | 16 +---- backend/app/services/patreon_ingester.py | 5 ++ backend/app/services/platforms/pixiv.py | 12 +++- .../app/services/subscribestar_ingester.py | 5 ++ .../subscriptions/SourceActions.vue | 3 +- tests/test_download_backends.py | 17 ++++- 8 files changed, 87 insertions(+), 42 deletions(-) diff --git a/backend/app/api/sources.py b/backend/app/api/sources.py index 4e202dc..5b3fcaf 100644 --- a/backend/app/api/sources.py +++ b/backend/app/api/sources.py @@ -238,6 +238,7 @@ async def preview_source_endpoint(source_id: int): ) cred = CredentialService(session, _get_crypto()) cookies_path = await cred.get_cookies_path(rec.platform) + auth_token = await cred.get_token(rec.platform) # The walk + ledger reads are sync (run off the request loop); the process # sync engine is the same one the download task uses. @@ -249,6 +250,7 @@ async def preview_source_endpoint(source_id: int): cookies_path=str(cookies_path) if cookies_path else None, images_root=Path("/images"), sync_session_factory=sync_session_factory(), + auth_token=auth_token, ) if "error" in result: return _bad("preview_failed", detail=result["error"], status=409) diff --git a/backend/app/services/download_backends.py b/backend/app/services/download_backends.py index 48e6ceb..9d799ef 100644 --- a/backend/app/services/download_backends.py +++ b/backend/app/services/download_backends.py @@ -27,12 +27,15 @@ from .gallery_dl import DownloadResult, ErrorType from .native_ingest_common import NativeIngestError from .patreon_ingester import PatreonIngester from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source +from .pixiv_client import user_id_from_url +from .pixiv_ingester import PixivIngester from .subscribestar_ingester import SubscribeStarIngester # Platforms whose download + verify go through the native ingester rather than -# gallery-dl. gallery-dl still serves the rest (hentaifoundry, discord, pixiv, -# deviantart) until they migrate too. -NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar"}) +# gallery-dl. gallery-dl still serves the rest (hentaifoundry, discord, +# deviantart — the latter slated for retirement, not migration) until they +# migrate too. +NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar", "pixiv"}) # Mirrors patreon_resolver._CAMPAIGNS_URL — surfaced in resolution-failure # messages so the operator sees the exact lookup endpoint that was hit. @@ -46,6 +49,7 @@ def _native_ingester_cls(platform: str): dispatch pick up the replacement.""" return { "patreon": PatreonIngester, + "pixiv": PixivIngester, "subscribestar": SubscribeStarIngester, }[platform] @@ -98,14 +102,34 @@ async def _resolve_native_campaign_id( platform: str, url: str, cookies_path: str | None, overrides: dict, ) -> tuple[str | None, str | None]: """`(campaign_id, resolved_campaign_id)` for a native source. SubscribeStar's - feed id IS the creator URL (no lookup → resolved None). Patreon resolves the + feed id IS the creator URL; Pixiv's is the numeric user id parsed straight + from it (no lookup → resolved None either way). Patreon resolves the campaign id from the vanity URL (resolved non-None when a lookup actually ran, so phase 3 caches it).""" if platform == "subscribestar": return url, None + if platform == "pixiv": + return user_id_from_url(url), None return await resolve_campaign_id_for_source(url, cookies_path, overrides) +def _campaign_resolution_error(platform: str, url: str) -> str: + """Operator-facing message for a native source whose campaign id could not + be resolved — names the platform's own lookup mechanism.""" + if platform == "pixiv": + return ( + f"Could not extract a pixiv user id. source_url={url!r} — expected " + "a URL like https://www.pixiv.net/users/." + ) + vanity = extract_vanity(url) + return ( + f"Could not resolve Patreon campaign id. source_url={url!r}; " + f"vanity={vanity!r}; " + f"lookup=GET {_CAMPAIGNS_API}?filter[vanity]={vanity or ''} " + "(vanity lookup failed — cookies expired or creator moved?)" + ) + + async def _run_native_ingester( ctx: dict, source_config, mode: str | None, gdl, sync_session_factory, ) -> tuple[DownloadResult, str | None]: @@ -123,9 +147,9 @@ async def _run_native_ingester( platform, ctx["url"], ctx["cookies_path"], overrides ) if not campaign_id: - # Only reachable for Patreon (SubscribeStar's campaign id is the URL). + # Patreon: vanity lookup failed. Pixiv: no numeric user id in the URL. + # (SubscribeStar's campaign id is the URL itself — never lands here.) url = ctx["url"] - vanity = extract_vanity(url) return ( DownloadResult( success=False, @@ -133,12 +157,7 @@ async def _run_native_ingester( artist_slug=ctx["artist_slug"], platform=platform, error_type=ErrorType.NOT_FOUND, - error_message=( - f"Could not resolve Patreon campaign id. source_url={url!r}; " - f"vanity={vanity!r}; " - f"lookup=GET {_CAMPAIGNS_API}?filter[vanity]={vanity or ''} " - "(vanity lookup failed — cookies expired or creator moved?)" - ), + error_message=_campaign_resolution_error(platform, url), ), None, ) @@ -161,6 +180,10 @@ async def _run_native_ingester( validate=gdl._validate_files, rate_limit=rate_limit, request_sleep=request_sleep, + # Uniform across adapters: token platforms (pixiv) authenticate with + # it, cookie platforms accept-and-ignore — so this construction stays + # platform-agnostic. + auth_token=ctx["auth_token"], ) loop = asyncio.get_running_loop() dl_result = await loop.run_in_executor( @@ -190,6 +213,7 @@ async def preview_source( cookies_path: str | None, images_root: Path, sync_session_factory, + auth_token: str | None = None, page_limit: int = 3, ) -> dict: """Dry-run preview for a native platform (plan #708 B4): resolve the campaign @@ -205,18 +229,12 @@ async def preview_source( platform, url, cookies_path, config_overrides or {} ) if not campaign_id: - vanity = extract_vanity(url) - return { - "error": ( - f"Couldn't resolve the campaign id. source_url={url!r}; " - f"vanity={vanity!r}; lookup=GET {_CAMPAIGNS_API}?filter[vanity]={vanity or ''} " - "(cookies expired, or the creator moved/renamed?)." - ) - } + return {"error": _campaign_resolution_error(platform, url)} ingester = _native_ingester_cls(platform)( images_root=images_root, cookies_path=cookies_path, session_factory=sync_session_factory, + auth_token=auth_token, ) loop = asyncio.get_running_loop() try: @@ -245,13 +263,18 @@ async def verify_source_credential( this and render the result. """ if uses_native_ingester(platform): - # Native ingester platforms verify via their own lightweight auth probe - # (one authenticated feed fetch). SubscribeStar's probe takes the creator - # URL directly; Patreon's resolves the campaign id first. + # Native ingester platforms verify via their own lightweight auth probe. + # SubscribeStar's probe takes the creator URL directly; Patreon's + # resolves the campaign id first; Pixiv's is one OAuth refresh (the + # exact call that fails when the token is bad — no feed walk). if platform == "subscribestar": from .subscribestar_ingester import verify_subscribestar_credential return await verify_subscribestar_credential(url, cookies_path, config_overrides) + if platform == "pixiv": + from .pixiv_ingester import verify_pixiv_credential + + return await verify_pixiv_credential(auth_token) from .patreon_ingester import verify_patreon_credential return await verify_patreon_credential(url, cookies_path, config_overrides) diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py index dea0057..748052e 100644 --- a/backend/app/services/gallery_dl.py +++ b/backend/app/services/gallery_dl.py @@ -298,8 +298,9 @@ class GalleryDLService: # removed at the plan-#697 cutover — it now uses the native ingester # (services/patreon_ingester.py), not gallery-dl. PLATFORM_DEFAULTS = { - # subscribestar removed — it's a native-ingester platform now (#71); the - # remaining entries are the gallery-dl platforms not yet migrated. + # subscribestar removed — native-ingester platform now (#71); pixiv + # removed likewise (#129). The remaining entries are the gallery-dl + # platforms not yet migrated. "hentaifoundry": { "content_types": ["all"], "directory": [], @@ -315,12 +316,6 @@ class GalleryDLService: "reactions": False, "threads": True, }, - "pixiv": { - "content_types": ["all"], - "directory": ["{category}"], - "filename": "{id}_{title[:50]}_{num:>02}.{extension}", - "ugoira": True, - }, "deviantart": { "content_types": ["all"], "directory": [], @@ -694,9 +689,6 @@ class GalleryDLService: if auth_token and platform == "discord": config["extractor"].setdefault("discord", {}) config["extractor"]["discord"]["token"] = auth_token - if auth_token and platform == "pixiv": - config["extractor"].setdefault("pixiv", {}) - config["extractor"]["pixiv"]["refresh-token"] = auth_token with tempfile.NamedTemporaryFile( mode="w", suffix=".json", delete=False, dir=str(self._config_dir), @@ -882,8 +874,6 @@ class GalleryDLService: config["extractor"]["cookies"] = cookies_path if auth_token and platform == "discord": config["extractor"].setdefault("discord", {})["token"] = auth_token - if auth_token and platform == "pixiv": - config["extractor"].setdefault("pixiv", {})["refresh-token"] = auth_token with tempfile.NamedTemporaryFile( mode="w", suffix=".json", delete=False, dir=str(self._config_dir), diff --git a/backend/app/services/patreon_ingester.py b/backend/app/services/patreon_ingester.py index 5acc086..2c1e756 100644 --- a/backend/app/services/patreon_ingester.py +++ b/backend/app/services/patreon_ingester.py @@ -87,9 +87,14 @@ class PatreonIngester(Ingester): validate: bool = True, rate_limit: float = 0.0, request_sleep: float = 0.0, + auth_token: str | None = None, client: PatreonClient | None = None, downloader: PatreonDownloader | None = None, ): + # auth_token: accepted for the uniform native-ingester construction + # (download_backends passes it to every adapter); Patreon + # authenticates by cookies, so it's unused here. + del auth_token self.images_root = Path(images_root) self.cookies_path = str(cookies_path) if cookies_path else None # Pacing (plan #703): request_sleep paces the API page fetches, diff --git a/backend/app/services/platforms/pixiv.py b/backend/app/services/platforms/pixiv.py index 8664a1e..5d14375 100644 --- a/backend/app/services/platforms/pixiv.py +++ b/backend/app/services/platforms/pixiv.py @@ -1,8 +1,14 @@ """Pixiv — one quirk. -post_url: gallery-dl's `url` is the image URL on `i.pximg.net`. The -post permalink follows /artworks/. external_post_id (= `id`) was -already correct, so no override there. +post_url: the sidecar's `url` (legacy gallery-dl era) is the image URL +on `i.pximg.net`, and the native post record (#129) writes no url key +at all — the permalink is synthesized from `id` here either way: +/artworks/. external_post_id (= `id`) was already correct, so no +override there. + +Downloads run through the native ingester (pixiv_ingester.py), not +gallery-dl; this registry entry still owns URL validation, sidecar +parsing, and the credential surface (the OAuth refresh token). """ from .base import GD_DEFAULTS, PlatformInfo, str_id_value diff --git a/backend/app/services/subscribestar_ingester.py b/backend/app/services/subscribestar_ingester.py index 26138b5..6049362 100644 --- a/backend/app/services/subscribestar_ingester.py +++ b/backend/app/services/subscribestar_ingester.py @@ -61,9 +61,14 @@ class SubscribeStarIngester(Ingester): validate: bool = True, rate_limit: float = 0.0, request_sleep: float = 0.0, + auth_token: str | None = None, client: SubscribeStarClient | None = None, downloader: SubscribeStarDownloader | None = None, ): + # auth_token: accepted for the uniform native-ingester construction + # (download_backends passes it to every adapter); SubscribeStar + # authenticates by cookies, so it's unused here. + del auth_token self.images_root = Path(images_root) self.cookies_path = str(cookies_path) if cookies_path else None resolved_client = ( diff --git a/frontend/src/components/subscriptions/SourceActions.vue b/frontend/src/components/subscriptions/SourceActions.vue index 06a21d6..5c87393 100644 --- a/frontend/src/components/subscriptions/SourceActions.vue +++ b/frontend/src/components/subscriptions/SourceActions.vue @@ -76,7 +76,8 @@ const recovering = computed(() => !!props.source.backfill_bypass_seen) const recapturing = computed(() => !!props.source.backfill_recapture) // Recover / recapture are native-ingester features (ledger-bypass re-walk and // post-text re-grab), available to every native platform — not just Patreon. -const NATIVE_PLATFORMS = ['patreon', 'subscribestar'] +// Mirrors backend download_backends.NATIVE_INGESTER_PLATFORMS. +const NATIVE_PLATFORMS = ['patreon', 'subscribestar', 'pixiv'] const isNative = computed(() => NATIVE_PLATFORMS.includes(props.source.platform)) diff --git a/tests/test_download_backends.py b/tests/test_download_backends.py index 6ce1774..b53da45 100644 --- a/tests/test_download_backends.py +++ b/tests/test_download_backends.py @@ -3,12 +3,15 @@ native ingester vs. gallery-dl. Pure, no DB.""" from backend.app.services.download_backends import ( NATIVE_INGESTER_PLATFORMS, + _campaign_resolution_error, + _native_ingester_cls, uses_native_ingester, ) +from backend.app.services.pixiv_ingester import PixivIngester def test_native_platforms(): - for platform in ("patreon", "subscribestar"): + for platform in ("patreon", "subscribestar", "pixiv"): assert uses_native_ingester(platform) is True assert platform in NATIVE_INGESTER_PLATFORMS @@ -16,9 +19,19 @@ def test_native_platforms(): def test_gallery_dl_platforms_are_not_native(): # The platforms still served by gallery-dl must NOT route to the native # ingester — guards an accidental over-broad migration. - for platform in ("hentaifoundry", "discord", "pixiv", "deviantart"): + for platform in ("hentaifoundry", "discord", "deviantart"): assert uses_native_ingester(platform) is False def test_unknown_platform_is_not_native(): assert uses_native_ingester("nonsense") is False + + +def test_pixiv_dispatches_to_its_ingester(): + assert _native_ingester_cls("pixiv") is PixivIngester + + +def test_pixiv_resolution_error_names_the_expected_url_shape(): + msg = _campaign_resolution_error("pixiv", "https://www.pixiv.net/artworks/1") + assert "pixiv user id" in msg + assert "users/" in msg