feat(pixiv): flip dispatch to the native ingester (#129 step 4)
pixiv joins NATIVE_INGESTER_PLATFORMS: download/verify/preview and the recover/recapture UI actions now route through PixivIngester. Campaign id is parsed straight from the source URL (numeric user id — no network resolver), with a platform-aware resolution-failure message. auth_token now rides the uniform adapter construction (token platforms use it, cookie platforms accept-and-ignore), and the preview endpoint fetches/threads it. The legacy gallery-dl pixiv path is fully removed (PLATFORM_DEFAULTS entry + the refresh-token config branches in download/verify) per no-legacy policy; gallery-dl keeps hentaifoundry/discord/deviantart until they migrate/retire. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM
This commit is contained in:
@@ -27,12 +27,15 @@ from .gallery_dl import DownloadResult, ErrorType
|
||||
from .native_ingest_common import NativeIngestError
|
||||
from .patreon_ingester import PatreonIngester
|
||||
from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source
|
||||
from .pixiv_client import user_id_from_url
|
||||
from .pixiv_ingester import PixivIngester
|
||||
from .subscribestar_ingester import SubscribeStarIngester
|
||||
|
||||
# Platforms whose download + verify go through the native ingester rather than
|
||||
# gallery-dl. gallery-dl still serves the rest (hentaifoundry, discord, pixiv,
|
||||
# deviantart) until they migrate too.
|
||||
NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar"})
|
||||
# gallery-dl. gallery-dl still serves the rest (hentaifoundry, discord,
|
||||
# deviantart — the latter slated for retirement, not migration) until they
|
||||
# migrate too.
|
||||
NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar", "pixiv"})
|
||||
|
||||
# Mirrors patreon_resolver._CAMPAIGNS_URL — surfaced in resolution-failure
|
||||
# messages so the operator sees the exact lookup endpoint that was hit.
|
||||
@@ -46,6 +49,7 @@ def _native_ingester_cls(platform: str):
|
||||
dispatch pick up the replacement."""
|
||||
return {
|
||||
"patreon": PatreonIngester,
|
||||
"pixiv": PixivIngester,
|
||||
"subscribestar": SubscribeStarIngester,
|
||||
}[platform]
|
||||
|
||||
@@ -98,14 +102,34 @@ async def _resolve_native_campaign_id(
|
||||
platform: str, url: str, cookies_path: str | None, overrides: dict,
|
||||
) -> tuple[str | None, str | None]:
|
||||
"""`(campaign_id, resolved_campaign_id)` for a native source. SubscribeStar's
|
||||
feed id IS the creator URL (no lookup → resolved None). Patreon resolves the
|
||||
feed id IS the creator URL; Pixiv's is the numeric user id parsed straight
|
||||
from it (no lookup → resolved None either way). Patreon resolves the
|
||||
campaign id from the vanity URL (resolved non-None when a lookup actually ran,
|
||||
so phase 3 caches it)."""
|
||||
if platform == "subscribestar":
|
||||
return url, None
|
||||
if platform == "pixiv":
|
||||
return user_id_from_url(url), None
|
||||
return await resolve_campaign_id_for_source(url, cookies_path, overrides)
|
||||
|
||||
|
||||
def _campaign_resolution_error(platform: str, url: str) -> str:
|
||||
"""Operator-facing message for a native source whose campaign id could not
|
||||
be resolved — names the platform's own lookup mechanism."""
|
||||
if platform == "pixiv":
|
||||
return (
|
||||
f"Could not extract a pixiv user id. source_url={url!r} — expected "
|
||||
"a URL like https://www.pixiv.net/users/<id>."
|
||||
)
|
||||
vanity = extract_vanity(url)
|
||||
return (
|
||||
f"Could not resolve Patreon campaign id. source_url={url!r}; "
|
||||
f"vanity={vanity!r}; "
|
||||
f"lookup=GET {_CAMPAIGNS_API}?filter[vanity]={vanity or ''} "
|
||||
"(vanity lookup failed — cookies expired or creator moved?)"
|
||||
)
|
||||
|
||||
|
||||
async def _run_native_ingester(
|
||||
ctx: dict, source_config, mode: str | None, gdl, sync_session_factory,
|
||||
) -> tuple[DownloadResult, str | None]:
|
||||
@@ -123,9 +147,9 @@ async def _run_native_ingester(
|
||||
platform, ctx["url"], ctx["cookies_path"], overrides
|
||||
)
|
||||
if not campaign_id:
|
||||
# Only reachable for Patreon (SubscribeStar's campaign id is the URL).
|
||||
# Patreon: vanity lookup failed. Pixiv: no numeric user id in the URL.
|
||||
# (SubscribeStar's campaign id is the URL itself — never lands here.)
|
||||
url = ctx["url"]
|
||||
vanity = extract_vanity(url)
|
||||
return (
|
||||
DownloadResult(
|
||||
success=False,
|
||||
@@ -133,12 +157,7 @@ async def _run_native_ingester(
|
||||
artist_slug=ctx["artist_slug"],
|
||||
platform=platform,
|
||||
error_type=ErrorType.NOT_FOUND,
|
||||
error_message=(
|
||||
f"Could not resolve Patreon campaign id. source_url={url!r}; "
|
||||
f"vanity={vanity!r}; "
|
||||
f"lookup=GET {_CAMPAIGNS_API}?filter[vanity]={vanity or ''} "
|
||||
"(vanity lookup failed — cookies expired or creator moved?)"
|
||||
),
|
||||
error_message=_campaign_resolution_error(platform, url),
|
||||
),
|
||||
None,
|
||||
)
|
||||
@@ -161,6 +180,10 @@ async def _run_native_ingester(
|
||||
validate=gdl._validate_files,
|
||||
rate_limit=rate_limit,
|
||||
request_sleep=request_sleep,
|
||||
# Uniform across adapters: token platforms (pixiv) authenticate with
|
||||
# it, cookie platforms accept-and-ignore — so this construction stays
|
||||
# platform-agnostic.
|
||||
auth_token=ctx["auth_token"],
|
||||
)
|
||||
loop = asyncio.get_running_loop()
|
||||
dl_result = await loop.run_in_executor(
|
||||
@@ -190,6 +213,7 @@ async def preview_source(
|
||||
cookies_path: str | None,
|
||||
images_root: Path,
|
||||
sync_session_factory,
|
||||
auth_token: str | None = None,
|
||||
page_limit: int = 3,
|
||||
) -> dict:
|
||||
"""Dry-run preview for a native platform (plan #708 B4): resolve the campaign
|
||||
@@ -205,18 +229,12 @@ async def preview_source(
|
||||
platform, url, cookies_path, config_overrides or {}
|
||||
)
|
||||
if not campaign_id:
|
||||
vanity = extract_vanity(url)
|
||||
return {
|
||||
"error": (
|
||||
f"Couldn't resolve the campaign id. source_url={url!r}; "
|
||||
f"vanity={vanity!r}; lookup=GET {_CAMPAIGNS_API}?filter[vanity]={vanity or ''} "
|
||||
"(cookies expired, or the creator moved/renamed?)."
|
||||
)
|
||||
}
|
||||
return {"error": _campaign_resolution_error(platform, url)}
|
||||
ingester = _native_ingester_cls(platform)(
|
||||
images_root=images_root,
|
||||
cookies_path=cookies_path,
|
||||
session_factory=sync_session_factory,
|
||||
auth_token=auth_token,
|
||||
)
|
||||
loop = asyncio.get_running_loop()
|
||||
try:
|
||||
@@ -245,13 +263,18 @@ async def verify_source_credential(
|
||||
this and render the result.
|
||||
"""
|
||||
if uses_native_ingester(platform):
|
||||
# Native ingester platforms verify via their own lightweight auth probe
|
||||
# (one authenticated feed fetch). SubscribeStar's probe takes the creator
|
||||
# URL directly; Patreon's resolves the campaign id first.
|
||||
# Native ingester platforms verify via their own lightweight auth probe.
|
||||
# SubscribeStar's probe takes the creator URL directly; Patreon's
|
||||
# resolves the campaign id first; Pixiv's is one OAuth refresh (the
|
||||
# exact call that fails when the token is bad — no feed walk).
|
||||
if platform == "subscribestar":
|
||||
from .subscribestar_ingester import verify_subscribestar_credential
|
||||
|
||||
return await verify_subscribestar_credential(url, cookies_path, config_overrides)
|
||||
if platform == "pixiv":
|
||||
from .pixiv_ingester import verify_pixiv_credential
|
||||
|
||||
return await verify_pixiv_credential(auth_token)
|
||||
from .patreon_ingester import verify_patreon_credential
|
||||
|
||||
return await verify_patreon_credential(url, cookies_path, config_overrides)
|
||||
|
||||
Reference in New Issue
Block a user