Pixiv native ingester (#129) + SubscribeStar .art fix, disable-clears-failure, system-tag suggestion floor #191

Merged
bvandeusen merged 9 commits from dev into main 2026-07-03 16:36:56 -04:00
8 changed files with 87 additions and 42 deletions
Showing only changes of commit 0bbcdee3bd - Show all commits
+2
View File
@@ -238,6 +238,7 @@ async def preview_source_endpoint(source_id: int):
)
cred = CredentialService(session, _get_crypto())
cookies_path = await cred.get_cookies_path(rec.platform)
auth_token = await cred.get_token(rec.platform)
# The walk + ledger reads are sync (run off the request loop); the process
# sync engine is the same one the download task uses.
@@ -249,6 +250,7 @@ async def preview_source_endpoint(source_id: int):
cookies_path=str(cookies_path) if cookies_path else None,
images_root=Path("/images"),
sync_session_factory=sync_session_factory(),
auth_token=auth_token,
)
if "error" in result:
return _bad("preview_failed", detail=result["error"], status=409)
+46 -23
View File
@@ -27,12 +27,15 @@ from .gallery_dl import DownloadResult, ErrorType
from .native_ingest_common import NativeIngestError
from .patreon_ingester import PatreonIngester
from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source
from .pixiv_client import user_id_from_url
from .pixiv_ingester import PixivIngester
from .subscribestar_ingester import SubscribeStarIngester
# Platforms whose download + verify go through the native ingester rather than
# gallery-dl. gallery-dl still serves the rest (hentaifoundry, discord, pixiv,
# deviantart) until they migrate too.
NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar"})
# gallery-dl. gallery-dl still serves the rest (hentaifoundry, discord,
# deviantart — the latter slated for retirement, not migration) until they
# migrate too.
NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar", "pixiv"})
# Mirrors patreon_resolver._CAMPAIGNS_URL — surfaced in resolution-failure
# messages so the operator sees the exact lookup endpoint that was hit.
@@ -46,6 +49,7 @@ def _native_ingester_cls(platform: str):
dispatch pick up the replacement."""
return {
"patreon": PatreonIngester,
"pixiv": PixivIngester,
"subscribestar": SubscribeStarIngester,
}[platform]
@@ -98,14 +102,34 @@ async def _resolve_native_campaign_id(
platform: str, url: str, cookies_path: str | None, overrides: dict,
) -> tuple[str | None, str | None]:
"""`(campaign_id, resolved_campaign_id)` for a native source. SubscribeStar's
feed id IS the creator URL (no lookup → resolved None). Patreon resolves the
feed id IS the creator URL; Pixiv's is the numeric user id parsed straight
from it (no lookup → resolved None either way). Patreon resolves the
campaign id from the vanity URL (resolved non-None when a lookup actually ran,
so phase 3 caches it)."""
if platform == "subscribestar":
return url, None
if platform == "pixiv":
return user_id_from_url(url), None
return await resolve_campaign_id_for_source(url, cookies_path, overrides)
def _campaign_resolution_error(platform: str, url: str) -> str:
"""Operator-facing message for a native source whose campaign id could not
be resolved — names the platform's own lookup mechanism."""
if platform == "pixiv":
return (
f"Could not extract a pixiv user id. source_url={url!r} — expected "
"a URL like https://www.pixiv.net/users/<id>."
)
vanity = extract_vanity(url)
return (
f"Could not resolve Patreon campaign id. source_url={url!r}; "
f"vanity={vanity!r}; "
f"lookup=GET {_CAMPAIGNS_API}?filter[vanity]={vanity or ''} "
"(vanity lookup failed — cookies expired or creator moved?)"
)
async def _run_native_ingester(
ctx: dict, source_config, mode: str | None, gdl, sync_session_factory,
) -> tuple[DownloadResult, str | None]:
@@ -123,9 +147,9 @@ async def _run_native_ingester(
platform, ctx["url"], ctx["cookies_path"], overrides
)
if not campaign_id:
# Only reachable for Patreon (SubscribeStar's campaign id is the URL).
# Patreon: vanity lookup failed. Pixiv: no numeric user id in the URL.
# (SubscribeStar's campaign id is the URL itself — never lands here.)
url = ctx["url"]
vanity = extract_vanity(url)
return (
DownloadResult(
success=False,
@@ -133,12 +157,7 @@ async def _run_native_ingester(
artist_slug=ctx["artist_slug"],
platform=platform,
error_type=ErrorType.NOT_FOUND,
error_message=(
f"Could not resolve Patreon campaign id. source_url={url!r}; "
f"vanity={vanity!r}; "
f"lookup=GET {_CAMPAIGNS_API}?filter[vanity]={vanity or ''} "
"(vanity lookup failed — cookies expired or creator moved?)"
),
error_message=_campaign_resolution_error(platform, url),
),
None,
)
@@ -161,6 +180,10 @@ async def _run_native_ingester(
validate=gdl._validate_files,
rate_limit=rate_limit,
request_sleep=request_sleep,
# Uniform across adapters: token platforms (pixiv) authenticate with
# it, cookie platforms accept-and-ignore — so this construction stays
# platform-agnostic.
auth_token=ctx["auth_token"],
)
loop = asyncio.get_running_loop()
dl_result = await loop.run_in_executor(
@@ -190,6 +213,7 @@ async def preview_source(
cookies_path: str | None,
images_root: Path,
sync_session_factory,
auth_token: str | None = None,
page_limit: int = 3,
) -> dict:
"""Dry-run preview for a native platform (plan #708 B4): resolve the campaign
@@ -205,18 +229,12 @@ async def preview_source(
platform, url, cookies_path, config_overrides or {}
)
if not campaign_id:
vanity = extract_vanity(url)
return {
"error": (
f"Couldn't resolve the campaign id. source_url={url!r}; "
f"vanity={vanity!r}; lookup=GET {_CAMPAIGNS_API}?filter[vanity]={vanity or ''} "
"(cookies expired, or the creator moved/renamed?)."
)
}
return {"error": _campaign_resolution_error(platform, url)}
ingester = _native_ingester_cls(platform)(
images_root=images_root,
cookies_path=cookies_path,
session_factory=sync_session_factory,
auth_token=auth_token,
)
loop = asyncio.get_running_loop()
try:
@@ -245,13 +263,18 @@ async def verify_source_credential(
this and render the result.
"""
if uses_native_ingester(platform):
# Native ingester platforms verify via their own lightweight auth probe
# (one authenticated feed fetch). SubscribeStar's probe takes the creator
# URL directly; Patreon's resolves the campaign id first.
# Native ingester platforms verify via their own lightweight auth probe.
# SubscribeStar's probe takes the creator URL directly; Patreon's
# resolves the campaign id first; Pixiv's is one OAuth refresh (the
# exact call that fails when the token is bad — no feed walk).
if platform == "subscribestar":
from .subscribestar_ingester import verify_subscribestar_credential
return await verify_subscribestar_credential(url, cookies_path, config_overrides)
if platform == "pixiv":
from .pixiv_ingester import verify_pixiv_credential
return await verify_pixiv_credential(auth_token)
from .patreon_ingester import verify_patreon_credential
return await verify_patreon_credential(url, cookies_path, config_overrides)
+3 -13
View File
@@ -298,8 +298,9 @@ class GalleryDLService:
# removed at the plan-#697 cutover — it now uses the native ingester
# (services/patreon_ingester.py), not gallery-dl.
PLATFORM_DEFAULTS = {
# subscribestar removed — it's a native-ingester platform now (#71); the
# remaining entries are the gallery-dl platforms not yet migrated.
# subscribestar removed — native-ingester platform now (#71); pixiv
# removed likewise (#129). The remaining entries are the gallery-dl
# platforms not yet migrated.
"hentaifoundry": {
"content_types": ["all"],
"directory": [],
@@ -315,12 +316,6 @@ class GalleryDLService:
"reactions": False,
"threads": True,
},
"pixiv": {
"content_types": ["all"],
"directory": ["{category}"],
"filename": "{id}_{title[:50]}_{num:>02}.{extension}",
"ugoira": True,
},
"deviantart": {
"content_types": ["all"],
"directory": [],
@@ -694,9 +689,6 @@ class GalleryDLService:
if auth_token and platform == "discord":
config["extractor"].setdefault("discord", {})
config["extractor"]["discord"]["token"] = auth_token
if auth_token and platform == "pixiv":
config["extractor"].setdefault("pixiv", {})
config["extractor"]["pixiv"]["refresh-token"] = auth_token
with tempfile.NamedTemporaryFile(
mode="w", suffix=".json", delete=False, dir=str(self._config_dir),
@@ -882,8 +874,6 @@ class GalleryDLService:
config["extractor"]["cookies"] = cookies_path
if auth_token and platform == "discord":
config["extractor"].setdefault("discord", {})["token"] = auth_token
if auth_token and platform == "pixiv":
config["extractor"].setdefault("pixiv", {})["refresh-token"] = auth_token
with tempfile.NamedTemporaryFile(
mode="w", suffix=".json", delete=False, dir=str(self._config_dir),
+5
View File
@@ -87,9 +87,14 @@ class PatreonIngester(Ingester):
validate: bool = True,
rate_limit: float = 0.0,
request_sleep: float = 0.0,
auth_token: str | None = None,
client: PatreonClient | None = None,
downloader: PatreonDownloader | None = None,
):
# auth_token: accepted for the uniform native-ingester construction
# (download_backends passes it to every adapter); Patreon
# authenticates by cookies, so it's unused here.
del auth_token
self.images_root = Path(images_root)
self.cookies_path = str(cookies_path) if cookies_path else None
# Pacing (plan #703): request_sleep paces the API page fetches,
+9 -3
View File
@@ -1,8 +1,14 @@
"""Pixiv — one quirk.
post_url: gallery-dl's `url` is the image URL on `i.pximg.net`. The
post permalink follows /artworks/<id>. external_post_id (= `id`) was
already correct, so no override there.
post_url: the sidecar's `url` (legacy gallery-dl era) is the image URL
on `i.pximg.net`, and the native post record (#129) writes no url key
at all — the permalink is synthesized from `id` here either way:
/artworks/<id>. external_post_id (= `id`) was already correct, so no
override there.
Downloads run through the native ingester (pixiv_ingester.py), not
gallery-dl; this registry entry still owns URL validation, sidecar
parsing, and the credential surface (the OAuth refresh token).
"""
from .base import GD_DEFAULTS, PlatformInfo, str_id_value
@@ -61,9 +61,14 @@ class SubscribeStarIngester(Ingester):
validate: bool = True,
rate_limit: float = 0.0,
request_sleep: float = 0.0,
auth_token: str | None = None,
client: SubscribeStarClient | None = None,
downloader: SubscribeStarDownloader | None = None,
):
# auth_token: accepted for the uniform native-ingester construction
# (download_backends passes it to every adapter); SubscribeStar
# authenticates by cookies, so it's unused here.
del auth_token
self.images_root = Path(images_root)
self.cookies_path = str(cookies_path) if cookies_path else None
resolved_client = (
@@ -76,7 +76,8 @@ const recovering = computed(() => !!props.source.backfill_bypass_seen)
const recapturing = computed(() => !!props.source.backfill_recapture)
// Recover / recapture are native-ingester features (ledger-bypass re-walk and
// post-text re-grab), available to every native platform — not just Patreon.
const NATIVE_PLATFORMS = ['patreon', 'subscribestar']
// Mirrors backend download_backends.NATIVE_INGESTER_PLATFORMS.
const NATIVE_PLATFORMS = ['patreon', 'subscribestar', 'pixiv']
const isNative = computed(() => NATIVE_PLATFORMS.includes(props.source.platform))
</script>
+15 -2
View File
@@ -3,12 +3,15 @@ native ingester vs. gallery-dl. Pure, no DB."""
from backend.app.services.download_backends import (
NATIVE_INGESTER_PLATFORMS,
_campaign_resolution_error,
_native_ingester_cls,
uses_native_ingester,
)
from backend.app.services.pixiv_ingester import PixivIngester
def test_native_platforms():
for platform in ("patreon", "subscribestar"):
for platform in ("patreon", "subscribestar", "pixiv"):
assert uses_native_ingester(platform) is True
assert platform in NATIVE_INGESTER_PLATFORMS
@@ -16,9 +19,19 @@ def test_native_platforms():
def test_gallery_dl_platforms_are_not_native():
# The platforms still served by gallery-dl must NOT route to the native
# ingester — guards an accidental over-broad migration.
for platform in ("hentaifoundry", "discord", "pixiv", "deviantart"):
for platform in ("hentaifoundry", "discord", "deviantart"):
assert uses_native_ingester(platform) is False
def test_unknown_platform_is_not_native():
assert uses_native_ingester("nonsense") is False
def test_pixiv_dispatches_to_its_ingester():
assert _native_ingester_cls("pixiv") is PixivIngester
def test_pixiv_resolution_error_names_the_expected_url_shape():
msg = _campaign_resolution_error("pixiv", "https://www.pixiv.net/artworks/1")
assert "pixiv user id" in msg
assert "users/<id>" in msg