feat(downloads): native Patreon verify + uniform backend dispatch (plan #697)
The credential Verify button still ran gallery-dl --simulate for Patreon after the cutover — testing the wrong path (and prone to the vanity "Failed to extract campaign ID" the native resolver fixes). Wire it to the native ingester, behind a DRY dispatch so callers never branch on platform. - services/download_backends.py (new): the ONE place that knows which platforms are native vs gallery-dl. `uses_native_ingester(platform)` is the shared predicate; `verify_source_credential(...)` is the uniform probe (same (ok|None, message) contract for both backends). As a platform migrates, it moves into NATIVE_INGESTER_PLATFORMS here and BOTH download routing and verify switch together. - PatreonClient.verify_auth(campaign_id): one authenticated /api/posts fetch → True (valid) / False (401/403/HTML-login) / None (drift or network — inconclusive, not a credential verdict). - patreon_ingester.verify_patreon_credential(): resolve campaign id, then verify_auth — the verify counterpart to the download path. - patreon_resolver.resolve_campaign_id_for_source(): extracted the override / id:-URL / vanity resolution into ONE helper now shared by the download ingester and verify (download_service no longer carries its own copy + regex; −`import re`). - download_service: routes on uses_native_ingester() instead of inline `== "patreon"` (3 sites); uses the shared resolver. - api/credentials: calls verify_source_credential — no platform branch. Tests: verify_auth mapping, resolve_campaign_id_for_source (override/id:/ vanity/none), the dispatch predicate, verify_patreon_credential glue, credentials endpoint proves Patreon uses the native path (gallery-dl verify asserted not-called); repointed the gallery-dl verify test to subscribestar. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -13,7 +13,6 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
@@ -25,6 +24,7 @@ from sqlalchemy.orm import joinedload
|
||||
|
||||
from ..models import Artist, DownloadEvent, Source
|
||||
from .credential_service import CredentialService
|
||||
from .download_backends import uses_native_ingester
|
||||
from .gallery_dl import (
|
||||
BACKFILL_CHUNK_SECONDS,
|
||||
BACKFILL_SKIP_VALUE,
|
||||
@@ -37,28 +37,13 @@ from .gallery_dl import (
|
||||
)
|
||||
from .importer import Importer
|
||||
from .patreon_ingester import PatreonIngester
|
||||
from .patreon_resolver import resolve_campaign_id
|
||||
from .patreon_resolver import resolve_campaign_id_for_source
|
||||
from .platforms import auth_type_for
|
||||
from .scheduler_service import set_platform_cooldown
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Vanity → campaign-id resolution is still needed by the native Patreon
|
||||
# ingester (phase 2 resolves the campaign id before the walk). gallery-dl's
|
||||
# reactive campaign-id retry + the `id:` effective-URL rewrite were removed at
|
||||
# the #697 cutover (Patreon no longer flows through gallery-dl).
|
||||
_PATREON_VANITY_RE = re.compile(
|
||||
r"^https?://(?:www\.)?patreon\.com/(?:c/)?(?!id:)([^/?#]+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _extract_patreon_vanity(url: str) -> str | None:
|
||||
m = _PATREON_VANITY_RE.match(url)
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
class DownloadService:
|
||||
"""Async orchestrator. The Celery task runs `asyncio.run(svc.download_source(N))`.
|
||||
|
||||
@@ -131,13 +116,13 @@ class DownloadService:
|
||||
skip_value: bool | str = BACKFILL_SKIP_VALUE
|
||||
source_config.timeout = BACKFILL_CHUNK_SECONDS
|
||||
pending_cursor = overrides.get("_backfill_cursor")
|
||||
if ctx["platform"] == "patreon" and pending_cursor:
|
||||
if uses_native_ingester(ctx["platform"]) and pending_cursor:
|
||||
source_config.resume_cursor = pending_cursor
|
||||
else:
|
||||
skip_value = TICK_SKIP_VALUE
|
||||
|
||||
resolved_campaign_id: str | None = None
|
||||
if ctx["platform"] == "patreon":
|
||||
if uses_native_ingester(ctx["platform"]):
|
||||
# Native ingester (plan #697) fully replaces gallery-dl for Patreon
|
||||
# in phase 2 — zero per-file HEADs, native cursor/resume, loud drift
|
||||
# detection. Returns a DownloadResult-shaped object so phase 3 is
|
||||
@@ -198,19 +183,12 @@ class DownloadService:
|
||||
silent empty success.
|
||||
"""
|
||||
overrides = ctx["config_overrides"] or {}
|
||||
campaign_id = overrides.get("patreon_campaign_id")
|
||||
resolved_campaign_id: str | None = None
|
||||
if not campaign_id:
|
||||
# A `.../id:<digits>` URL already carries the campaign id — no lookup
|
||||
# needed (and the vanity regex deliberately excludes the id: form).
|
||||
id_match = re.search(r"/id:(\d+)", ctx["url"])
|
||||
if id_match:
|
||||
campaign_id = id_match.group(1)
|
||||
else:
|
||||
vanity = _extract_patreon_vanity(ctx["url"])
|
||||
if vanity:
|
||||
campaign_id = await resolve_campaign_id(vanity, ctx["cookies_path"])
|
||||
resolved_campaign_id = campaign_id
|
||||
# Shared resolution path (override / id: URL / vanity lookup) — the same
|
||||
# helper the credential-verify probe uses. resolved_campaign_id is
|
||||
# non-None only when a vanity lookup ran, so phase 3 caches it.
|
||||
campaign_id, resolved_campaign_id = await resolve_campaign_id_for_source(
|
||||
ctx["url"], ctx["cookies_path"], overrides
|
||||
)
|
||||
|
||||
if not campaign_id:
|
||||
return (
|
||||
@@ -540,7 +518,7 @@ class DownloadService:
|
||||
# top), so they advance only by the download archive growing.
|
||||
new_cursor = (
|
||||
parse_last_cursor(dl_result.stdout, dl_result.stderr)
|
||||
if ctx["platform"] == "patreon" else None
|
||||
if uses_native_ingester(ctx["platform"]) else None
|
||||
)
|
||||
advanced = bool(
|
||||
(new_cursor and new_cursor != old_cursor)
|
||||
|
||||
Reference in New Issue
Block a user