feat(downloads): native Patreon verify + uniform backend dispatch (plan #697)
CI / lint (push) Successful in 2s
CI / backend-lint-and-test (push) Successful in 12s
CI / frontend-build (push) Successful in 19s
CI / integration (push) Successful in 2m59s

The credential Verify button still ran gallery-dl --simulate for Patreon
after the cutover — testing the wrong path (and prone to the vanity
"Failed to extract campaign ID" the native resolver fixes). Wire it to the
native ingester, behind a DRY dispatch so callers never branch on platform.

- services/download_backends.py (new): the ONE place that knows which
  platforms are native vs gallery-dl. `uses_native_ingester(platform)` is
  the shared predicate; `verify_source_credential(...)` is the uniform
  probe (same (ok|None, message) contract for both backends). As a platform
  migrates, it moves into NATIVE_INGESTER_PLATFORMS here and BOTH download
  routing and verify switch together.
- PatreonClient.verify_auth(campaign_id): one authenticated /api/posts
  fetch → True (valid) / False (401/403/HTML-login) / None (drift or
  network — inconclusive, not a credential verdict).
- patreon_ingester.verify_patreon_credential(): resolve campaign id, then
  verify_auth — the verify counterpart to the download path.
- patreon_resolver.resolve_campaign_id_for_source(): extracted the
  override / id:-URL / vanity resolution into ONE helper now shared by the
  download ingester and verify (download_service no longer carries its own
  copy + regex; −`import re`).
- download_service: routes on uses_native_ingester() instead of inline
  `== "patreon"` (3 sites); uses the shared resolver.
- api/credentials: calls verify_source_credential — no platform branch.

Tests: verify_auth mapping, resolve_campaign_id_for_source (override/id:/
vanity/none), the dispatch predicate, verify_patreon_credential glue,
credentials endpoint proves Patreon uses the native path (gallery-dl verify
asserted not-called); repointed the gallery-dl verify test to subscribestar.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-05 22:49:43 -04:00
parent ec43e823e1
commit 218bfebb92
12 changed files with 371 additions and 50 deletions
+12 -10
View File
@@ -121,13 +121,15 @@ async def delete_credential(platform: str):
@credentials_bp.route("/<platform>/verify", methods=["POST"])
async def verify_credential(platform: str):
"""Test the stored credential by running gallery-dl --simulate
against one of the platform's enabled sources. On success stamps
last_verified. Returns {valid: bool|null, reason, last_verified?}.
valid=null means "couldn't test" (no credential, or no enabled
source to point at)."""
"""Test the stored credential against one of the platform's enabled sources,
WITHOUT downloading. Routes through the platform's backend
(download_backends.verify_credential) — native ingester for Patreon, an
authenticated API page; gallery-dl --simulate for the rest. On success
stamps last_verified. Returns {valid: bool|null, reason, last_verified?};
valid=null means "couldn't test" (no credential, no enabled source, or an
inconclusive network/drift result)."""
from ..models import Artist, Source
from ..services.gallery_dl import GalleryDLService, SourceConfig
from ..services.download_backends import verify_source_credential
async with get_session() as session:
if not await _ext_key_ok(session):
@@ -154,14 +156,14 @@ async def verify_credential(platform: str):
cookies_path = await svc.get_cookies_path(platform)
auth_token = await svc.get_token(platform)
gdl = GalleryDLService(images_root=Path("/images"))
ok, message = await gdl.verify(
ok, message = await verify_source_credential(
platform=platform,
url=source.url,
artist_slug=artist.slug,
platform=platform,
source_config=SourceConfig.from_dict(source.config_overrides or {}),
config_overrides=source.config_overrides or {},
cookies_path=str(cookies_path) if cookies_path else None,
auth_token=auth_token,
images_root=Path("/images"),
)
last_verified = None
+71
View File
@@ -0,0 +1,71 @@
"""Platform → download-backend dispatch (one place that knows which platforms
are served by the native FC ingester vs. the gallery-dl subprocess).
gallery-dl wasn't built to be driven by an automated scheduler — no native
checkpoint/resume, no structured logs, per-file HEADs that dominate wall-clock.
The native ingester (services/patreon_ingester.py, plan #697) replaces it for
Patreon and is the path we grow as more platforms migrate. To keep that
migration DRY, every caller that has to behave differently per backend —
download routing, the credential-verify probe, cursor handling — asks THIS
module instead of testing ``platform == "patreon"`` inline. When a platform gets
a native ingester, it moves into ``NATIVE_INGESTER_PLATFORMS`` here and both the
download path and verify switch over together.
The backend surfaces share a UNIFORM signature so a caller invokes the same
function regardless of platform:
- verify_credential(...) → (ok: bool|None, message: str)
- (download stays in download_service for now; uses_native_ingester() is the
shared predicate it routes on, so the decision lives here too.)
"""
from __future__ import annotations
from pathlib import Path
# Platforms whose download + verify go through the native ingester rather than
# gallery-dl. gallery-dl still serves every other platform (subscribestar,
# hentaifoundry, discord, pixiv, deviantart) unchanged.
NATIVE_INGESTER_PLATFORMS = frozenset({"patreon"})
def uses_native_ingester(platform: str) -> bool:
"""True when `platform` is served by the native ingester (not gallery-dl).
The single predicate the download path and verify both route on."""
return platform in NATIVE_INGESTER_PLATFORMS
async def verify_source_credential(
*,
platform: str,
url: str,
artist_slug: str,
config_overrides: dict | None,
cookies_path: str | None,
auth_token: str | None,
images_root: Path,
) -> tuple[bool | None, str]:
"""Uniform credential probe across backends. Returns `(ok, message)`:
True = authenticated, False = rejected, None = inconclusive (drift /
network / nothing to test). Callers don't branch on platform — they call
this and render the result.
"""
if uses_native_ingester(platform):
# Native ingester platforms verify via their own lightweight auth probe
# (resolve campaign id + one authenticated API page). Patreon today.
from .patreon_ingester import verify_patreon_credential
return await verify_patreon_credential(url, cookies_path, config_overrides)
# gallery-dl platforms: --simulate one item; the extractor errors before it
# can list if auth is bad.
from .gallery_dl import GalleryDLService, SourceConfig
gdl = GalleryDLService(images_root=images_root)
return await gdl.verify(
url=url,
artist_slug=artist_slug,
platform=platform,
source_config=SourceConfig.from_dict(config_overrides or {}),
cookies_path=cookies_path,
auth_token=auth_token,
)
+11 -33
View File
@@ -13,7 +13,6 @@ from __future__ import annotations
import asyncio
import logging
import re
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
@@ -25,6 +24,7 @@ from sqlalchemy.orm import joinedload
from ..models import Artist, DownloadEvent, Source
from .credential_service import CredentialService
from .download_backends import uses_native_ingester
from .gallery_dl import (
BACKFILL_CHUNK_SECONDS,
BACKFILL_SKIP_VALUE,
@@ -37,28 +37,13 @@ from .gallery_dl import (
)
from .importer import Importer
from .patreon_ingester import PatreonIngester
from .patreon_resolver import resolve_campaign_id
from .patreon_resolver import resolve_campaign_id_for_source
from .platforms import auth_type_for
from .scheduler_service import set_platform_cooldown
log = logging.getLogger(__name__)
# Vanity → campaign-id resolution is still needed by the native Patreon
# ingester (phase 2 resolves the campaign id before the walk). gallery-dl's
# reactive campaign-id retry + the `id:` effective-URL rewrite were removed at
# the #697 cutover (Patreon no longer flows through gallery-dl).
_PATREON_VANITY_RE = re.compile(
r"^https?://(?:www\.)?patreon\.com/(?:c/)?(?!id:)([^/?#]+)",
re.IGNORECASE,
)
def _extract_patreon_vanity(url: str) -> str | None:
m = _PATREON_VANITY_RE.match(url)
return m.group(1) if m else None
class DownloadService:
"""Async orchestrator. The Celery task runs `asyncio.run(svc.download_source(N))`.
@@ -131,13 +116,13 @@ class DownloadService:
skip_value: bool | str = BACKFILL_SKIP_VALUE
source_config.timeout = BACKFILL_CHUNK_SECONDS
pending_cursor = overrides.get("_backfill_cursor")
if ctx["platform"] == "patreon" and pending_cursor:
if uses_native_ingester(ctx["platform"]) and pending_cursor:
source_config.resume_cursor = pending_cursor
else:
skip_value = TICK_SKIP_VALUE
resolved_campaign_id: str | None = None
if ctx["platform"] == "patreon":
if uses_native_ingester(ctx["platform"]):
# Native ingester (plan #697) fully replaces gallery-dl for Patreon
# in phase 2 — zero per-file HEADs, native cursor/resume, loud drift
# detection. Returns a DownloadResult-shaped object so phase 3 is
@@ -198,19 +183,12 @@ class DownloadService:
silent empty success.
"""
overrides = ctx["config_overrides"] or {}
campaign_id = overrides.get("patreon_campaign_id")
resolved_campaign_id: str | None = None
if not campaign_id:
# A `.../id:<digits>` URL already carries the campaign id — no lookup
# needed (and the vanity regex deliberately excludes the id: form).
id_match = re.search(r"/id:(\d+)", ctx["url"])
if id_match:
campaign_id = id_match.group(1)
else:
vanity = _extract_patreon_vanity(ctx["url"])
if vanity:
campaign_id = await resolve_campaign_id(vanity, ctx["cookies_path"])
resolved_campaign_id = campaign_id
# Shared resolution path (override / id: URL / vanity lookup) — the same
# helper the credential-verify probe uses. resolved_campaign_id is
# non-None only when a vanity lookup ran, so phase 3 caches it.
campaign_id, resolved_campaign_id = await resolve_campaign_id_for_source(
ctx["url"], ctx["cookies_path"], overrides
)
if not campaign_id:
return (
@@ -540,7 +518,7 @@ class DownloadService:
# top), so they advance only by the download archive growing.
new_cursor = (
parse_last_cursor(dl_result.stdout, dl_result.stderr)
if ctx["platform"] == "patreon" else None
if uses_native_ingester(ctx["platform"]) else None
)
advanced = bool(
(new_cursor and new_cursor != old_cursor)
+24
View File
@@ -489,6 +489,30 @@ class PatreonClient:
return
current_cursor = next_cursor
# -- verify ------------------------------------------------------------
def verify_auth(self, campaign_id: str) -> tuple[bool | None, str]:
"""Cheap auth probe: fetch the first `/api/posts` page and report whether
the credential authenticated, WITHOUT downloading anything.
Returns `(ok, message)` matching the credential-verify contract:
- True — authenticated (the feed returned a valid JSON:API page).
- False — the credential was rejected (PatreonAuthError: 401/403, or an
HTML login page → cookies expired / tier insufficient).
- None — inconclusive: API drift (our parser is stale, not a cred
problem) or a transient network/HTTP error.
"""
try:
response = self._fetch(campaign_id, None)
self._validate_response(response)
except PatreonAuthError as exc:
return False, f"Patreon rejected the credential — {exc}"
except PatreonDriftError as exc:
return None, f"Couldn't verify — Patreon's API shape changed: {exc}"
except PatreonAPIError as exc:
return None, f"Couldn't verify (network/HTTP issue): {exc}"
return True, "Credentials valid — the Patreon feed authenticated."
def _dedup_by_filehash(items: list[MediaItem]) -> list[MediaItem]:
"""Drop later items sharing a filehash with an earlier one (first wins).
+25
View File
@@ -38,6 +38,7 @@ FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
from __future__ import annotations
import asyncio
import logging
import time
from collections.abc import Callable
@@ -56,6 +57,7 @@ from .patreon_client import (
PatreonDriftError,
)
from .patreon_downloader import PatreonDownloader
from .patreon_resolver import resolve_campaign_id_for_source
log = logging.getLogger(__name__)
@@ -377,3 +379,26 @@ class PatreonIngester:
)
session.execute(stmt)
session.commit()
async def verify_patreon_credential(
url: str,
cookies_path: str | None,
overrides: dict | None,
) -> tuple[bool | None, str]:
"""Native Patreon credential probe — the verify counterpart to the ingester's
download path, sharing its campaign-id resolution. Resolves the campaign id
(override / id: URL / vanity) then does ONE authenticated `/api/posts` fetch
via PatreonClient.verify_auth. Returns the uniform `(ok, message)` contract
(True / False / None) so download_backends.verify_credential can treat it
interchangeably with the gallery-dl probe. No download, no DB.
"""
campaign_id, _ = await resolve_campaign_id_for_source(url, cookies_path, overrides)
if not campaign_id:
return None, (
"Couldn't resolve the Patreon campaign id from the source URL — "
"can't verify (cookies expired, or the creator moved/renamed?)."
)
client = PatreonClient(cookies_path)
loop = asyncio.get_running_loop()
return await loop.run_in_executor(None, client.verify_auth, campaign_id)
+45
View File
@@ -19,12 +19,22 @@ import asyncio
import http.cookiejar
import logging
import os
import re
import requests
log = logging.getLogger(__name__)
_CAMPAIGNS_URL = "https://www.patreon.com/api/campaigns"
# A source URL of the form `.../id:<digits>` already carries the campaign id
# (no lookup needed). The vanity regex deliberately EXCLUDES the id: form so the
# two paths don't overlap.
_ID_URL_RE = re.compile(r"/id:(\d+)")
_VANITY_RE = re.compile(
r"^https?://(?:www\.)?patreon\.com/(?:c/)?(?!id:)([^/?#]+)",
re.IGNORECASE,
)
_USER_AGENT = (
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
@@ -100,3 +110,38 @@ async def resolve_campaign_id(
Never raises."""
loop = asyncio.get_running_loop()
return await loop.run_in_executor(None, _sync_lookup, vanity, cookies_path)
def extract_vanity(url: str) -> str | None:
"""The vanity slug from a Patreon creator URL, or None for an `id:` URL."""
m = _VANITY_RE.match(url or "")
return m.group(1) if m else None
async def resolve_campaign_id_for_source(
url: str,
cookies_path: str | None,
overrides: dict | None,
) -> tuple[str | None, str | None]:
"""Resolve a Patreon source to its campaign id — the single resolution path
shared by the download ingester and the credential-verify probe.
Order: cached `patreon_campaign_id` override → an `id:<digits>` URL → a
vanity lookup against the campaigns API. Returns
`(campaign_id, newly_resolved_id)`: `newly_resolved_id` is non-None ONLY when
a vanity lookup actually ran, so the caller knows to cache it on the source
(the override/id: paths needed no lookup). `(None, None)` when unresolvable.
Never raises.
"""
overrides = overrides or {}
cached = overrides.get("patreon_campaign_id")
if cached:
return cached, None
id_match = _ID_URL_RE.search(url or "")
if id_match:
return id_match.group(1), None
vanity = extract_vanity(url)
if vanity:
resolved = await resolve_campaign_id(vanity, cookies_path)
return resolved, resolved
return None, None
+41 -4
View File
@@ -155,6 +155,8 @@ async def test_verify_no_enabled_source_is_untestable(client):
@pytest.mark.asyncio
async def test_verify_runs_gallery_dl_and_stamps_on_success(client, db, monkeypatch):
"""A gallery-dl platform routes through GalleryDLService.verify; success
stamps last_verified (platform-agnostic endpoint behavior)."""
from backend.app.models import Artist, Source
from backend.app.services import gallery_dl as gdl_mod
@@ -163,6 +165,45 @@ async def test_verify_runs_gallery_dl_and_stamps_on_success(client, db, monkeypa
return (True, "Credentials valid — the feed authenticated.")
monkeypatch.setattr(gdl_mod.GalleryDLService, "verify", _fake_verify)
await client.post("/api/credentials", json={
"platform": "subscribestar", "credential_type": "cookies", "data": _NETSCAPE,
})
artist = Artist(name="Maewix", slug="maewix")
db.add(artist)
await db.flush()
db.add(Source(
artist_id=artist.id, platform="subscribestar",
url="https://www.subscribestar.com/maewix", enabled=True, config_overrides={},
))
await db.commit()
resp = await client.post("/api/credentials/subscribestar/verify")
body = await resp.get_json()
assert body["valid"] is True
assert body["last_verified"] is not None
# The stamp is persisted on the credential record.
rec = await (await client.get("/api/credentials/subscribestar")).get_json()
assert rec["last_verified"] is not None
@pytest.mark.asyncio
async def test_verify_patreon_uses_native_ingester_not_gallery_dl(client, db, monkeypatch):
"""plan #697 cutover: Patreon credential verify routes through the native
ingester (verify_patreon_credential), NOT gallery-dl --simulate."""
from backend.app.models import Artist, Source
from backend.app.services import gallery_dl as gdl_mod
from backend.app.services import patreon_ingester as pi_mod
async def _native_verify(url, cookies_path, overrides):
return (True, "Credentials valid — the Patreon feed authenticated.")
monkeypatch.setattr(pi_mod, "verify_patreon_credential", _native_verify)
# gallery-dl must NOT be consulted for Patreon.
async def _boom(self, *args, **kwargs):
raise AssertionError("gallery-dl verify must not run for patreon")
monkeypatch.setattr(gdl_mod.GalleryDLService, "verify", _boom)
await client.post("/api/credentials", json={
"platform": "patreon", "credential_type": "cookies", "data": _NETSCAPE,
})
@@ -180,10 +221,6 @@ async def test_verify_runs_gallery_dl_and_stamps_on_success(client, db, monkeypa
assert body["valid"] is True
assert body["last_verified"] is not None
# The stamp is persisted on the credential record.
rec = await (await client.get("/api/credentials/patreon")).get_json()
assert rec["last_verified"] is not None
@pytest.mark.asyncio
async def test_verify_reports_auth_failure(client, db, monkeypatch):
+23
View File
@@ -0,0 +1,23 @@
"""download_backends — the single predicate that routes a platform to the
native ingester vs. gallery-dl. Pure, no DB."""
from backend.app.services.download_backends import (
NATIVE_INGESTER_PLATFORMS,
uses_native_ingester,
)
def test_patreon_is_native():
assert uses_native_ingester("patreon") is True
assert "patreon" in NATIVE_INGESTER_PLATFORMS
def test_gallery_dl_platforms_are_not_native():
# The five platforms still served by gallery-dl must NOT route to the
# native ingester — guards an accidental over-broad migration.
for platform in ("subscribestar", "hentaifoundry", "discord", "pixiv", "deviantart"):
assert uses_native_ingester(platform) is False
def test_unknown_platform_is_not_native():
assert uses_native_ingester("nonsense") is False
+4 -2
View File
@@ -288,7 +288,8 @@ async def test_run_patreon_ingester_resolves_vanity_and_runs(
_artist, source = seed_artist_and_source
monkeypatch.setattr(
dl_mod, "resolve_campaign_id", AsyncMock(return_value="4242"),
dl_mod, "resolve_campaign_id_for_source",
AsyncMock(return_value=("4242", "4242")),
)
run_kwargs = {}
@@ -335,7 +336,8 @@ async def test_run_patreon_ingester_unresolvable_fails_loud(
_artist, source = seed_artist_and_source
monkeypatch.setattr(
dl_mod, "resolve_campaign_id", AsyncMock(return_value=None),
dl_mod, "resolve_campaign_id_for_source",
AsyncMock(return_value=(None, None)),
)
svc = DownloadService(
async_session=db, sync_session=db_sync,
+37
View File
@@ -238,3 +238,40 @@ def test_fetch_other_status_raises_api_error_with_code(monkeypatch, status):
def test_fetch_ok_returns_payload(monkeypatch):
client = _client_returning(monkeypatch, _FakeResp(200, json_data={"data": []}))
assert client._fetch("5555", None) == {"data": []}
# -- verify_auth (credential probe; (True/False/None, message) contract) ---
def test_verify_auth_ok_when_page_authenticates(monkeypatch):
client = _client_returning(monkeypatch, _FakeResp(200, json_data={"data": []}))
ok, msg = client.verify_auth("5555")
assert ok is True
assert "valid" in msg.lower()
@pytest.mark.parametrize("status", [401, 403])
def test_verify_auth_false_when_rejected(monkeypatch, status):
client = _client_returning(monkeypatch, _FakeResp(status))
ok, _msg = client.verify_auth("5555")
assert ok is False
def test_verify_auth_inconclusive_on_drift(monkeypatch):
# 200 + JSON but missing top-level 'data' → drift → inconclusive, NOT a
# credential verdict (our parser is stale, the cookie may be fine).
client = _client_returning(monkeypatch, _FakeResp(200, json_data={"included": []}))
ok, msg = client.verify_auth("5555")
assert ok is None
assert "api shape changed" in msg.lower()
def test_verify_auth_inconclusive_on_network(monkeypatch):
import requests
client = PatreonClient(cookies_path=None)
def _raise(*a, **k):
raise requests.ConnectionError("boom")
monkeypatch.setattr(client._session, "get", _raise)
ok, _msg = client.verify_auth("5555")
assert ok is None
+29
View File
@@ -370,3 +370,32 @@ def test_ledger_key_video_has_no_filehash():
kind="postfile", filehash=None, post_id="p9",
)
assert _ledger_key(video) == "p9:clip.mp4"
# --- verify_patreon_credential (the verify counterpart, plan #697) ---------
@pytest.mark.asyncio
async def test_verify_patreon_credential_unresolvable_is_inconclusive():
from backend.app.services.patreon_ingester import verify_patreon_credential
# No campaign id resolvable from a non-Patreon URL → can't test → None.
ok, msg = await verify_patreon_credential("not-a-patreon-url", None, {})
assert ok is None
assert "campaign id" in msg.lower()
@pytest.mark.asyncio
async def test_verify_patreon_credential_delegates_to_client(monkeypatch):
import backend.app.services.patreon_ingester as mod
async def _fake_resolve(url, cookies_path, overrides):
return "123", None
monkeypatch.setattr(mod, "resolve_campaign_id_for_source", _fake_resolve)
monkeypatch.setattr(
mod.PatreonClient, "verify_auth",
lambda self, campaign_id: (True, f"ok:{campaign_id}"),
)
ok, msg = await mod.verify_patreon_credential("https://patreon.com/x", None, {})
assert ok is True
assert msg == "ok:123"
+49 -1
View File
@@ -2,7 +2,10 @@ from unittest.mock import MagicMock, patch
import pytest
from backend.app.services.patreon_resolver import resolve_campaign_id
from backend.app.services.patreon_resolver import (
resolve_campaign_id,
resolve_campaign_id_for_source,
)
@pytest.mark.asyncio
@@ -86,3 +89,48 @@ async def test_loads_cookies_file_when_present(tmp_path):
assert result == "9"
_, kwargs = mock_get.call_args
assert kwargs.get("cookies") is not None
# -- resolve_campaign_id_for_source (shared by download + verify) ----------
@pytest.mark.asyncio
async def test_for_source_uses_cached_override_without_lookup():
with patch("backend.app.services.patreon_resolver.requests.get") as mock_get:
cid, resolved = await resolve_campaign_id_for_source(
"https://patreon.com/alice", None, {"patreon_campaign_id": "999"},
)
assert cid == "999"
assert resolved is None # cached → no newly-resolved id to cache
mock_get.assert_not_called()
@pytest.mark.asyncio
async def test_for_source_extracts_id_url_without_lookup():
with patch("backend.app.services.patreon_resolver.requests.get") as mock_get:
cid, resolved = await resolve_campaign_id_for_source(
"https://www.patreon.com/id:4242", None, {},
)
assert cid == "4242"
assert resolved is None
mock_get.assert_not_called()
@pytest.mark.asyncio
async def test_for_source_resolves_vanity_and_reports_it():
fake = MagicMock()
fake.status_code = 200
fake.json.return_value = {"data": [{"id": "777", "type": "campaign"}]}
with patch("backend.app.services.patreon_resolver.requests.get", return_value=fake):
cid, resolved = await resolve_campaign_id_for_source(
"https://patreon.com/alice", None, {},
)
assert cid == "777"
assert resolved == "777" # newly resolved → caller caches it
@pytest.mark.asyncio
async def test_for_source_unresolvable_returns_none():
cid, resolved = await resolve_campaign_id_for_source("not-a-patreon-url", None, {})
assert cid is None
assert resolved is None