feat(downloads): native Patreon verify + uniform backend dispatch (plan #697)
The credential Verify button still ran gallery-dl --simulate for Patreon after the cutover — testing the wrong path (and prone to the vanity "Failed to extract campaign ID" the native resolver fixes). Wire it to the native ingester, behind a DRY dispatch so callers never branch on platform. - services/download_backends.py (new): the ONE place that knows which platforms are native vs gallery-dl. `uses_native_ingester(platform)` is the shared predicate; `verify_source_credential(...)` is the uniform probe (same (ok|None, message) contract for both backends). As a platform migrates, it moves into NATIVE_INGESTER_PLATFORMS here and BOTH download routing and verify switch together. - PatreonClient.verify_auth(campaign_id): one authenticated /api/posts fetch → True (valid) / False (401/403/HTML-login) / None (drift or network — inconclusive, not a credential verdict). - patreon_ingester.verify_patreon_credential(): resolve campaign id, then verify_auth — the verify counterpart to the download path. - patreon_resolver.resolve_campaign_id_for_source(): extracted the override / id:-URL / vanity resolution into ONE helper now shared by the download ingester and verify (download_service no longer carries its own copy + regex; −`import re`). - download_service: routes on uses_native_ingester() instead of inline `== "patreon"` (3 sites); uses the shared resolver. - api/credentials: calls verify_source_credential — no platform branch. Tests: verify_auth mapping, resolve_campaign_id_for_source (override/id:/ vanity/none), the dispatch predicate, verify_patreon_credential glue, credentials endpoint proves Patreon uses the native path (gallery-dl verify asserted not-called); repointed the gallery-dl verify test to subscribestar. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -121,13 +121,15 @@ async def delete_credential(platform: str):
|
|||||||
|
|
||||||
@credentials_bp.route("/<platform>/verify", methods=["POST"])
|
@credentials_bp.route("/<platform>/verify", methods=["POST"])
|
||||||
async def verify_credential(platform: str):
|
async def verify_credential(platform: str):
|
||||||
"""Test the stored credential by running gallery-dl --simulate
|
"""Test the stored credential against one of the platform's enabled sources,
|
||||||
against one of the platform's enabled sources. On success stamps
|
WITHOUT downloading. Routes through the platform's backend
|
||||||
last_verified. Returns {valid: bool|null, reason, last_verified?}.
|
(download_backends.verify_credential) — native ingester for Patreon, an
|
||||||
valid=null means "couldn't test" (no credential, or no enabled
|
authenticated API page; gallery-dl --simulate for the rest. On success
|
||||||
source to point at)."""
|
stamps last_verified. Returns {valid: bool|null, reason, last_verified?};
|
||||||
|
valid=null means "couldn't test" (no credential, no enabled source, or an
|
||||||
|
inconclusive network/drift result)."""
|
||||||
from ..models import Artist, Source
|
from ..models import Artist, Source
|
||||||
from ..services.gallery_dl import GalleryDLService, SourceConfig
|
from ..services.download_backends import verify_source_credential
|
||||||
|
|
||||||
async with get_session() as session:
|
async with get_session() as session:
|
||||||
if not await _ext_key_ok(session):
|
if not await _ext_key_ok(session):
|
||||||
@@ -154,14 +156,14 @@ async def verify_credential(platform: str):
|
|||||||
cookies_path = await svc.get_cookies_path(platform)
|
cookies_path = await svc.get_cookies_path(platform)
|
||||||
auth_token = await svc.get_token(platform)
|
auth_token = await svc.get_token(platform)
|
||||||
|
|
||||||
gdl = GalleryDLService(images_root=Path("/images"))
|
ok, message = await verify_source_credential(
|
||||||
ok, message = await gdl.verify(
|
platform=platform,
|
||||||
url=source.url,
|
url=source.url,
|
||||||
artist_slug=artist.slug,
|
artist_slug=artist.slug,
|
||||||
platform=platform,
|
config_overrides=source.config_overrides or {},
|
||||||
source_config=SourceConfig.from_dict(source.config_overrides or {}),
|
|
||||||
cookies_path=str(cookies_path) if cookies_path else None,
|
cookies_path=str(cookies_path) if cookies_path else None,
|
||||||
auth_token=auth_token,
|
auth_token=auth_token,
|
||||||
|
images_root=Path("/images"),
|
||||||
)
|
)
|
||||||
|
|
||||||
last_verified = None
|
last_verified = None
|
||||||
|
|||||||
@@ -0,0 +1,71 @@
|
|||||||
|
"""Platform → download-backend dispatch (one place that knows which platforms
|
||||||
|
are served by the native FC ingester vs. the gallery-dl subprocess).
|
||||||
|
|
||||||
|
gallery-dl wasn't built to be driven by an automated scheduler — no native
|
||||||
|
checkpoint/resume, no structured logs, per-file HEADs that dominate wall-clock.
|
||||||
|
The native ingester (services/patreon_ingester.py, plan #697) replaces it for
|
||||||
|
Patreon and is the path we grow as more platforms migrate. To keep that
|
||||||
|
migration DRY, every caller that has to behave differently per backend —
|
||||||
|
download routing, the credential-verify probe, cursor handling — asks THIS
|
||||||
|
module instead of testing ``platform == "patreon"`` inline. When a platform gets
|
||||||
|
a native ingester, it moves into ``NATIVE_INGESTER_PLATFORMS`` here and both the
|
||||||
|
download path and verify switch over together.
|
||||||
|
|
||||||
|
The backend surfaces share a UNIFORM signature so a caller invokes the same
|
||||||
|
function regardless of platform:
|
||||||
|
- verify_credential(...) → (ok: bool|None, message: str)
|
||||||
|
- (download stays in download_service for now; uses_native_ingester() is the
|
||||||
|
shared predicate it routes on, so the decision lives here too.)
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
# Platforms whose download + verify go through the native ingester rather than
|
||||||
|
# gallery-dl. gallery-dl still serves every other platform (subscribestar,
|
||||||
|
# hentaifoundry, discord, pixiv, deviantart) unchanged.
|
||||||
|
NATIVE_INGESTER_PLATFORMS = frozenset({"patreon"})
|
||||||
|
|
||||||
|
|
||||||
|
def uses_native_ingester(platform: str) -> bool:
|
||||||
|
"""True when `platform` is served by the native ingester (not gallery-dl).
|
||||||
|
The single predicate the download path and verify both route on."""
|
||||||
|
return platform in NATIVE_INGESTER_PLATFORMS
|
||||||
|
|
||||||
|
|
||||||
|
async def verify_source_credential(
|
||||||
|
*,
|
||||||
|
platform: str,
|
||||||
|
url: str,
|
||||||
|
artist_slug: str,
|
||||||
|
config_overrides: dict | None,
|
||||||
|
cookies_path: str | None,
|
||||||
|
auth_token: str | None,
|
||||||
|
images_root: Path,
|
||||||
|
) -> tuple[bool | None, str]:
|
||||||
|
"""Uniform credential probe across backends. Returns `(ok, message)`:
|
||||||
|
True = authenticated, False = rejected, None = inconclusive (drift /
|
||||||
|
network / nothing to test). Callers don't branch on platform — they call
|
||||||
|
this and render the result.
|
||||||
|
"""
|
||||||
|
if uses_native_ingester(platform):
|
||||||
|
# Native ingester platforms verify via their own lightweight auth probe
|
||||||
|
# (resolve campaign id + one authenticated API page). Patreon today.
|
||||||
|
from .patreon_ingester import verify_patreon_credential
|
||||||
|
|
||||||
|
return await verify_patreon_credential(url, cookies_path, config_overrides)
|
||||||
|
|
||||||
|
# gallery-dl platforms: --simulate one item; the extractor errors before it
|
||||||
|
# can list if auth is bad.
|
||||||
|
from .gallery_dl import GalleryDLService, SourceConfig
|
||||||
|
|
||||||
|
gdl = GalleryDLService(images_root=images_root)
|
||||||
|
return await gdl.verify(
|
||||||
|
url=url,
|
||||||
|
artist_slug=artist_slug,
|
||||||
|
platform=platform,
|
||||||
|
source_config=SourceConfig.from_dict(config_overrides or {}),
|
||||||
|
cookies_path=cookies_path,
|
||||||
|
auth_token=auth_token,
|
||||||
|
)
|
||||||
@@ -13,7 +13,6 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
import re
|
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
@@ -25,6 +24,7 @@ from sqlalchemy.orm import joinedload
|
|||||||
|
|
||||||
from ..models import Artist, DownloadEvent, Source
|
from ..models import Artist, DownloadEvent, Source
|
||||||
from .credential_service import CredentialService
|
from .credential_service import CredentialService
|
||||||
|
from .download_backends import uses_native_ingester
|
||||||
from .gallery_dl import (
|
from .gallery_dl import (
|
||||||
BACKFILL_CHUNK_SECONDS,
|
BACKFILL_CHUNK_SECONDS,
|
||||||
BACKFILL_SKIP_VALUE,
|
BACKFILL_SKIP_VALUE,
|
||||||
@@ -37,28 +37,13 @@ from .gallery_dl import (
|
|||||||
)
|
)
|
||||||
from .importer import Importer
|
from .importer import Importer
|
||||||
from .patreon_ingester import PatreonIngester
|
from .patreon_ingester import PatreonIngester
|
||||||
from .patreon_resolver import resolve_campaign_id
|
from .patreon_resolver import resolve_campaign_id_for_source
|
||||||
from .platforms import auth_type_for
|
from .platforms import auth_type_for
|
||||||
from .scheduler_service import set_platform_cooldown
|
from .scheduler_service import set_platform_cooldown
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
# Vanity → campaign-id resolution is still needed by the native Patreon
|
|
||||||
# ingester (phase 2 resolves the campaign id before the walk). gallery-dl's
|
|
||||||
# reactive campaign-id retry + the `id:` effective-URL rewrite were removed at
|
|
||||||
# the #697 cutover (Patreon no longer flows through gallery-dl).
|
|
||||||
_PATREON_VANITY_RE = re.compile(
|
|
||||||
r"^https?://(?:www\.)?patreon\.com/(?:c/)?(?!id:)([^/?#]+)",
|
|
||||||
re.IGNORECASE,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _extract_patreon_vanity(url: str) -> str | None:
|
|
||||||
m = _PATREON_VANITY_RE.match(url)
|
|
||||||
return m.group(1) if m else None
|
|
||||||
|
|
||||||
|
|
||||||
class DownloadService:
|
class DownloadService:
|
||||||
"""Async orchestrator. The Celery task runs `asyncio.run(svc.download_source(N))`.
|
"""Async orchestrator. The Celery task runs `asyncio.run(svc.download_source(N))`.
|
||||||
|
|
||||||
@@ -131,13 +116,13 @@ class DownloadService:
|
|||||||
skip_value: bool | str = BACKFILL_SKIP_VALUE
|
skip_value: bool | str = BACKFILL_SKIP_VALUE
|
||||||
source_config.timeout = BACKFILL_CHUNK_SECONDS
|
source_config.timeout = BACKFILL_CHUNK_SECONDS
|
||||||
pending_cursor = overrides.get("_backfill_cursor")
|
pending_cursor = overrides.get("_backfill_cursor")
|
||||||
if ctx["platform"] == "patreon" and pending_cursor:
|
if uses_native_ingester(ctx["platform"]) and pending_cursor:
|
||||||
source_config.resume_cursor = pending_cursor
|
source_config.resume_cursor = pending_cursor
|
||||||
else:
|
else:
|
||||||
skip_value = TICK_SKIP_VALUE
|
skip_value = TICK_SKIP_VALUE
|
||||||
|
|
||||||
resolved_campaign_id: str | None = None
|
resolved_campaign_id: str | None = None
|
||||||
if ctx["platform"] == "patreon":
|
if uses_native_ingester(ctx["platform"]):
|
||||||
# Native ingester (plan #697) fully replaces gallery-dl for Patreon
|
# Native ingester (plan #697) fully replaces gallery-dl for Patreon
|
||||||
# in phase 2 — zero per-file HEADs, native cursor/resume, loud drift
|
# in phase 2 — zero per-file HEADs, native cursor/resume, loud drift
|
||||||
# detection. Returns a DownloadResult-shaped object so phase 3 is
|
# detection. Returns a DownloadResult-shaped object so phase 3 is
|
||||||
@@ -198,19 +183,12 @@ class DownloadService:
|
|||||||
silent empty success.
|
silent empty success.
|
||||||
"""
|
"""
|
||||||
overrides = ctx["config_overrides"] or {}
|
overrides = ctx["config_overrides"] or {}
|
||||||
campaign_id = overrides.get("patreon_campaign_id")
|
# Shared resolution path (override / id: URL / vanity lookup) — the same
|
||||||
resolved_campaign_id: str | None = None
|
# helper the credential-verify probe uses. resolved_campaign_id is
|
||||||
if not campaign_id:
|
# non-None only when a vanity lookup ran, so phase 3 caches it.
|
||||||
# A `.../id:<digits>` URL already carries the campaign id — no lookup
|
campaign_id, resolved_campaign_id = await resolve_campaign_id_for_source(
|
||||||
# needed (and the vanity regex deliberately excludes the id: form).
|
ctx["url"], ctx["cookies_path"], overrides
|
||||||
id_match = re.search(r"/id:(\d+)", ctx["url"])
|
)
|
||||||
if id_match:
|
|
||||||
campaign_id = id_match.group(1)
|
|
||||||
else:
|
|
||||||
vanity = _extract_patreon_vanity(ctx["url"])
|
|
||||||
if vanity:
|
|
||||||
campaign_id = await resolve_campaign_id(vanity, ctx["cookies_path"])
|
|
||||||
resolved_campaign_id = campaign_id
|
|
||||||
|
|
||||||
if not campaign_id:
|
if not campaign_id:
|
||||||
return (
|
return (
|
||||||
@@ -540,7 +518,7 @@ class DownloadService:
|
|||||||
# top), so they advance only by the download archive growing.
|
# top), so they advance only by the download archive growing.
|
||||||
new_cursor = (
|
new_cursor = (
|
||||||
parse_last_cursor(dl_result.stdout, dl_result.stderr)
|
parse_last_cursor(dl_result.stdout, dl_result.stderr)
|
||||||
if ctx["platform"] == "patreon" else None
|
if uses_native_ingester(ctx["platform"]) else None
|
||||||
)
|
)
|
||||||
advanced = bool(
|
advanced = bool(
|
||||||
(new_cursor and new_cursor != old_cursor)
|
(new_cursor and new_cursor != old_cursor)
|
||||||
|
|||||||
@@ -489,6 +489,30 @@ class PatreonClient:
|
|||||||
return
|
return
|
||||||
current_cursor = next_cursor
|
current_cursor = next_cursor
|
||||||
|
|
||||||
|
# -- verify ------------------------------------------------------------
|
||||||
|
|
||||||
|
def verify_auth(self, campaign_id: str) -> tuple[bool | None, str]:
|
||||||
|
"""Cheap auth probe: fetch the first `/api/posts` page and report whether
|
||||||
|
the credential authenticated, WITHOUT downloading anything.
|
||||||
|
|
||||||
|
Returns `(ok, message)` matching the credential-verify contract:
|
||||||
|
- True — authenticated (the feed returned a valid JSON:API page).
|
||||||
|
- False — the credential was rejected (PatreonAuthError: 401/403, or an
|
||||||
|
HTML login page → cookies expired / tier insufficient).
|
||||||
|
- None — inconclusive: API drift (our parser is stale, not a cred
|
||||||
|
problem) or a transient network/HTTP error.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
response = self._fetch(campaign_id, None)
|
||||||
|
self._validate_response(response)
|
||||||
|
except PatreonAuthError as exc:
|
||||||
|
return False, f"Patreon rejected the credential — {exc}"
|
||||||
|
except PatreonDriftError as exc:
|
||||||
|
return None, f"Couldn't verify — Patreon's API shape changed: {exc}"
|
||||||
|
except PatreonAPIError as exc:
|
||||||
|
return None, f"Couldn't verify (network/HTTP issue): {exc}"
|
||||||
|
return True, "Credentials valid — the Patreon feed authenticated."
|
||||||
|
|
||||||
|
|
||||||
def _dedup_by_filehash(items: list[MediaItem]) -> list[MediaItem]:
|
def _dedup_by_filehash(items: list[MediaItem]) -> list[MediaItem]:
|
||||||
"""Drop later items sharing a filehash with an earlier one (first wins).
|
"""Drop later items sharing a filehash with an earlier one (first wins).
|
||||||
|
|||||||
@@ -38,6 +38,7 @@ FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
import time
|
import time
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
@@ -56,6 +57,7 @@ from .patreon_client import (
|
|||||||
PatreonDriftError,
|
PatreonDriftError,
|
||||||
)
|
)
|
||||||
from .patreon_downloader import PatreonDownloader
|
from .patreon_downloader import PatreonDownloader
|
||||||
|
from .patreon_resolver import resolve_campaign_id_for_source
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -377,3 +379,26 @@ class PatreonIngester:
|
|||||||
)
|
)
|
||||||
session.execute(stmt)
|
session.execute(stmt)
|
||||||
session.commit()
|
session.commit()
|
||||||
|
|
||||||
|
|
||||||
|
async def verify_patreon_credential(
|
||||||
|
url: str,
|
||||||
|
cookies_path: str | None,
|
||||||
|
overrides: dict | None,
|
||||||
|
) -> tuple[bool | None, str]:
|
||||||
|
"""Native Patreon credential probe — the verify counterpart to the ingester's
|
||||||
|
download path, sharing its campaign-id resolution. Resolves the campaign id
|
||||||
|
(override / id: URL / vanity) then does ONE authenticated `/api/posts` fetch
|
||||||
|
via PatreonClient.verify_auth. Returns the uniform `(ok, message)` contract
|
||||||
|
(True / False / None) so download_backends.verify_credential can treat it
|
||||||
|
interchangeably with the gallery-dl probe. No download, no DB.
|
||||||
|
"""
|
||||||
|
campaign_id, _ = await resolve_campaign_id_for_source(url, cookies_path, overrides)
|
||||||
|
if not campaign_id:
|
||||||
|
return None, (
|
||||||
|
"Couldn't resolve the Patreon campaign id from the source URL — "
|
||||||
|
"can't verify (cookies expired, or the creator moved/renamed?)."
|
||||||
|
)
|
||||||
|
client = PatreonClient(cookies_path)
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
return await loop.run_in_executor(None, client.verify_auth, campaign_id)
|
||||||
|
|||||||
@@ -19,12 +19,22 @@ import asyncio
|
|||||||
import http.cookiejar
|
import http.cookiejar
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
_CAMPAIGNS_URL = "https://www.patreon.com/api/campaigns"
|
_CAMPAIGNS_URL = "https://www.patreon.com/api/campaigns"
|
||||||
|
|
||||||
|
# A source URL of the form `.../id:<digits>` already carries the campaign id
|
||||||
|
# (no lookup needed). The vanity regex deliberately EXCLUDES the id: form so the
|
||||||
|
# two paths don't overlap.
|
||||||
|
_ID_URL_RE = re.compile(r"/id:(\d+)")
|
||||||
|
_VANITY_RE = re.compile(
|
||||||
|
r"^https?://(?:www\.)?patreon\.com/(?:c/)?(?!id:)([^/?#]+)",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
_USER_AGENT = (
|
_USER_AGENT = (
|
||||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||||
"(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
|
"(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
|
||||||
@@ -100,3 +110,38 @@ async def resolve_campaign_id(
|
|||||||
Never raises."""
|
Never raises."""
|
||||||
loop = asyncio.get_running_loop()
|
loop = asyncio.get_running_loop()
|
||||||
return await loop.run_in_executor(None, _sync_lookup, vanity, cookies_path)
|
return await loop.run_in_executor(None, _sync_lookup, vanity, cookies_path)
|
||||||
|
|
||||||
|
|
||||||
|
def extract_vanity(url: str) -> str | None:
|
||||||
|
"""The vanity slug from a Patreon creator URL, or None for an `id:` URL."""
|
||||||
|
m = _VANITY_RE.match(url or "")
|
||||||
|
return m.group(1) if m else None
|
||||||
|
|
||||||
|
|
||||||
|
async def resolve_campaign_id_for_source(
|
||||||
|
url: str,
|
||||||
|
cookies_path: str | None,
|
||||||
|
overrides: dict | None,
|
||||||
|
) -> tuple[str | None, str | None]:
|
||||||
|
"""Resolve a Patreon source to its campaign id — the single resolution path
|
||||||
|
shared by the download ingester and the credential-verify probe.
|
||||||
|
|
||||||
|
Order: cached `patreon_campaign_id` override → an `id:<digits>` URL → a
|
||||||
|
vanity lookup against the campaigns API. Returns
|
||||||
|
`(campaign_id, newly_resolved_id)`: `newly_resolved_id` is non-None ONLY when
|
||||||
|
a vanity lookup actually ran, so the caller knows to cache it on the source
|
||||||
|
(the override/id: paths needed no lookup). `(None, None)` when unresolvable.
|
||||||
|
Never raises.
|
||||||
|
"""
|
||||||
|
overrides = overrides or {}
|
||||||
|
cached = overrides.get("patreon_campaign_id")
|
||||||
|
if cached:
|
||||||
|
return cached, None
|
||||||
|
id_match = _ID_URL_RE.search(url or "")
|
||||||
|
if id_match:
|
||||||
|
return id_match.group(1), None
|
||||||
|
vanity = extract_vanity(url)
|
||||||
|
if vanity:
|
||||||
|
resolved = await resolve_campaign_id(vanity, cookies_path)
|
||||||
|
return resolved, resolved
|
||||||
|
return None, None
|
||||||
|
|||||||
@@ -155,6 +155,8 @@ async def test_verify_no_enabled_source_is_untestable(client):
|
|||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_verify_runs_gallery_dl_and_stamps_on_success(client, db, monkeypatch):
|
async def test_verify_runs_gallery_dl_and_stamps_on_success(client, db, monkeypatch):
|
||||||
|
"""A gallery-dl platform routes through GalleryDLService.verify; success
|
||||||
|
stamps last_verified (platform-agnostic endpoint behavior)."""
|
||||||
from backend.app.models import Artist, Source
|
from backend.app.models import Artist, Source
|
||||||
from backend.app.services import gallery_dl as gdl_mod
|
from backend.app.services import gallery_dl as gdl_mod
|
||||||
|
|
||||||
@@ -163,6 +165,45 @@ async def test_verify_runs_gallery_dl_and_stamps_on_success(client, db, monkeypa
|
|||||||
return (True, "Credentials valid — the feed authenticated.")
|
return (True, "Credentials valid — the feed authenticated.")
|
||||||
monkeypatch.setattr(gdl_mod.GalleryDLService, "verify", _fake_verify)
|
monkeypatch.setattr(gdl_mod.GalleryDLService, "verify", _fake_verify)
|
||||||
|
|
||||||
|
await client.post("/api/credentials", json={
|
||||||
|
"platform": "subscribestar", "credential_type": "cookies", "data": _NETSCAPE,
|
||||||
|
})
|
||||||
|
artist = Artist(name="Maewix", slug="maewix")
|
||||||
|
db.add(artist)
|
||||||
|
await db.flush()
|
||||||
|
db.add(Source(
|
||||||
|
artist_id=artist.id, platform="subscribestar",
|
||||||
|
url="https://www.subscribestar.com/maewix", enabled=True, config_overrides={},
|
||||||
|
))
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
resp = await client.post("/api/credentials/subscribestar/verify")
|
||||||
|
body = await resp.get_json()
|
||||||
|
assert body["valid"] is True
|
||||||
|
assert body["last_verified"] is not None
|
||||||
|
|
||||||
|
# The stamp is persisted on the credential record.
|
||||||
|
rec = await (await client.get("/api/credentials/subscribestar")).get_json()
|
||||||
|
assert rec["last_verified"] is not None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_verify_patreon_uses_native_ingester_not_gallery_dl(client, db, monkeypatch):
|
||||||
|
"""plan #697 cutover: Patreon credential verify routes through the native
|
||||||
|
ingester (verify_patreon_credential), NOT gallery-dl --simulate."""
|
||||||
|
from backend.app.models import Artist, Source
|
||||||
|
from backend.app.services import gallery_dl as gdl_mod
|
||||||
|
from backend.app.services import patreon_ingester as pi_mod
|
||||||
|
|
||||||
|
async def _native_verify(url, cookies_path, overrides):
|
||||||
|
return (True, "Credentials valid — the Patreon feed authenticated.")
|
||||||
|
monkeypatch.setattr(pi_mod, "verify_patreon_credential", _native_verify)
|
||||||
|
|
||||||
|
# gallery-dl must NOT be consulted for Patreon.
|
||||||
|
async def _boom(self, *args, **kwargs):
|
||||||
|
raise AssertionError("gallery-dl verify must not run for patreon")
|
||||||
|
monkeypatch.setattr(gdl_mod.GalleryDLService, "verify", _boom)
|
||||||
|
|
||||||
await client.post("/api/credentials", json={
|
await client.post("/api/credentials", json={
|
||||||
"platform": "patreon", "credential_type": "cookies", "data": _NETSCAPE,
|
"platform": "patreon", "credential_type": "cookies", "data": _NETSCAPE,
|
||||||
})
|
})
|
||||||
@@ -180,10 +221,6 @@ async def test_verify_runs_gallery_dl_and_stamps_on_success(client, db, monkeypa
|
|||||||
assert body["valid"] is True
|
assert body["valid"] is True
|
||||||
assert body["last_verified"] is not None
|
assert body["last_verified"] is not None
|
||||||
|
|
||||||
# The stamp is persisted on the credential record.
|
|
||||||
rec = await (await client.get("/api/credentials/patreon")).get_json()
|
|
||||||
assert rec["last_verified"] is not None
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_verify_reports_auth_failure(client, db, monkeypatch):
|
async def test_verify_reports_auth_failure(client, db, monkeypatch):
|
||||||
|
|||||||
@@ -0,0 +1,23 @@
|
|||||||
|
"""download_backends — the single predicate that routes a platform to the
|
||||||
|
native ingester vs. gallery-dl. Pure, no DB."""
|
||||||
|
|
||||||
|
from backend.app.services.download_backends import (
|
||||||
|
NATIVE_INGESTER_PLATFORMS,
|
||||||
|
uses_native_ingester,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_patreon_is_native():
|
||||||
|
assert uses_native_ingester("patreon") is True
|
||||||
|
assert "patreon" in NATIVE_INGESTER_PLATFORMS
|
||||||
|
|
||||||
|
|
||||||
|
def test_gallery_dl_platforms_are_not_native():
|
||||||
|
# The five platforms still served by gallery-dl must NOT route to the
|
||||||
|
# native ingester — guards an accidental over-broad migration.
|
||||||
|
for platform in ("subscribestar", "hentaifoundry", "discord", "pixiv", "deviantart"):
|
||||||
|
assert uses_native_ingester(platform) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_unknown_platform_is_not_native():
|
||||||
|
assert uses_native_ingester("nonsense") is False
|
||||||
@@ -288,7 +288,8 @@ async def test_run_patreon_ingester_resolves_vanity_and_runs(
|
|||||||
_artist, source = seed_artist_and_source
|
_artist, source = seed_artist_and_source
|
||||||
|
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
dl_mod, "resolve_campaign_id", AsyncMock(return_value="4242"),
|
dl_mod, "resolve_campaign_id_for_source",
|
||||||
|
AsyncMock(return_value=("4242", "4242")),
|
||||||
)
|
)
|
||||||
run_kwargs = {}
|
run_kwargs = {}
|
||||||
|
|
||||||
@@ -335,7 +336,8 @@ async def test_run_patreon_ingester_unresolvable_fails_loud(
|
|||||||
_artist, source = seed_artist_and_source
|
_artist, source = seed_artist_and_source
|
||||||
|
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
dl_mod, "resolve_campaign_id", AsyncMock(return_value=None),
|
dl_mod, "resolve_campaign_id_for_source",
|
||||||
|
AsyncMock(return_value=(None, None)),
|
||||||
)
|
)
|
||||||
svc = DownloadService(
|
svc = DownloadService(
|
||||||
async_session=db, sync_session=db_sync,
|
async_session=db, sync_session=db_sync,
|
||||||
|
|||||||
@@ -238,3 +238,40 @@ def test_fetch_other_status_raises_api_error_with_code(monkeypatch, status):
|
|||||||
def test_fetch_ok_returns_payload(monkeypatch):
|
def test_fetch_ok_returns_payload(monkeypatch):
|
||||||
client = _client_returning(monkeypatch, _FakeResp(200, json_data={"data": []}))
|
client = _client_returning(monkeypatch, _FakeResp(200, json_data={"data": []}))
|
||||||
assert client._fetch("5555", None) == {"data": []}
|
assert client._fetch("5555", None) == {"data": []}
|
||||||
|
|
||||||
|
|
||||||
|
# -- verify_auth (credential probe; (True/False/None, message) contract) ---
|
||||||
|
|
||||||
|
|
||||||
|
def test_verify_auth_ok_when_page_authenticates(monkeypatch):
|
||||||
|
client = _client_returning(monkeypatch, _FakeResp(200, json_data={"data": []}))
|
||||||
|
ok, msg = client.verify_auth("5555")
|
||||||
|
assert ok is True
|
||||||
|
assert "valid" in msg.lower()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("status", [401, 403])
|
||||||
|
def test_verify_auth_false_when_rejected(monkeypatch, status):
|
||||||
|
client = _client_returning(monkeypatch, _FakeResp(status))
|
||||||
|
ok, _msg = client.verify_auth("5555")
|
||||||
|
assert ok is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_verify_auth_inconclusive_on_drift(monkeypatch):
|
||||||
|
# 200 + JSON but missing top-level 'data' → drift → inconclusive, NOT a
|
||||||
|
# credential verdict (our parser is stale, the cookie may be fine).
|
||||||
|
client = _client_returning(monkeypatch, _FakeResp(200, json_data={"included": []}))
|
||||||
|
ok, msg = client.verify_auth("5555")
|
||||||
|
assert ok is None
|
||||||
|
assert "api shape changed" in msg.lower()
|
||||||
|
|
||||||
|
|
||||||
|
def test_verify_auth_inconclusive_on_network(monkeypatch):
|
||||||
|
import requests
|
||||||
|
client = PatreonClient(cookies_path=None)
|
||||||
|
|
||||||
|
def _raise(*a, **k):
|
||||||
|
raise requests.ConnectionError("boom")
|
||||||
|
monkeypatch.setattr(client._session, "get", _raise)
|
||||||
|
ok, _msg = client.verify_auth("5555")
|
||||||
|
assert ok is None
|
||||||
|
|||||||
@@ -370,3 +370,32 @@ def test_ledger_key_video_has_no_filehash():
|
|||||||
kind="postfile", filehash=None, post_id="p9",
|
kind="postfile", filehash=None, post_id="p9",
|
||||||
)
|
)
|
||||||
assert _ledger_key(video) == "p9:clip.mp4"
|
assert _ledger_key(video) == "p9:clip.mp4"
|
||||||
|
|
||||||
|
|
||||||
|
# --- verify_patreon_credential (the verify counterpart, plan #697) ---------
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_verify_patreon_credential_unresolvable_is_inconclusive():
|
||||||
|
from backend.app.services.patreon_ingester import verify_patreon_credential
|
||||||
|
|
||||||
|
# No campaign id resolvable from a non-Patreon URL → can't test → None.
|
||||||
|
ok, msg = await verify_patreon_credential("not-a-patreon-url", None, {})
|
||||||
|
assert ok is None
|
||||||
|
assert "campaign id" in msg.lower()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_verify_patreon_credential_delegates_to_client(monkeypatch):
|
||||||
|
import backend.app.services.patreon_ingester as mod
|
||||||
|
|
||||||
|
async def _fake_resolve(url, cookies_path, overrides):
|
||||||
|
return "123", None
|
||||||
|
monkeypatch.setattr(mod, "resolve_campaign_id_for_source", _fake_resolve)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
mod.PatreonClient, "verify_auth",
|
||||||
|
lambda self, campaign_id: (True, f"ok:{campaign_id}"),
|
||||||
|
)
|
||||||
|
ok, msg = await mod.verify_patreon_credential("https://patreon.com/x", None, {})
|
||||||
|
assert ok is True
|
||||||
|
assert msg == "ok:123"
|
||||||
|
|||||||
@@ -2,7 +2,10 @@ from unittest.mock import MagicMock, patch
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from backend.app.services.patreon_resolver import resolve_campaign_id
|
from backend.app.services.patreon_resolver import (
|
||||||
|
resolve_campaign_id,
|
||||||
|
resolve_campaign_id_for_source,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@@ -86,3 +89,48 @@ async def test_loads_cookies_file_when_present(tmp_path):
|
|||||||
assert result == "9"
|
assert result == "9"
|
||||||
_, kwargs = mock_get.call_args
|
_, kwargs = mock_get.call_args
|
||||||
assert kwargs.get("cookies") is not None
|
assert kwargs.get("cookies") is not None
|
||||||
|
|
||||||
|
|
||||||
|
# -- resolve_campaign_id_for_source (shared by download + verify) ----------
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_for_source_uses_cached_override_without_lookup():
|
||||||
|
with patch("backend.app.services.patreon_resolver.requests.get") as mock_get:
|
||||||
|
cid, resolved = await resolve_campaign_id_for_source(
|
||||||
|
"https://patreon.com/alice", None, {"patreon_campaign_id": "999"},
|
||||||
|
)
|
||||||
|
assert cid == "999"
|
||||||
|
assert resolved is None # cached → no newly-resolved id to cache
|
||||||
|
mock_get.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_for_source_extracts_id_url_without_lookup():
|
||||||
|
with patch("backend.app.services.patreon_resolver.requests.get") as mock_get:
|
||||||
|
cid, resolved = await resolve_campaign_id_for_source(
|
||||||
|
"https://www.patreon.com/id:4242", None, {},
|
||||||
|
)
|
||||||
|
assert cid == "4242"
|
||||||
|
assert resolved is None
|
||||||
|
mock_get.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_for_source_resolves_vanity_and_reports_it():
|
||||||
|
fake = MagicMock()
|
||||||
|
fake.status_code = 200
|
||||||
|
fake.json.return_value = {"data": [{"id": "777", "type": "campaign"}]}
|
||||||
|
with patch("backend.app.services.patreon_resolver.requests.get", return_value=fake):
|
||||||
|
cid, resolved = await resolve_campaign_id_for_source(
|
||||||
|
"https://patreon.com/alice", None, {},
|
||||||
|
)
|
||||||
|
assert cid == "777"
|
||||||
|
assert resolved == "777" # newly resolved → caller caches it
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_for_source_unresolvable_returns_none():
|
||||||
|
cid, resolved = await resolve_campaign_id_for_source("not-a-patreon-url", None, {})
|
||||||
|
assert cid is None
|
||||||
|
assert resolved is None
|
||||||
|
|||||||
Reference in New Issue
Block a user