feat(downloads): add patreon URL-rewrite and error-pattern helpers
This commit is contained in:
@@ -2,6 +2,7 @@
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Optional
|
||||
|
||||
@@ -70,6 +71,48 @@ def _get_db_setting_sync(key: str, default=None):
|
||||
return asyncio.run(_fetch())
|
||||
|
||||
|
||||
# --- Patreon campaign-ID auto-resolution helpers -------------------------
|
||||
# These support auto-healing gallery-dl's "Failed to extract campaign ID"
|
||||
# failures on Patreon sources. See docs/superpowers/specs/2026-04-18-patreon-campaign-resolver-design.md.
|
||||
|
||||
_PATREON_VANITY_RE = re.compile(
|
||||
r"^https?://(?:www\.)?patreon\.com/(?:c/)?(?!id:)([^/?#]+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_CAMPAIGN_ID_FAILURE_PATTERN = "failed to extract campaign id"
|
||||
|
||||
|
||||
def _effective_patreon_url(platform: str, source_url: str, metadata: Optional[dict]) -> str:
|
||||
"""Return the URL to pass to gallery-dl for this source.
|
||||
|
||||
For Patreon sources with a cached campaign_id in metadata, returns
|
||||
patreon.com/id:<id> to bypass the broken vanity-URL lookup path.
|
||||
All other cases return source_url unchanged.
|
||||
"""
|
||||
if platform == "patreon" and metadata:
|
||||
campaign_id = metadata.get("patreon_campaign_id")
|
||||
if campaign_id:
|
||||
return f"https://www.patreon.com/id:{campaign_id}"
|
||||
return source_url
|
||||
|
||||
|
||||
def _looks_like_campaign_id_failure(dl_result) -> bool:
|
||||
"""True if the gallery-dl output contains the campaign-ID extraction failure marker."""
|
||||
stdout = getattr(dl_result, "stdout", None) or ""
|
||||
stderr = getattr(dl_result, "stderr", None) or ""
|
||||
return _CAMPAIGN_ID_FAILURE_PATTERN in f"{stdout}\n{stderr}".lower()
|
||||
|
||||
|
||||
def _extract_vanity_from_url(url: str) -> Optional[str]:
|
||||
"""Return the vanity (creator slug) from a Patreon URL, or None if not extractable.
|
||||
|
||||
Handles patreon.com/<vanity>, patreon.com/c/<vanity>[/...], with or without www.
|
||||
Returns None for id:<N> URLs, bare domain, and non-Patreon URLs.
|
||||
"""
|
||||
m = _PATREON_VANITY_RE.match(url)
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
async def _download_source_async(source_id: int, download_id: int = None) -> dict:
|
||||
"""Async implementation of source download.
|
||||
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Unit tests for the pure helpers added to downloads.py."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from app.tasks.downloads import (
|
||||
_effective_patreon_url,
|
||||
_extract_vanity_from_url,
|
||||
_looks_like_campaign_id_failure,
|
||||
)
|
||||
|
||||
|
||||
# --- _effective_patreon_url -------------------------------------------------
|
||||
|
||||
def test_effective_url_patreon_with_cached_id_uses_id_url():
|
||||
assert (
|
||||
_effective_patreon_url("patreon", "https://www.patreon.com/mstsu", {"patreon_campaign_id": "12345"})
|
||||
== "https://www.patreon.com/id:12345"
|
||||
)
|
||||
|
||||
|
||||
def test_effective_url_patreon_without_cached_id_uses_original():
|
||||
assert (
|
||||
_effective_patreon_url("patreon", "https://www.patreon.com/mstsu", {})
|
||||
== "https://www.patreon.com/mstsu"
|
||||
)
|
||||
|
||||
|
||||
def test_effective_url_patreon_none_metadata_uses_original():
|
||||
assert (
|
||||
_effective_patreon_url("patreon", "https://www.patreon.com/mstsu", None)
|
||||
== "https://www.patreon.com/mstsu"
|
||||
)
|
||||
|
||||
|
||||
def test_effective_url_non_patreon_ignores_cached_id():
|
||||
# Even if campaign_id somehow got set on a non-patreon source, don't rewrite.
|
||||
assert (
|
||||
_effective_patreon_url("subscribestar", "https://subscribestar.adult/foo", {"patreon_campaign_id": "12345"})
|
||||
== "https://subscribestar.adult/foo"
|
||||
)
|
||||
|
||||
|
||||
# --- _looks_like_campaign_id_failure ---------------------------------------
|
||||
|
||||
def _result(stdout="", stderr=""):
|
||||
return SimpleNamespace(stdout=stdout, stderr=stderr)
|
||||
|
||||
|
||||
def test_failure_pattern_in_stderr():
|
||||
assert _looks_like_campaign_id_failure(
|
||||
_result(stderr="[patreon][error] Failed to extract campaign ID")
|
||||
) is True
|
||||
|
||||
|
||||
def test_failure_pattern_in_stdout():
|
||||
assert _looks_like_campaign_id_failure(
|
||||
_result(stdout="something Failed to extract campaign ID something")
|
||||
) is True
|
||||
|
||||
|
||||
def test_failure_pattern_case_insensitive():
|
||||
assert _looks_like_campaign_id_failure(
|
||||
_result(stderr="FAILED TO EXTRACT CAMPAIGN ID")
|
||||
) is True
|
||||
|
||||
|
||||
def test_failure_pattern_absent():
|
||||
assert _looks_like_campaign_id_failure(
|
||||
_result(stderr="HTTP 401 Unauthorized")
|
||||
) is False
|
||||
|
||||
|
||||
def test_failure_pattern_empty():
|
||||
assert _looks_like_campaign_id_failure(_result()) is False
|
||||
|
||||
|
||||
def test_failure_pattern_none_streams():
|
||||
assert _looks_like_campaign_id_failure(
|
||||
SimpleNamespace(stdout=None, stderr=None)
|
||||
) is False
|
||||
|
||||
|
||||
# --- _extract_vanity_from_url ----------------------------------------------
|
||||
|
||||
def test_vanity_plain_patreon_url():
|
||||
assert _extract_vanity_from_url("https://www.patreon.com/mstsu") == "mstsu"
|
||||
|
||||
|
||||
def test_vanity_with_c_prefix():
|
||||
assert _extract_vanity_from_url("https://www.patreon.com/c/mstsu/posts") == "mstsu"
|
||||
|
||||
|
||||
def test_vanity_with_c_prefix_no_trailing_path():
|
||||
assert _extract_vanity_from_url("https://www.patreon.com/c/mstsu") == "mstsu"
|
||||
|
||||
|
||||
def test_vanity_without_www():
|
||||
assert _extract_vanity_from_url("https://patreon.com/mstsu") == "mstsu"
|
||||
|
||||
|
||||
def test_vanity_http_scheme():
|
||||
assert _extract_vanity_from_url("http://www.patreon.com/mstsu") == "mstsu"
|
||||
|
||||
|
||||
def test_vanity_id_url_returns_none():
|
||||
assert _extract_vanity_from_url("https://www.patreon.com/id:12345") is None
|
||||
|
||||
|
||||
def test_vanity_bare_domain_returns_none():
|
||||
assert _extract_vanity_from_url("https://www.patreon.com/") is None
|
||||
|
||||
|
||||
def test_vanity_unrelated_url_returns_none():
|
||||
assert _extract_vanity_from_url("https://subscribestar.adult/mstsu") is None
|
||||
|
||||
|
||||
def test_vanity_ignores_trailing_query_and_hash():
|
||||
assert _extract_vanity_from_url("https://www.patreon.com/mstsu?x=1") == "mstsu"
|
||||
assert _extract_vanity_from_url("https://www.patreon.com/mstsu#anchor") == "mstsu"
|
||||
Reference in New Issue
Block a user