feat(downloads): add patreon URL-rewrite and error-pattern helpers

This commit is contained in:
2026-04-18 13:43:21 -04:00
parent 3642acc41c
commit 8e3da096ec
3 changed files with 161 additions and 0 deletions
+43
View File
@@ -2,6 +2,7 @@
import asyncio
import logging
import re
from datetime import datetime, timedelta
from typing import Optional
@@ -70,6 +71,48 @@ def _get_db_setting_sync(key: str, default=None):
return asyncio.run(_fetch())
# --- Patreon campaign-ID auto-resolution helpers -------------------------
# These support auto-healing gallery-dl's "Failed to extract campaign ID"
# failures on Patreon sources. See docs/superpowers/specs/2026-04-18-patreon-campaign-resolver-design.md.
_PATREON_VANITY_RE = re.compile(
r"^https?://(?:www\.)?patreon\.com/(?:c/)?(?!id:)([^/?#]+)",
re.IGNORECASE,
)
_CAMPAIGN_ID_FAILURE_PATTERN = "failed to extract campaign id"
def _effective_patreon_url(platform: str, source_url: str, metadata: Optional[dict]) -> str:
"""Return the URL to pass to gallery-dl for this source.
For Patreon sources with a cached campaign_id in metadata, returns
patreon.com/id:<id> to bypass the broken vanity-URL lookup path.
All other cases return source_url unchanged.
"""
if platform == "patreon" and metadata:
campaign_id = metadata.get("patreon_campaign_id")
if campaign_id:
return f"https://www.patreon.com/id:{campaign_id}"
return source_url
def _looks_like_campaign_id_failure(dl_result) -> bool:
"""True if the gallery-dl output contains the campaign-ID extraction failure marker."""
stdout = getattr(dl_result, "stdout", None) or ""
stderr = getattr(dl_result, "stderr", None) or ""
return _CAMPAIGN_ID_FAILURE_PATTERN in f"{stdout}\n{stderr}".lower()
def _extract_vanity_from_url(url: str) -> Optional[str]:
"""Return the vanity (creator slug) from a Patreon URL, or None if not extractable.
Handles patreon.com/<vanity>, patreon.com/c/<vanity>[/...], with or without www.
Returns None for id:<N> URLs, bare domain, and non-Patreon URLs.
"""
m = _PATREON_VANITY_RE.match(url)
return m.group(1) if m else None
async def _download_source_async(source_id: int, download_id: int = None) -> dict:
"""Async implementation of source download.
View File
@@ -0,0 +1,118 @@
"""Unit tests for the pure helpers added to downloads.py."""
from types import SimpleNamespace
from app.tasks.downloads import (
_effective_patreon_url,
_extract_vanity_from_url,
_looks_like_campaign_id_failure,
)
# --- _effective_patreon_url -------------------------------------------------
def test_effective_url_patreon_with_cached_id_uses_id_url():
assert (
_effective_patreon_url("patreon", "https://www.patreon.com/mstsu", {"patreon_campaign_id": "12345"})
== "https://www.patreon.com/id:12345"
)
def test_effective_url_patreon_without_cached_id_uses_original():
assert (
_effective_patreon_url("patreon", "https://www.patreon.com/mstsu", {})
== "https://www.patreon.com/mstsu"
)
def test_effective_url_patreon_none_metadata_uses_original():
assert (
_effective_patreon_url("patreon", "https://www.patreon.com/mstsu", None)
== "https://www.patreon.com/mstsu"
)
def test_effective_url_non_patreon_ignores_cached_id():
# Even if campaign_id somehow got set on a non-patreon source, don't rewrite.
assert (
_effective_patreon_url("subscribestar", "https://subscribestar.adult/foo", {"patreon_campaign_id": "12345"})
== "https://subscribestar.adult/foo"
)
# --- _looks_like_campaign_id_failure ---------------------------------------
def _result(stdout="", stderr=""):
return SimpleNamespace(stdout=stdout, stderr=stderr)
def test_failure_pattern_in_stderr():
assert _looks_like_campaign_id_failure(
_result(stderr="[patreon][error] Failed to extract campaign ID")
) is True
def test_failure_pattern_in_stdout():
assert _looks_like_campaign_id_failure(
_result(stdout="something Failed to extract campaign ID something")
) is True
def test_failure_pattern_case_insensitive():
assert _looks_like_campaign_id_failure(
_result(stderr="FAILED TO EXTRACT CAMPAIGN ID")
) is True
def test_failure_pattern_absent():
assert _looks_like_campaign_id_failure(
_result(stderr="HTTP 401 Unauthorized")
) is False
def test_failure_pattern_empty():
assert _looks_like_campaign_id_failure(_result()) is False
def test_failure_pattern_none_streams():
assert _looks_like_campaign_id_failure(
SimpleNamespace(stdout=None, stderr=None)
) is False
# --- _extract_vanity_from_url ----------------------------------------------
def test_vanity_plain_patreon_url():
assert _extract_vanity_from_url("https://www.patreon.com/mstsu") == "mstsu"
def test_vanity_with_c_prefix():
assert _extract_vanity_from_url("https://www.patreon.com/c/mstsu/posts") == "mstsu"
def test_vanity_with_c_prefix_no_trailing_path():
assert _extract_vanity_from_url("https://www.patreon.com/c/mstsu") == "mstsu"
def test_vanity_without_www():
assert _extract_vanity_from_url("https://patreon.com/mstsu") == "mstsu"
def test_vanity_http_scheme():
assert _extract_vanity_from_url("http://www.patreon.com/mstsu") == "mstsu"
def test_vanity_id_url_returns_none():
assert _extract_vanity_from_url("https://www.patreon.com/id:12345") is None
def test_vanity_bare_domain_returns_none():
assert _extract_vanity_from_url("https://www.patreon.com/") is None
def test_vanity_unrelated_url_returns_none():
assert _extract_vanity_from_url("https://subscribestar.adult/mstsu") is None
def test_vanity_ignores_trailing_query_and_hash():
assert _extract_vanity_from_url("https://www.patreon.com/mstsu?x=1") == "mstsu"
assert _extract_vanity_from_url("https://www.patreon.com/mstsu#anchor") == "mstsu"