From 8e3da096ecaf23a5bcdb29f80a8f33b8f16cded5 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 18 Apr 2026 13:43:21 -0400 Subject: [PATCH] feat(downloads): add patreon URL-rewrite and error-pattern helpers --- backend/app/tasks/downloads.py | 43 +++++++ backend/tests/tasks/__init__.py | 0 backend/tests/tasks/test_downloads_helpers.py | 118 ++++++++++++++++++ 3 files changed, 161 insertions(+) create mode 100644 backend/tests/tasks/__init__.py create mode 100644 backend/tests/tasks/test_downloads_helpers.py diff --git a/backend/app/tasks/downloads.py b/backend/app/tasks/downloads.py index 8390e98..720cb51 100644 --- a/backend/app/tasks/downloads.py +++ b/backend/app/tasks/downloads.py @@ -2,6 +2,7 @@ import asyncio import logging +import re from datetime import datetime, timedelta from typing import Optional @@ -70,6 +71,48 @@ def _get_db_setting_sync(key: str, default=None): return asyncio.run(_fetch()) +# --- Patreon campaign-ID auto-resolution helpers ------------------------- +# These support auto-healing gallery-dl's "Failed to extract campaign ID" +# failures on Patreon sources. See docs/superpowers/specs/2026-04-18-patreon-campaign-resolver-design.md. + +_PATREON_VANITY_RE = re.compile( + r"^https?://(?:www\.)?patreon\.com/(?:c/)?(?!id:)([^/?#]+)", + re.IGNORECASE, +) +_CAMPAIGN_ID_FAILURE_PATTERN = "failed to extract campaign id" + + +def _effective_patreon_url(platform: str, source_url: str, metadata: Optional[dict]) -> str: + """Return the URL to pass to gallery-dl for this source. + + For Patreon sources with a cached campaign_id in metadata, returns + patreon.com/id: to bypass the broken vanity-URL lookup path. + All other cases return source_url unchanged. + """ + if platform == "patreon" and metadata: + campaign_id = metadata.get("patreon_campaign_id") + if campaign_id: + return f"https://www.patreon.com/id:{campaign_id}" + return source_url + + +def _looks_like_campaign_id_failure(dl_result) -> bool: + """True if the gallery-dl output contains the campaign-ID extraction failure marker.""" + stdout = getattr(dl_result, "stdout", None) or "" + stderr = getattr(dl_result, "stderr", None) or "" + return _CAMPAIGN_ID_FAILURE_PATTERN in f"{stdout}\n{stderr}".lower() + + +def _extract_vanity_from_url(url: str) -> Optional[str]: + """Return the vanity (creator slug) from a Patreon URL, or None if not extractable. + + Handles patreon.com/, patreon.com/c/[/...], with or without www. + Returns None for id: URLs, bare domain, and non-Patreon URLs. + """ + m = _PATREON_VANITY_RE.match(url) + return m.group(1) if m else None + + async def _download_source_async(source_id: int, download_id: int = None) -> dict: """Async implementation of source download. diff --git a/backend/tests/tasks/__init__.py b/backend/tests/tasks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/tests/tasks/test_downloads_helpers.py b/backend/tests/tasks/test_downloads_helpers.py new file mode 100644 index 0000000..8812243 --- /dev/null +++ b/backend/tests/tasks/test_downloads_helpers.py @@ -0,0 +1,118 @@ +"""Unit tests for the pure helpers added to downloads.py.""" +from types import SimpleNamespace + +from app.tasks.downloads import ( + _effective_patreon_url, + _extract_vanity_from_url, + _looks_like_campaign_id_failure, +) + + +# --- _effective_patreon_url ------------------------------------------------- + +def test_effective_url_patreon_with_cached_id_uses_id_url(): + assert ( + _effective_patreon_url("patreon", "https://www.patreon.com/mstsu", {"patreon_campaign_id": "12345"}) + == "https://www.patreon.com/id:12345" + ) + + +def test_effective_url_patreon_without_cached_id_uses_original(): + assert ( + _effective_patreon_url("patreon", "https://www.patreon.com/mstsu", {}) + == "https://www.patreon.com/mstsu" + ) + + +def test_effective_url_patreon_none_metadata_uses_original(): + assert ( + _effective_patreon_url("patreon", "https://www.patreon.com/mstsu", None) + == "https://www.patreon.com/mstsu" + ) + + +def test_effective_url_non_patreon_ignores_cached_id(): + # Even if campaign_id somehow got set on a non-patreon source, don't rewrite. + assert ( + _effective_patreon_url("subscribestar", "https://subscribestar.adult/foo", {"patreon_campaign_id": "12345"}) + == "https://subscribestar.adult/foo" + ) + + +# --- _looks_like_campaign_id_failure --------------------------------------- + +def _result(stdout="", stderr=""): + return SimpleNamespace(stdout=stdout, stderr=stderr) + + +def test_failure_pattern_in_stderr(): + assert _looks_like_campaign_id_failure( + _result(stderr="[patreon][error] Failed to extract campaign ID") + ) is True + + +def test_failure_pattern_in_stdout(): + assert _looks_like_campaign_id_failure( + _result(stdout="something Failed to extract campaign ID something") + ) is True + + +def test_failure_pattern_case_insensitive(): + assert _looks_like_campaign_id_failure( + _result(stderr="FAILED TO EXTRACT CAMPAIGN ID") + ) is True + + +def test_failure_pattern_absent(): + assert _looks_like_campaign_id_failure( + _result(stderr="HTTP 401 Unauthorized") + ) is False + + +def test_failure_pattern_empty(): + assert _looks_like_campaign_id_failure(_result()) is False + + +def test_failure_pattern_none_streams(): + assert _looks_like_campaign_id_failure( + SimpleNamespace(stdout=None, stderr=None) + ) is False + + +# --- _extract_vanity_from_url ---------------------------------------------- + +def test_vanity_plain_patreon_url(): + assert _extract_vanity_from_url("https://www.patreon.com/mstsu") == "mstsu" + + +def test_vanity_with_c_prefix(): + assert _extract_vanity_from_url("https://www.patreon.com/c/mstsu/posts") == "mstsu" + + +def test_vanity_with_c_prefix_no_trailing_path(): + assert _extract_vanity_from_url("https://www.patreon.com/c/mstsu") == "mstsu" + + +def test_vanity_without_www(): + assert _extract_vanity_from_url("https://patreon.com/mstsu") == "mstsu" + + +def test_vanity_http_scheme(): + assert _extract_vanity_from_url("http://www.patreon.com/mstsu") == "mstsu" + + +def test_vanity_id_url_returns_none(): + assert _extract_vanity_from_url("https://www.patreon.com/id:12345") is None + + +def test_vanity_bare_domain_returns_none(): + assert _extract_vanity_from_url("https://www.patreon.com/") is None + + +def test_vanity_unrelated_url_returns_none(): + assert _extract_vanity_from_url("https://subscribestar.adult/mstsu") is None + + +def test_vanity_ignores_trailing_query_and_hash(): + assert _extract_vanity_from_url("https://www.patreon.com/mstsu?x=1") == "mstsu" + assert _extract_vanity_from_url("https://www.patreon.com/mstsu#anchor") == "mstsu"