feat(downloads): auto-heal patreon campaign-ID failures via resolver retry

2026-04-18 14:11:11 -04:00
parent 8e3da096ec
commit 35589e996a
2 changed files with 308 additions and 3 deletions
@@ -3,6 +3,7 @@
 import asyncio
 import logging
 import re
+from dataclasses import dataclass
 from datetime import datetime, timedelta
 from typing import Optional

@@ -19,6 +20,7 @@ from app.models.download import Download, DownloadStatus, ErrorType as DBErrorTy
 from app.models.setting import Setting, DEFAULT_SETTINGS
 from app.services.gallery_dl import GalleryDLService, SourceConfig, DownloadResult
 from app.services.credential_manager import CredentialManager
+from app.services.patreon_resolver import resolve_campaign_id
 from app.events import publish_event

 logger = logging.getLogger(__name__)
@@ -113,6 +115,71 @@ def _extract_vanity_from_url(url: str) -> Optional[str]:
    return m.group(1) if m else None


+@dataclass
+class _DownloadAttempt:
+    """Result of executing a download, possibly with a retry after campaign-ID resolution."""
+    dl_result: DownloadResult
+    resolved_campaign_id: Optional[str]
+
+
+async def _execute_download_with_retry(
+    *,
+    gdl_service,
+    platform: str,
+    source_url: str,
+    source_metadata: dict,
+    subscription_name: str,
+    source_config,
+    cookies_path: Optional[str],
+    auth_token: Optional[str],
+) -> _DownloadAttempt:
+    """Run the download once, auto-heal Patreon campaign-ID failures with one retry.
+
+    Returns a _DownloadAttempt wrapping:
+      - the final DownloadResult (retry's result if we retried, else the first)
+      - resolved_campaign_id: the newly-resolved ID (if we resolved one this call), else None
+
+    The caller is responsible for persisting resolved_campaign_id into source.metadata_.
+    """
+    effective_url = _effective_patreon_url(platform, source_url, source_metadata)
+    dl_result = await gdl_service.download(
+        url=effective_url,
+        subscription_name=subscription_name,
+        platform=platform,
+        source_config=source_config,
+        cookies_path=cookies_path,
+        auth_token=auth_token,
+    )
+
+    # Only retry if: patreon + campaign-ID failure + no cached id + extractable vanity
+    if (
+        platform == "patreon"
+        and not dl_result.success
+        and "patreon_campaign_id" not in (source_metadata or {})
+        and _looks_like_campaign_id_failure(dl_result)
+    ):
+        vanity = _extract_vanity_from_url(source_url)
+        if vanity:
+            logger.info(
+                f"Attempting campaign-ID resolution for {subscription_name}/patreon ({vanity})"
+            )
+            campaign_id = await resolve_campaign_id(vanity, cookies_path)
+            if campaign_id:
+                retry_result = await gdl_service.download(
+                    url=f"https://www.patreon.com/id:{campaign_id}",
+                    subscription_name=subscription_name,
+                    platform=platform,
+                    source_config=source_config,
+                    cookies_path=cookies_path,
+                    auth_token=auth_token,
+                )
+                return _DownloadAttempt(
+                    dl_result=retry_result, resolved_campaign_id=campaign_id
+                )
+
+    return _DownloadAttempt(dl_result=dl_result, resolved_campaign_id=None)
+
+
 async def _download_source_async(source_id: int, download_id: int = None) -> dict:
    """Async implementation of source download.

@@ -255,17 +322,22 @@ async def _download_source_async(source_id: int, download_id: int = None) -> dic
    source_config = SourceConfig.from_dict(source_metadata)
    dl_result = None
    dl_error = None
+    resolved_campaign_id: Optional[str] = None

    try:
        gdl_service = GalleryDLService(rate_limit=db_rate_limit)
-        dl_result = await gdl_service.download(
-            url=source_url,
-            subscription_name=subscription_name,
+        attempt = await _execute_download_with_retry(
+            gdl_service=gdl_service,
            platform=source_platform,
+            source_url=source_url,
+            source_metadata=source_metadata,
+            subscription_name=subscription_name,
            source_config=source_config,
            cookies_path=cookies_path,
            auth_token=auth_token,
        )
+        dl_result = attempt.dl_result
+        resolved_campaign_id = attempt.resolved_campaign_id
    except Exception as e:
        logger.exception(f"Unexpected error downloading {subscription_name}/{source_platform}: {e}")
        dl_error = e
@@ -287,6 +359,13 @@ async def _download_source_async(source_id: int, download_id: int = None) -> dic
                logger.error(f"Could not re-fetch download {actual_download_id} or source {source_id} for result update")
                return {"source_id": source_id, "status": "error", "error": "Lost DB records"}

+            # Persist resolved campaign ID (if the retry hook resolved one this run)
+            if resolved_campaign_id:
+                source.metadata_ = {
+                    **(source.metadata_ or {}),
+                    "patreon_campaign_id": resolved_campaign_id,
+                }
+
            if dl_error:
                download.status = DownloadStatus.FAILED
                download.error_type = DBErrorType.UNKNOWN_ERROR
@@ -0,0 +1,226 @@
+"""Integration tests for the Patreon campaign-ID retry hook in downloads.py.
+
+These tests exercise _execute_download_with_retry — the pure async helper
+that owns the download → resolver → retry logic — without standing up a
+full Celery + DB environment.
+"""
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+from app.tasks import downloads as downloads_module
+from app.tasks.downloads import _execute_download_with_retry
+
+
+def _make_gdl_service(results):
+    """Build a fake GalleryDLService whose .download() returns results in order."""
+    service = SimpleNamespace()
+    service.download = AsyncMock(side_effect=list(results))
+    return service
+
+
+def _success(url="https://www.patreon.com/id:12345"):
+    return SimpleNamespace(
+        success=True, stdout="", stderr="",
+        files_downloaded=3, url=url, error_type=None, error_message=None,
+    )
+
+
+def _campaign_id_failure(url="https://www.patreon.com/mstsu"):
+    return SimpleNamespace(
+        success=False,
+        stdout="",
+        stderr="[patreon][error] Failed to extract campaign ID",
+        files_downloaded=0, url=url, error_type=None,
+        error_message="Failed to extract campaign ID",
+    )
+
+
+def _other_failure(url="https://www.patreon.com/mstsu"):
+    return SimpleNamespace(
+        success=False, stdout="", stderr="HTTP 401 Unauthorized",
+        files_downloaded=0, url=url, error_type=None,
+        error_message="Authentication failed",
+    )
+
+
+async def test_happy_path_heal(monkeypatch):
+    """Patreon source fails with campaign-ID error, resolver returns ID, retry succeeds."""
+    gdl = _make_gdl_service([_campaign_id_failure(), _success()])
+    monkeypatch.setattr(
+        downloads_module,
+        "resolve_campaign_id",
+        AsyncMock(return_value="12345"),
+    )
+
+    result = await _execute_download_with_retry(
+        gdl_service=gdl,
+        platform="patreon",
+        source_url="https://www.patreon.com/mstsu",
+        source_metadata={},
+        subscription_name="MSTSU",
+        source_config=SimpleNamespace(),
+        cookies_path="/data/cookies/patreon_cookies.txt",
+        auth_token=None,
+    )
+
+    assert result.dl_result.success is True
+    assert result.resolved_campaign_id == "12345"
+    assert gdl.download.call_count == 2
+    first_call_url = gdl.download.call_args_list[0].kwargs["url"]
+    second_call_url = gdl.download.call_args_list[1].kwargs["url"]
+    assert first_call_url == "https://www.patreon.com/mstsu"
+    assert second_call_url == "https://www.patreon.com/id:12345"
+
+
+async def test_resolver_returns_none(monkeypatch):
+    """Campaign-ID failure + resolver returns None → no retry, no metadata, original failure surfaced."""
+    first_failure = _campaign_id_failure()
+    gdl = _make_gdl_service([first_failure])
+    monkeypatch.setattr(
+        downloads_module,
+        "resolve_campaign_id",
+        AsyncMock(return_value=None),
+    )
+
+    result = await _execute_download_with_retry(
+        gdl_service=gdl,
+        platform="patreon",
+        source_url="https://www.patreon.com/mstsu",
+        source_metadata={},
+        subscription_name="MSTSU",
+        source_config=SimpleNamespace(),
+        cookies_path="/data/cookies/patreon_cookies.txt",
+        auth_token=None,
+    )
+
+    assert result.dl_result is first_failure
+    assert result.resolved_campaign_id is None
+    assert gdl.download.call_count == 1
+
+
+async def test_already_cached_campaign_id(monkeypatch):
+    """Source has cached campaign_id → first download uses id-URL, no resolver call."""
+    gdl = _make_gdl_service([_success()])
+    resolver_mock = AsyncMock(return_value="should-not-be-called")
+    monkeypatch.setattr(downloads_module, "resolve_campaign_id", resolver_mock)
+
+    result = await _execute_download_with_retry(
+        gdl_service=gdl,
+        platform="patreon",
+        source_url="https://www.patreon.com/mstsu",
+        source_metadata={"patreon_campaign_id": "99999"},
+        subscription_name="MSTSU",
+        source_config=SimpleNamespace(),
+        cookies_path="/data/cookies/patreon_cookies.txt",
+        auth_token=None,
+    )
+
+    assert result.dl_result.success is True
+    assert result.resolved_campaign_id is None
+    assert gdl.download.call_count == 1
+    assert gdl.download.call_args.kwargs["url"] == "https://www.patreon.com/id:99999"
+    resolver_mock.assert_not_called()
+
+
+async def test_non_patreon_failure_skips_retry(monkeypatch):
+    """Non-Patreon source with similar-looking output → retry hook doesn't fire."""
+    failure = SimpleNamespace(
+        success=False,
+        stdout="",
+        stderr="some error that contains failed to extract campaign id text",
+        files_downloaded=0, url="https://subscribestar.adult/foo",
+        error_type=None, error_message="unknown",
+    )
+    gdl = _make_gdl_service([failure])
+    resolver_mock = AsyncMock(return_value="should-not-be-called")
+    monkeypatch.setattr(downloads_module, "resolve_campaign_id", resolver_mock)
+
+    result = await _execute_download_with_retry(
+        gdl_service=gdl,
+        platform="subscribestar",
+        source_url="https://subscribestar.adult/foo",
+        source_metadata={},
+        subscription_name="Foo",
+        source_config=SimpleNamespace(),
+        cookies_path=None,
+        auth_token=None,
+    )
+
+    assert result.dl_result is failure
+    assert result.resolved_campaign_id is None
+    resolver_mock.assert_not_called()
+    assert gdl.download.call_count == 1
+
+
+async def test_patreon_different_failure_skips_retry(monkeypatch):
+    """Patreon source with non-campaign-ID error → no resolver, no retry."""
+    failure = _other_failure()
+    gdl = _make_gdl_service([failure])
+    resolver_mock = AsyncMock(return_value="should-not-be-called")
+    monkeypatch.setattr(downloads_module, "resolve_campaign_id", resolver_mock)
+
+    result = await _execute_download_with_retry(
+        gdl_service=gdl,
+        platform="patreon",
+        source_url="https://www.patreon.com/mstsu",
+        source_metadata={},
+        subscription_name="MSTSU",
+        source_config=SimpleNamespace(),
+        cookies_path="/data/cookies/patreon_cookies.txt",
+        auth_token=None,
+    )
+
+    assert result.dl_result is failure
+    assert result.resolved_campaign_id is None
+    resolver_mock.assert_not_called()
+    assert gdl.download.call_count == 1
+
+
+async def test_unextractable_vanity_skips_retry(monkeypatch):
+    """Patreon source whose URL can't yield a vanity → no resolver call."""
+    failure = _campaign_id_failure(url="https://www.patreon.com/")
+    gdl = _make_gdl_service([failure])
+    resolver_mock = AsyncMock(return_value="should-not-be-called")
+    monkeypatch.setattr(downloads_module, "resolve_campaign_id", resolver_mock)
+
+    result = await _execute_download_with_retry(
+        gdl_service=gdl,
+        platform="patreon",
+        source_url="https://www.patreon.com/",
+        source_metadata={},
+        subscription_name="?",
+        source_config=SimpleNamespace(),
+        cookies_path=None,
+        auth_token=None,
+    )
+
+    assert result.dl_result is failure
+    resolver_mock.assert_not_called()
+    assert gdl.download.call_count == 1
+
+
+async def test_retry_itself_fails(monkeypatch):
+    """Resolver succeeds but retry fails → cache id, surface retry's failure."""
+    first_failure = _campaign_id_failure()
+    retry_failure = _other_failure()
+    gdl = _make_gdl_service([first_failure, retry_failure])
+    monkeypatch.setattr(
+        downloads_module, "resolve_campaign_id", AsyncMock(return_value="12345")
+    )
+
+    result = await _execute_download_with_retry(
+        gdl_service=gdl,
+        platform="patreon",
+        source_url="https://www.patreon.com/mstsu",
+        source_metadata={},
+        subscription_name="MSTSU",
+        source_config=SimpleNamespace(),
+        cookies_path="/data/cookies/patreon_cookies.txt",
+        auth_token=None,
+    )
+
+    assert result.dl_result is retry_failure  # retry's result wins
+    assert result.resolved_campaign_id == "12345"  # ID still cached
+    assert gdl.download.call_count == 2