From 35589e996a841718238da320531f2b8ca73049fb Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 18 Apr 2026 14:11:11 -0400 Subject: [PATCH] feat(downloads): auto-heal patreon campaign-ID failures via resolver retry --- backend/app/tasks/downloads.py | 85 ++++++- .../tasks/test_downloads_patreon_retry.py | 226 ++++++++++++++++++ 2 files changed, 308 insertions(+), 3 deletions(-) create mode 100644 backend/tests/tasks/test_downloads_patreon_retry.py diff --git a/backend/app/tasks/downloads.py b/backend/app/tasks/downloads.py index 720cb51..78acc22 100644 --- a/backend/app/tasks/downloads.py +++ b/backend/app/tasks/downloads.py @@ -3,6 +3,7 @@ import asyncio import logging import re +from dataclasses import dataclass from datetime import datetime, timedelta from typing import Optional @@ -19,6 +20,7 @@ from app.models.download import Download, DownloadStatus, ErrorType as DBErrorTy from app.models.setting import Setting, DEFAULT_SETTINGS from app.services.gallery_dl import GalleryDLService, SourceConfig, DownloadResult from app.services.credential_manager import CredentialManager +from app.services.patreon_resolver import resolve_campaign_id from app.events import publish_event logger = logging.getLogger(__name__) @@ -113,6 +115,71 @@ def _extract_vanity_from_url(url: str) -> Optional[str]: return m.group(1) if m else None +@dataclass +class _DownloadAttempt: + """Result of executing a download, possibly with a retry after campaign-ID resolution.""" + dl_result: DownloadResult + resolved_campaign_id: Optional[str] + + +async def _execute_download_with_retry( + *, + gdl_service, + platform: str, + source_url: str, + source_metadata: dict, + subscription_name: str, + source_config, + cookies_path: Optional[str], + auth_token: Optional[str], +) -> _DownloadAttempt: + """Run the download once, auto-heal Patreon campaign-ID failures with one retry. + + Returns a _DownloadAttempt wrapping: + - the final DownloadResult (retry's result if we retried, else the first) + - resolved_campaign_id: the newly-resolved ID (if we resolved one this call), else None + + The caller is responsible for persisting resolved_campaign_id into source.metadata_. + """ + effective_url = _effective_patreon_url(platform, source_url, source_metadata) + dl_result = await gdl_service.download( + url=effective_url, + subscription_name=subscription_name, + platform=platform, + source_config=source_config, + cookies_path=cookies_path, + auth_token=auth_token, + ) + + # Only retry if: patreon + campaign-ID failure + no cached id + extractable vanity + if ( + platform == "patreon" + and not dl_result.success + and "patreon_campaign_id" not in (source_metadata or {}) + and _looks_like_campaign_id_failure(dl_result) + ): + vanity = _extract_vanity_from_url(source_url) + if vanity: + logger.info( + f"Attempting campaign-ID resolution for {subscription_name}/patreon ({vanity})" + ) + campaign_id = await resolve_campaign_id(vanity, cookies_path) + if campaign_id: + retry_result = await gdl_service.download( + url=f"https://www.patreon.com/id:{campaign_id}", + subscription_name=subscription_name, + platform=platform, + source_config=source_config, + cookies_path=cookies_path, + auth_token=auth_token, + ) + return _DownloadAttempt( + dl_result=retry_result, resolved_campaign_id=campaign_id + ) + + return _DownloadAttempt(dl_result=dl_result, resolved_campaign_id=None) + + async def _download_source_async(source_id: int, download_id: int = None) -> dict: """Async implementation of source download. @@ -255,17 +322,22 @@ async def _download_source_async(source_id: int, download_id: int = None) -> dic source_config = SourceConfig.from_dict(source_metadata) dl_result = None dl_error = None + resolved_campaign_id: Optional[str] = None try: gdl_service = GalleryDLService(rate_limit=db_rate_limit) - dl_result = await gdl_service.download( - url=source_url, - subscription_name=subscription_name, + attempt = await _execute_download_with_retry( + gdl_service=gdl_service, platform=source_platform, + source_url=source_url, + source_metadata=source_metadata, + subscription_name=subscription_name, source_config=source_config, cookies_path=cookies_path, auth_token=auth_token, ) + dl_result = attempt.dl_result + resolved_campaign_id = attempt.resolved_campaign_id except Exception as e: logger.exception(f"Unexpected error downloading {subscription_name}/{source_platform}: {e}") dl_error = e @@ -287,6 +359,13 @@ async def _download_source_async(source_id: int, download_id: int = None) -> dic logger.error(f"Could not re-fetch download {actual_download_id} or source {source_id} for result update") return {"source_id": source_id, "status": "error", "error": "Lost DB records"} + # Persist resolved campaign ID (if the retry hook resolved one this run) + if resolved_campaign_id: + source.metadata_ = { + **(source.metadata_ or {}), + "patreon_campaign_id": resolved_campaign_id, + } + if dl_error: download.status = DownloadStatus.FAILED download.error_type = DBErrorType.UNKNOWN_ERROR diff --git a/backend/tests/tasks/test_downloads_patreon_retry.py b/backend/tests/tasks/test_downloads_patreon_retry.py new file mode 100644 index 0000000..361724d --- /dev/null +++ b/backend/tests/tasks/test_downloads_patreon_retry.py @@ -0,0 +1,226 @@ +"""Integration tests for the Patreon campaign-ID retry hook in downloads.py. + +These tests exercise _execute_download_with_retry — the pure async helper +that owns the download → resolver → retry logic — without standing up a +full Celery + DB environment. +""" +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from app.tasks import downloads as downloads_module +from app.tasks.downloads import _execute_download_with_retry + + +def _make_gdl_service(results): + """Build a fake GalleryDLService whose .download() returns results in order.""" + service = SimpleNamespace() + service.download = AsyncMock(side_effect=list(results)) + return service + + +def _success(url="https://www.patreon.com/id:12345"): + return SimpleNamespace( + success=True, stdout="", stderr="", + files_downloaded=3, url=url, error_type=None, error_message=None, + ) + + +def _campaign_id_failure(url="https://www.patreon.com/mstsu"): + return SimpleNamespace( + success=False, + stdout="", + stderr="[patreon][error] Failed to extract campaign ID", + files_downloaded=0, url=url, error_type=None, + error_message="Failed to extract campaign ID", + ) + + +def _other_failure(url="https://www.patreon.com/mstsu"): + return SimpleNamespace( + success=False, stdout="", stderr="HTTP 401 Unauthorized", + files_downloaded=0, url=url, error_type=None, + error_message="Authentication failed", + ) + + +async def test_happy_path_heal(monkeypatch): + """Patreon source fails with campaign-ID error, resolver returns ID, retry succeeds.""" + gdl = _make_gdl_service([_campaign_id_failure(), _success()]) + monkeypatch.setattr( + downloads_module, + "resolve_campaign_id", + AsyncMock(return_value="12345"), + ) + + result = await _execute_download_with_retry( + gdl_service=gdl, + platform="patreon", + source_url="https://www.patreon.com/mstsu", + source_metadata={}, + subscription_name="MSTSU", + source_config=SimpleNamespace(), + cookies_path="/data/cookies/patreon_cookies.txt", + auth_token=None, + ) + + assert result.dl_result.success is True + assert result.resolved_campaign_id == "12345" + assert gdl.download.call_count == 2 + first_call_url = gdl.download.call_args_list[0].kwargs["url"] + second_call_url = gdl.download.call_args_list[1].kwargs["url"] + assert first_call_url == "https://www.patreon.com/mstsu" + assert second_call_url == "https://www.patreon.com/id:12345" + + +async def test_resolver_returns_none(monkeypatch): + """Campaign-ID failure + resolver returns None → no retry, no metadata, original failure surfaced.""" + first_failure = _campaign_id_failure() + gdl = _make_gdl_service([first_failure]) + monkeypatch.setattr( + downloads_module, + "resolve_campaign_id", + AsyncMock(return_value=None), + ) + + result = await _execute_download_with_retry( + gdl_service=gdl, + platform="patreon", + source_url="https://www.patreon.com/mstsu", + source_metadata={}, + subscription_name="MSTSU", + source_config=SimpleNamespace(), + cookies_path="/data/cookies/patreon_cookies.txt", + auth_token=None, + ) + + assert result.dl_result is first_failure + assert result.resolved_campaign_id is None + assert gdl.download.call_count == 1 + + +async def test_already_cached_campaign_id(monkeypatch): + """Source has cached campaign_id → first download uses id-URL, no resolver call.""" + gdl = _make_gdl_service([_success()]) + resolver_mock = AsyncMock(return_value="should-not-be-called") + monkeypatch.setattr(downloads_module, "resolve_campaign_id", resolver_mock) + + result = await _execute_download_with_retry( + gdl_service=gdl, + platform="patreon", + source_url="https://www.patreon.com/mstsu", + source_metadata={"patreon_campaign_id": "99999"}, + subscription_name="MSTSU", + source_config=SimpleNamespace(), + cookies_path="/data/cookies/patreon_cookies.txt", + auth_token=None, + ) + + assert result.dl_result.success is True + assert result.resolved_campaign_id is None + assert gdl.download.call_count == 1 + assert gdl.download.call_args.kwargs["url"] == "https://www.patreon.com/id:99999" + resolver_mock.assert_not_called() + + +async def test_non_patreon_failure_skips_retry(monkeypatch): + """Non-Patreon source with similar-looking output → retry hook doesn't fire.""" + failure = SimpleNamespace( + success=False, + stdout="", + stderr="some error that contains failed to extract campaign id text", + files_downloaded=0, url="https://subscribestar.adult/foo", + error_type=None, error_message="unknown", + ) + gdl = _make_gdl_service([failure]) + resolver_mock = AsyncMock(return_value="should-not-be-called") + monkeypatch.setattr(downloads_module, "resolve_campaign_id", resolver_mock) + + result = await _execute_download_with_retry( + gdl_service=gdl, + platform="subscribestar", + source_url="https://subscribestar.adult/foo", + source_metadata={}, + subscription_name="Foo", + source_config=SimpleNamespace(), + cookies_path=None, + auth_token=None, + ) + + assert result.dl_result is failure + assert result.resolved_campaign_id is None + resolver_mock.assert_not_called() + assert gdl.download.call_count == 1 + + +async def test_patreon_different_failure_skips_retry(monkeypatch): + """Patreon source with non-campaign-ID error → no resolver, no retry.""" + failure = _other_failure() + gdl = _make_gdl_service([failure]) + resolver_mock = AsyncMock(return_value="should-not-be-called") + monkeypatch.setattr(downloads_module, "resolve_campaign_id", resolver_mock) + + result = await _execute_download_with_retry( + gdl_service=gdl, + platform="patreon", + source_url="https://www.patreon.com/mstsu", + source_metadata={}, + subscription_name="MSTSU", + source_config=SimpleNamespace(), + cookies_path="/data/cookies/patreon_cookies.txt", + auth_token=None, + ) + + assert result.dl_result is failure + assert result.resolved_campaign_id is None + resolver_mock.assert_not_called() + assert gdl.download.call_count == 1 + + +async def test_unextractable_vanity_skips_retry(monkeypatch): + """Patreon source whose URL can't yield a vanity → no resolver call.""" + failure = _campaign_id_failure(url="https://www.patreon.com/") + gdl = _make_gdl_service([failure]) + resolver_mock = AsyncMock(return_value="should-not-be-called") + monkeypatch.setattr(downloads_module, "resolve_campaign_id", resolver_mock) + + result = await _execute_download_with_retry( + gdl_service=gdl, + platform="patreon", + source_url="https://www.patreon.com/", + source_metadata={}, + subscription_name="?", + source_config=SimpleNamespace(), + cookies_path=None, + auth_token=None, + ) + + assert result.dl_result is failure + resolver_mock.assert_not_called() + assert gdl.download.call_count == 1 + + +async def test_retry_itself_fails(monkeypatch): + """Resolver succeeds but retry fails → cache id, surface retry's failure.""" + first_failure = _campaign_id_failure() + retry_failure = _other_failure() + gdl = _make_gdl_service([first_failure, retry_failure]) + monkeypatch.setattr( + downloads_module, "resolve_campaign_id", AsyncMock(return_value="12345") + ) + + result = await _execute_download_with_retry( + gdl_service=gdl, + platform="patreon", + source_url="https://www.patreon.com/mstsu", + source_metadata={}, + subscription_name="MSTSU", + source_config=SimpleNamespace(), + cookies_path="/data/cookies/patreon_cookies.txt", + auth_token=None, + ) + + assert result.dl_result is retry_failure # retry's result wins + assert result.resolved_campaign_id == "12345" # ID still cached + assert gdl.download.call_count == 2