feat(downloads): auto-heal patreon campaign-ID failures via resolver retry
This commit is contained in:
@@ -3,6 +3,7 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Optional
|
||||
|
||||
@@ -19,6 +20,7 @@ from app.models.download import Download, DownloadStatus, ErrorType as DBErrorTy
|
||||
from app.models.setting import Setting, DEFAULT_SETTINGS
|
||||
from app.services.gallery_dl import GalleryDLService, SourceConfig, DownloadResult
|
||||
from app.services.credential_manager import CredentialManager
|
||||
from app.services.patreon_resolver import resolve_campaign_id
|
||||
from app.events import publish_event
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -113,6 +115,71 @@ def _extract_vanity_from_url(url: str) -> Optional[str]:
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
@dataclass
|
||||
class _DownloadAttempt:
|
||||
"""Result of executing a download, possibly with a retry after campaign-ID resolution."""
|
||||
dl_result: DownloadResult
|
||||
resolved_campaign_id: Optional[str]
|
||||
|
||||
|
||||
async def _execute_download_with_retry(
|
||||
*,
|
||||
gdl_service,
|
||||
platform: str,
|
||||
source_url: str,
|
||||
source_metadata: dict,
|
||||
subscription_name: str,
|
||||
source_config,
|
||||
cookies_path: Optional[str],
|
||||
auth_token: Optional[str],
|
||||
) -> _DownloadAttempt:
|
||||
"""Run the download once, auto-heal Patreon campaign-ID failures with one retry.
|
||||
|
||||
Returns a _DownloadAttempt wrapping:
|
||||
- the final DownloadResult (retry's result if we retried, else the first)
|
||||
- resolved_campaign_id: the newly-resolved ID (if we resolved one this call), else None
|
||||
|
||||
The caller is responsible for persisting resolved_campaign_id into source.metadata_.
|
||||
"""
|
||||
effective_url = _effective_patreon_url(platform, source_url, source_metadata)
|
||||
dl_result = await gdl_service.download(
|
||||
url=effective_url,
|
||||
subscription_name=subscription_name,
|
||||
platform=platform,
|
||||
source_config=source_config,
|
||||
cookies_path=cookies_path,
|
||||
auth_token=auth_token,
|
||||
)
|
||||
|
||||
# Only retry if: patreon + campaign-ID failure + no cached id + extractable vanity
|
||||
if (
|
||||
platform == "patreon"
|
||||
and not dl_result.success
|
||||
and "patreon_campaign_id" not in (source_metadata or {})
|
||||
and _looks_like_campaign_id_failure(dl_result)
|
||||
):
|
||||
vanity = _extract_vanity_from_url(source_url)
|
||||
if vanity:
|
||||
logger.info(
|
||||
f"Attempting campaign-ID resolution for {subscription_name}/patreon ({vanity})"
|
||||
)
|
||||
campaign_id = await resolve_campaign_id(vanity, cookies_path)
|
||||
if campaign_id:
|
||||
retry_result = await gdl_service.download(
|
||||
url=f"https://www.patreon.com/id:{campaign_id}",
|
||||
subscription_name=subscription_name,
|
||||
platform=platform,
|
||||
source_config=source_config,
|
||||
cookies_path=cookies_path,
|
||||
auth_token=auth_token,
|
||||
)
|
||||
return _DownloadAttempt(
|
||||
dl_result=retry_result, resolved_campaign_id=campaign_id
|
||||
)
|
||||
|
||||
return _DownloadAttempt(dl_result=dl_result, resolved_campaign_id=None)
|
||||
|
||||
|
||||
async def _download_source_async(source_id: int, download_id: int = None) -> dict:
|
||||
"""Async implementation of source download.
|
||||
|
||||
@@ -255,17 +322,22 @@ async def _download_source_async(source_id: int, download_id: int = None) -> dic
|
||||
source_config = SourceConfig.from_dict(source_metadata)
|
||||
dl_result = None
|
||||
dl_error = None
|
||||
resolved_campaign_id: Optional[str] = None
|
||||
|
||||
try:
|
||||
gdl_service = GalleryDLService(rate_limit=db_rate_limit)
|
||||
dl_result = await gdl_service.download(
|
||||
url=source_url,
|
||||
subscription_name=subscription_name,
|
||||
attempt = await _execute_download_with_retry(
|
||||
gdl_service=gdl_service,
|
||||
platform=source_platform,
|
||||
source_url=source_url,
|
||||
source_metadata=source_metadata,
|
||||
subscription_name=subscription_name,
|
||||
source_config=source_config,
|
||||
cookies_path=cookies_path,
|
||||
auth_token=auth_token,
|
||||
)
|
||||
dl_result = attempt.dl_result
|
||||
resolved_campaign_id = attempt.resolved_campaign_id
|
||||
except Exception as e:
|
||||
logger.exception(f"Unexpected error downloading {subscription_name}/{source_platform}: {e}")
|
||||
dl_error = e
|
||||
@@ -287,6 +359,13 @@ async def _download_source_async(source_id: int, download_id: int = None) -> dic
|
||||
logger.error(f"Could not re-fetch download {actual_download_id} or source {source_id} for result update")
|
||||
return {"source_id": source_id, "status": "error", "error": "Lost DB records"}
|
||||
|
||||
# Persist resolved campaign ID (if the retry hook resolved one this run)
|
||||
if resolved_campaign_id:
|
||||
source.metadata_ = {
|
||||
**(source.metadata_ or {}),
|
||||
"patreon_campaign_id": resolved_campaign_id,
|
||||
}
|
||||
|
||||
if dl_error:
|
||||
download.status = DownloadStatus.FAILED
|
||||
download.error_type = DBErrorType.UNKNOWN_ERROR
|
||||
|
||||
@@ -0,0 +1,226 @@
|
||||
"""Integration tests for the Patreon campaign-ID retry hook in downloads.py.
|
||||
|
||||
These tests exercise _execute_download_with_retry — the pure async helper
|
||||
that owns the download → resolver → retry logic — without standing up a
|
||||
full Celery + DB environment.
|
||||
"""
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
from app.tasks import downloads as downloads_module
|
||||
from app.tasks.downloads import _execute_download_with_retry
|
||||
|
||||
|
||||
def _make_gdl_service(results):
|
||||
"""Build a fake GalleryDLService whose .download() returns results in order."""
|
||||
service = SimpleNamespace()
|
||||
service.download = AsyncMock(side_effect=list(results))
|
||||
return service
|
||||
|
||||
|
||||
def _success(url="https://www.patreon.com/id:12345"):
|
||||
return SimpleNamespace(
|
||||
success=True, stdout="", stderr="",
|
||||
files_downloaded=3, url=url, error_type=None, error_message=None,
|
||||
)
|
||||
|
||||
|
||||
def _campaign_id_failure(url="https://www.patreon.com/mstsu"):
|
||||
return SimpleNamespace(
|
||||
success=False,
|
||||
stdout="",
|
||||
stderr="[patreon][error] Failed to extract campaign ID",
|
||||
files_downloaded=0, url=url, error_type=None,
|
||||
error_message="Failed to extract campaign ID",
|
||||
)
|
||||
|
||||
|
||||
def _other_failure(url="https://www.patreon.com/mstsu"):
|
||||
return SimpleNamespace(
|
||||
success=False, stdout="", stderr="HTTP 401 Unauthorized",
|
||||
files_downloaded=0, url=url, error_type=None,
|
||||
error_message="Authentication failed",
|
||||
)
|
||||
|
||||
|
||||
async def test_happy_path_heal(monkeypatch):
|
||||
"""Patreon source fails with campaign-ID error, resolver returns ID, retry succeeds."""
|
||||
gdl = _make_gdl_service([_campaign_id_failure(), _success()])
|
||||
monkeypatch.setattr(
|
||||
downloads_module,
|
||||
"resolve_campaign_id",
|
||||
AsyncMock(return_value="12345"),
|
||||
)
|
||||
|
||||
result = await _execute_download_with_retry(
|
||||
gdl_service=gdl,
|
||||
platform="patreon",
|
||||
source_url="https://www.patreon.com/mstsu",
|
||||
source_metadata={},
|
||||
subscription_name="MSTSU",
|
||||
source_config=SimpleNamespace(),
|
||||
cookies_path="/data/cookies/patreon_cookies.txt",
|
||||
auth_token=None,
|
||||
)
|
||||
|
||||
assert result.dl_result.success is True
|
||||
assert result.resolved_campaign_id == "12345"
|
||||
assert gdl.download.call_count == 2
|
||||
first_call_url = gdl.download.call_args_list[0].kwargs["url"]
|
||||
second_call_url = gdl.download.call_args_list[1].kwargs["url"]
|
||||
assert first_call_url == "https://www.patreon.com/mstsu"
|
||||
assert second_call_url == "https://www.patreon.com/id:12345"
|
||||
|
||||
|
||||
async def test_resolver_returns_none(monkeypatch):
|
||||
"""Campaign-ID failure + resolver returns None → no retry, no metadata, original failure surfaced."""
|
||||
first_failure = _campaign_id_failure()
|
||||
gdl = _make_gdl_service([first_failure])
|
||||
monkeypatch.setattr(
|
||||
downloads_module,
|
||||
"resolve_campaign_id",
|
||||
AsyncMock(return_value=None),
|
||||
)
|
||||
|
||||
result = await _execute_download_with_retry(
|
||||
gdl_service=gdl,
|
||||
platform="patreon",
|
||||
source_url="https://www.patreon.com/mstsu",
|
||||
source_metadata={},
|
||||
subscription_name="MSTSU",
|
||||
source_config=SimpleNamespace(),
|
||||
cookies_path="/data/cookies/patreon_cookies.txt",
|
||||
auth_token=None,
|
||||
)
|
||||
|
||||
assert result.dl_result is first_failure
|
||||
assert result.resolved_campaign_id is None
|
||||
assert gdl.download.call_count == 1
|
||||
|
||||
|
||||
async def test_already_cached_campaign_id(monkeypatch):
|
||||
"""Source has cached campaign_id → first download uses id-URL, no resolver call."""
|
||||
gdl = _make_gdl_service([_success()])
|
||||
resolver_mock = AsyncMock(return_value="should-not-be-called")
|
||||
monkeypatch.setattr(downloads_module, "resolve_campaign_id", resolver_mock)
|
||||
|
||||
result = await _execute_download_with_retry(
|
||||
gdl_service=gdl,
|
||||
platform="patreon",
|
||||
source_url="https://www.patreon.com/mstsu",
|
||||
source_metadata={"patreon_campaign_id": "99999"},
|
||||
subscription_name="MSTSU",
|
||||
source_config=SimpleNamespace(),
|
||||
cookies_path="/data/cookies/patreon_cookies.txt",
|
||||
auth_token=None,
|
||||
)
|
||||
|
||||
assert result.dl_result.success is True
|
||||
assert result.resolved_campaign_id is None
|
||||
assert gdl.download.call_count == 1
|
||||
assert gdl.download.call_args.kwargs["url"] == "https://www.patreon.com/id:99999"
|
||||
resolver_mock.assert_not_called()
|
||||
|
||||
|
||||
async def test_non_patreon_failure_skips_retry(monkeypatch):
|
||||
"""Non-Patreon source with similar-looking output → retry hook doesn't fire."""
|
||||
failure = SimpleNamespace(
|
||||
success=False,
|
||||
stdout="",
|
||||
stderr="some error that contains failed to extract campaign id text",
|
||||
files_downloaded=0, url="https://subscribestar.adult/foo",
|
||||
error_type=None, error_message="unknown",
|
||||
)
|
||||
gdl = _make_gdl_service([failure])
|
||||
resolver_mock = AsyncMock(return_value="should-not-be-called")
|
||||
monkeypatch.setattr(downloads_module, "resolve_campaign_id", resolver_mock)
|
||||
|
||||
result = await _execute_download_with_retry(
|
||||
gdl_service=gdl,
|
||||
platform="subscribestar",
|
||||
source_url="https://subscribestar.adult/foo",
|
||||
source_metadata={},
|
||||
subscription_name="Foo",
|
||||
source_config=SimpleNamespace(),
|
||||
cookies_path=None,
|
||||
auth_token=None,
|
||||
)
|
||||
|
||||
assert result.dl_result is failure
|
||||
assert result.resolved_campaign_id is None
|
||||
resolver_mock.assert_not_called()
|
||||
assert gdl.download.call_count == 1
|
||||
|
||||
|
||||
async def test_patreon_different_failure_skips_retry(monkeypatch):
|
||||
"""Patreon source with non-campaign-ID error → no resolver, no retry."""
|
||||
failure = _other_failure()
|
||||
gdl = _make_gdl_service([failure])
|
||||
resolver_mock = AsyncMock(return_value="should-not-be-called")
|
||||
monkeypatch.setattr(downloads_module, "resolve_campaign_id", resolver_mock)
|
||||
|
||||
result = await _execute_download_with_retry(
|
||||
gdl_service=gdl,
|
||||
platform="patreon",
|
||||
source_url="https://www.patreon.com/mstsu",
|
||||
source_metadata={},
|
||||
subscription_name="MSTSU",
|
||||
source_config=SimpleNamespace(),
|
||||
cookies_path="/data/cookies/patreon_cookies.txt",
|
||||
auth_token=None,
|
||||
)
|
||||
|
||||
assert result.dl_result is failure
|
||||
assert result.resolved_campaign_id is None
|
||||
resolver_mock.assert_not_called()
|
||||
assert gdl.download.call_count == 1
|
||||
|
||||
|
||||
async def test_unextractable_vanity_skips_retry(monkeypatch):
|
||||
"""Patreon source whose URL can't yield a vanity → no resolver call."""
|
||||
failure = _campaign_id_failure(url="https://www.patreon.com/")
|
||||
gdl = _make_gdl_service([failure])
|
||||
resolver_mock = AsyncMock(return_value="should-not-be-called")
|
||||
monkeypatch.setattr(downloads_module, "resolve_campaign_id", resolver_mock)
|
||||
|
||||
result = await _execute_download_with_retry(
|
||||
gdl_service=gdl,
|
||||
platform="patreon",
|
||||
source_url="https://www.patreon.com/",
|
||||
source_metadata={},
|
||||
subscription_name="?",
|
||||
source_config=SimpleNamespace(),
|
||||
cookies_path=None,
|
||||
auth_token=None,
|
||||
)
|
||||
|
||||
assert result.dl_result is failure
|
||||
resolver_mock.assert_not_called()
|
||||
assert gdl.download.call_count == 1
|
||||
|
||||
|
||||
async def test_retry_itself_fails(monkeypatch):
|
||||
"""Resolver succeeds but retry fails → cache id, surface retry's failure."""
|
||||
first_failure = _campaign_id_failure()
|
||||
retry_failure = _other_failure()
|
||||
gdl = _make_gdl_service([first_failure, retry_failure])
|
||||
monkeypatch.setattr(
|
||||
downloads_module, "resolve_campaign_id", AsyncMock(return_value="12345")
|
||||
)
|
||||
|
||||
result = await _execute_download_with_retry(
|
||||
gdl_service=gdl,
|
||||
platform="patreon",
|
||||
source_url="https://www.patreon.com/mstsu",
|
||||
source_metadata={},
|
||||
subscription_name="MSTSU",
|
||||
source_config=SimpleNamespace(),
|
||||
cookies_path="/data/cookies/patreon_cookies.txt",
|
||||
auth_token=None,
|
||||
)
|
||||
|
||||
assert result.dl_result is retry_failure # retry's result wins
|
||||
assert result.resolved_campaign_id == "12345" # ID still cached
|
||||
assert gdl.download.call_count == 2
|
||||
Reference in New Issue
Block a user