feat(downloads): auto-heal patreon campaign-ID failures via resolver retry
This commit is contained in:
@@ -3,6 +3,7 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Optional
|
||||
|
||||
@@ -19,6 +20,7 @@ from app.models.download import Download, DownloadStatus, ErrorType as DBErrorTy
|
||||
from app.models.setting import Setting, DEFAULT_SETTINGS
|
||||
from app.services.gallery_dl import GalleryDLService, SourceConfig, DownloadResult
|
||||
from app.services.credential_manager import CredentialManager
|
||||
from app.services.patreon_resolver import resolve_campaign_id
|
||||
from app.events import publish_event
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -113,6 +115,71 @@ def _extract_vanity_from_url(url: str) -> Optional[str]:
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
@dataclass
|
||||
class _DownloadAttempt:
|
||||
"""Result of executing a download, possibly with a retry after campaign-ID resolution."""
|
||||
dl_result: DownloadResult
|
||||
resolved_campaign_id: Optional[str]
|
||||
|
||||
|
||||
async def _execute_download_with_retry(
|
||||
*,
|
||||
gdl_service,
|
||||
platform: str,
|
||||
source_url: str,
|
||||
source_metadata: dict,
|
||||
subscription_name: str,
|
||||
source_config,
|
||||
cookies_path: Optional[str],
|
||||
auth_token: Optional[str],
|
||||
) -> _DownloadAttempt:
|
||||
"""Run the download once, auto-heal Patreon campaign-ID failures with one retry.
|
||||
|
||||
Returns a _DownloadAttempt wrapping:
|
||||
- the final DownloadResult (retry's result if we retried, else the first)
|
||||
- resolved_campaign_id: the newly-resolved ID (if we resolved one this call), else None
|
||||
|
||||
The caller is responsible for persisting resolved_campaign_id into source.metadata_.
|
||||
"""
|
||||
effective_url = _effective_patreon_url(platform, source_url, source_metadata)
|
||||
dl_result = await gdl_service.download(
|
||||
url=effective_url,
|
||||
subscription_name=subscription_name,
|
||||
platform=platform,
|
||||
source_config=source_config,
|
||||
cookies_path=cookies_path,
|
||||
auth_token=auth_token,
|
||||
)
|
||||
|
||||
# Only retry if: patreon + campaign-ID failure + no cached id + extractable vanity
|
||||
if (
|
||||
platform == "patreon"
|
||||
and not dl_result.success
|
||||
and "patreon_campaign_id" not in (source_metadata or {})
|
||||
and _looks_like_campaign_id_failure(dl_result)
|
||||
):
|
||||
vanity = _extract_vanity_from_url(source_url)
|
||||
if vanity:
|
||||
logger.info(
|
||||
f"Attempting campaign-ID resolution for {subscription_name}/patreon ({vanity})"
|
||||
)
|
||||
campaign_id = await resolve_campaign_id(vanity, cookies_path)
|
||||
if campaign_id:
|
||||
retry_result = await gdl_service.download(
|
||||
url=f"https://www.patreon.com/id:{campaign_id}",
|
||||
subscription_name=subscription_name,
|
||||
platform=platform,
|
||||
source_config=source_config,
|
||||
cookies_path=cookies_path,
|
||||
auth_token=auth_token,
|
||||
)
|
||||
return _DownloadAttempt(
|
||||
dl_result=retry_result, resolved_campaign_id=campaign_id
|
||||
)
|
||||
|
||||
return _DownloadAttempt(dl_result=dl_result, resolved_campaign_id=None)
|
||||
|
||||
|
||||
async def _download_source_async(source_id: int, download_id: int = None) -> dict:
|
||||
"""Async implementation of source download.
|
||||
|
||||
@@ -255,17 +322,22 @@ async def _download_source_async(source_id: int, download_id: int = None) -> dic
|
||||
source_config = SourceConfig.from_dict(source_metadata)
|
||||
dl_result = None
|
||||
dl_error = None
|
||||
resolved_campaign_id: Optional[str] = None
|
||||
|
||||
try:
|
||||
gdl_service = GalleryDLService(rate_limit=db_rate_limit)
|
||||
dl_result = await gdl_service.download(
|
||||
url=source_url,
|
||||
subscription_name=subscription_name,
|
||||
attempt = await _execute_download_with_retry(
|
||||
gdl_service=gdl_service,
|
||||
platform=source_platform,
|
||||
source_url=source_url,
|
||||
source_metadata=source_metadata,
|
||||
subscription_name=subscription_name,
|
||||
source_config=source_config,
|
||||
cookies_path=cookies_path,
|
||||
auth_token=auth_token,
|
||||
)
|
||||
dl_result = attempt.dl_result
|
||||
resolved_campaign_id = attempt.resolved_campaign_id
|
||||
except Exception as e:
|
||||
logger.exception(f"Unexpected error downloading {subscription_name}/{source_platform}: {e}")
|
||||
dl_error = e
|
||||
@@ -287,6 +359,13 @@ async def _download_source_async(source_id: int, download_id: int = None) -> dic
|
||||
logger.error(f"Could not re-fetch download {actual_download_id} or source {source_id} for result update")
|
||||
return {"source_id": source_id, "status": "error", "error": "Lost DB records"}
|
||||
|
||||
# Persist resolved campaign ID (if the retry hook resolved one this run)
|
||||
if resolved_campaign_id:
|
||||
source.metadata_ = {
|
||||
**(source.metadata_ or {}),
|
||||
"patreon_campaign_id": resolved_campaign_id,
|
||||
}
|
||||
|
||||
if dl_error:
|
||||
download.status = DownloadStatus.FAILED
|
||||
download.error_type = DBErrorType.UNKNOWN_ERROR
|
||||
|
||||
Reference in New Issue
Block a user