feat(patreon): ingester rate-limit resilience — #703 step 1
A single 429 mid-walk used to fail the run → RATE_LIMITED → platform-wide
cooldown → every Patreon source dark ("testing dead"). The native path also
ignored the operator's existing politeness setting. Fixed both:
- Pacing (avoid 429s): honor download_rate_limit_seconds (gallery-dl's
`rate_limit`, read off self.gdl) as a pre-download sleep on real media
downloads only (skips don't pace); pace /api/posts page fetches with the
per-source sleep_request override, defaulting to max(0.5, rate_limit/4) —
the same API-pacing default gallery-dl used for `sleep-request`.
- 429 backoff (ride out transient limits): PatreonClient._fetch retries a
429 with backoff (honor Retry-After, else exponential 2·2^(n-1), capped
30s, ≤3 tries); only a PERSISTENT 429 propagates as terminal
RATE_LIMITED. Light 2-retry on a media-GET 429 too.
Threaded via PatreonIngester(rate_limit=, request_sleep=) →
PatreonClient/PatreonDownloader; download_service sources them. Injected
test client/downloader are unaffected (carry their own pacing).
Tests mock time.sleep (no real sleeping): retry-then-success, persistent
429 raises after N, Retry-After honored, request_sleep paces, media pacing
per real download, skips don't pace, media 429 retried.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -33,6 +33,7 @@ import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
@@ -42,13 +43,16 @@ from urllib.parse import urlsplit
|
||||
import requests
|
||||
|
||||
from .file_validator import is_validatable, validate_file
|
||||
from .patreon_client import _load_session
|
||||
from .patreon_client import _load_session, _retry_after_seconds
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_TITLE_MAX = 40
|
||||
_TIMEOUT_SECONDS = 120.0
|
||||
_CHUNK = 1 << 16
|
||||
# A CDN media GET rarely 429s, but if it does, a couple of backoff retries keep
|
||||
# one throttled file from becoming a per-item error (plan #703).
|
||||
_MAX_MEDIA_429_RETRIES = 2
|
||||
|
||||
# Referer/Origin yt-dlp must send for Mux-hosted Patreon video. Mux's JWT
|
||||
# playback policy checks Referer/Origin on every request, so yt-dlp must send
|
||||
@@ -148,11 +152,17 @@ class PatreonDownloader:
|
||||
cookies_path: str | None = None,
|
||||
*,
|
||||
validate: bool = True,
|
||||
rate_limit: float = 0.0,
|
||||
session: requests.Session | None = None,
|
||||
):
|
||||
self.images_root = Path(images_root)
|
||||
self.cookies_path = str(cookies_path) if cookies_path else None
|
||||
self._validate = validate
|
||||
# Politeness: seconds to sleep before each actual media download (paces
|
||||
# the CDN; honors ImportSettings.download_rate_limit_seconds, the same
|
||||
# value gallery-dl used as its between-downloads `sleep`). 0 = no pacing.
|
||||
# Applied only to real downloads, not to seen/disk skips. plan #703.
|
||||
self._rate_limit = rate_limit or 0.0
|
||||
# Build a cookie-loaded session the same way patreon_client does, so the
|
||||
# CDN GETs carry the creator's auth.
|
||||
self.session = session if session is not None else _load_session(cookies_path)
|
||||
@@ -227,6 +237,10 @@ class PatreonDownloader:
|
||||
|
||||
post_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Pace real downloads only (the skips above already returned). plan #703.
|
||||
if self._rate_limit > 0:
|
||||
time.sleep(self._rate_limit)
|
||||
|
||||
if _is_video_url(media.url):
|
||||
out_path = self._run_ytdlp(media.url, media_path, _VIDEO_HEADERS)
|
||||
if out_path is None or not Path(out_path).exists():
|
||||
@@ -267,8 +281,25 @@ class PatreonDownloader:
|
||||
return dest
|
||||
|
||||
def _fetch_to_file(self, url: str, dest: Path) -> None:
|
||||
"""Stream a non-video URL to `dest` via the (stubbable) session."""
|
||||
resp = self.session.get(url, stream=True, timeout=_TIMEOUT_SECONDS)
|
||||
"""Stream a non-video URL to `dest` via the (stubbable) session.
|
||||
|
||||
Retries a transient 429 with backoff (plan #703); any other non-2xx
|
||||
falls through to raise_for_status → HTTPError, which download_post turns
|
||||
into a resilient per-item error outcome.
|
||||
"""
|
||||
attempt = 0
|
||||
while True:
|
||||
resp = self.session.get(url, stream=True, timeout=_TIMEOUT_SECONDS)
|
||||
if resp.status_code == 429 and attempt < _MAX_MEDIA_429_RETRIES:
|
||||
attempt += 1
|
||||
delay = _retry_after_seconds(resp, attempt)
|
||||
log.warning(
|
||||
"Patreon media 429 (%s) — backing off %.1fs (retry %d/%d)",
|
||||
url, delay, attempt, _MAX_MEDIA_429_RETRIES,
|
||||
)
|
||||
time.sleep(delay)
|
||||
continue
|
||||
break
|
||||
resp.raise_for_status()
|
||||
with open(dest, "wb") as fh:
|
||||
for chunk in resp.iter_content(chunk_size=_CHUNK):
|
||||
|
||||
Reference in New Issue
Block a user