feat(patreon): ingester rate-limit resilience — #703 step 1
A single 429 mid-walk used to fail the run → RATE_LIMITED → platform-wide
cooldown → every Patreon source dark ("testing dead"). The native path also
ignored the operator's existing politeness setting. Fixed both:
- Pacing (avoid 429s): honor download_rate_limit_seconds (gallery-dl's
`rate_limit`, read off self.gdl) as a pre-download sleep on real media
downloads only (skips don't pace); pace /api/posts page fetches with the
per-source sleep_request override, defaulting to max(0.5, rate_limit/4) —
the same API-pacing default gallery-dl used for `sleep-request`.
- 429 backoff (ride out transient limits): PatreonClient._fetch retries a
429 with backoff (honor Retry-After, else exponential 2·2^(n-1), capped
30s, ≤3 tries); only a PERSISTENT 429 propagates as terminal
RATE_LIMITED. Light 2-retry on a media-GET 429 too.
Threaded via PatreonIngester(rate_limit=, request_sleep=) →
PatreonClient/PatreonDownloader; download_service sources them. Injected
test client/downloader are unaffected (carry their own pacing).
Tests mock time.sleep (no real sleeping): retry-then-success, persistent
429 raises after N, Retry-After honored, request_sleep paces, media pacing
per real download, skips don't pace, media 429 retried.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -10,7 +10,9 @@ from __future__ import annotations
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from backend.app.services import patreon_downloader as pd_mod
|
||||
from backend.app.services.patreon_client import MediaItem
|
||||
from backend.app.services.patreon_downloader import (
|
||||
MediaOutcome,
|
||||
@@ -26,10 +28,14 @@ _PNG_BYTES = _PNG_HEAD + (b"\x00" * 16) + _PNG_TAIL
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, payload: bytes):
|
||||
def __init__(self, payload: bytes, status_code: int = 200, headers=None):
|
||||
self._payload = payload
|
||||
self.status_code = status_code
|
||||
self.headers = headers or {}
|
||||
|
||||
def raise_for_status(self):
|
||||
if self.status_code >= 400:
|
||||
raise requests.HTTPError(f"HTTP {self.status_code}")
|
||||
return None
|
||||
|
||||
def iter_content(self, chunk_size=65536):
|
||||
@@ -49,6 +55,18 @@ class _FakeSession:
|
||||
return _FakeResponse(self.payloads.get(url, _PNG_BYTES))
|
||||
|
||||
|
||||
class _SeqSession:
|
||||
"""Returns the queued responses in order (for retry tests)."""
|
||||
|
||||
def __init__(self, responses: list[_FakeResponse]):
|
||||
self._responses = list(responses)
|
||||
self.calls: list[str] = []
|
||||
|
||||
def get(self, url, stream=False, timeout=None):
|
||||
self.calls.append(url)
|
||||
return self._responses.pop(0)
|
||||
|
||||
|
||||
def _post(post_id="1001", title="My Post Title", published="2026-05-01T12:00:00Z"):
|
||||
return {
|
||||
"id": post_id,
|
||||
@@ -250,3 +268,46 @@ def test_media_outcome_shape():
|
||||
o = MediaOutcome(media=None, status="downloaded", path=Path("/tmp/x"), error=None)
|
||||
assert o.status == "downloaded"
|
||||
assert o.path == Path("/tmp/x")
|
||||
|
||||
|
||||
# -- pacing + 429 backoff (plan #703) --------------------------------------
|
||||
|
||||
|
||||
def test_rate_limit_paces_each_real_download(tmp_path, monkeypatch):
|
||||
slept: list[float] = []
|
||||
monkeypatch.setattr(pd_mod.time, "sleep", lambda s: slept.append(s))
|
||||
dl = PatreonDownloader(
|
||||
images_root=tmp_path, cookies_path=None, validate=False,
|
||||
rate_limit=2.0, session=_FakeSession(),
|
||||
)
|
||||
dl.download_post(_post(), [_img("a.png"), _img("b.png")], "artist-x")
|
||||
# One pacing sleep per actual download (two media downloaded).
|
||||
assert slept == [2.0, 2.0]
|
||||
|
||||
|
||||
def test_skips_do_not_pace(tmp_path, monkeypatch):
|
||||
slept: list[float] = []
|
||||
monkeypatch.setattr(pd_mod.time, "sleep", lambda s: slept.append(s))
|
||||
dl = PatreonDownloader(
|
||||
images_root=tmp_path, cookies_path=None, validate=False,
|
||||
rate_limit=2.0, session=_FakeSession(),
|
||||
)
|
||||
# is_seen → skipped_seen, never reaches the download/pacing path.
|
||||
dl.download_post(_post(), [_img("a.png")], "artist-x", is_seen=lambda m: True)
|
||||
assert slept == []
|
||||
|
||||
|
||||
def test_media_429_retried_then_succeeds(tmp_path, monkeypatch):
|
||||
slept: list[float] = []
|
||||
monkeypatch.setattr(pd_mod.time, "sleep", lambda s: slept.append(s))
|
||||
session = _SeqSession([
|
||||
_FakeResponse(b"", status_code=429, headers={"Retry-After": "4"}),
|
||||
_FakeResponse(_PNG_BYTES, status_code=200),
|
||||
])
|
||||
dl = PatreonDownloader(
|
||||
images_root=tmp_path, cookies_path=None, validate=False, session=session,
|
||||
)
|
||||
outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x")
|
||||
assert outcomes[0].status == "downloaded"
|
||||
assert 4.0 in slept # backed off before the retry
|
||||
assert len(session.calls) == 2
|
||||
|
||||
Reference in New Issue
Block a user