feat(patreon): ingester rate-limit resilience — #703 step 1
A single 429 mid-walk used to fail the run → RATE_LIMITED → platform-wide
cooldown → every Patreon source dark ("testing dead"). The native path also
ignored the operator's existing politeness setting. Fixed both:
- Pacing (avoid 429s): honor download_rate_limit_seconds (gallery-dl's
`rate_limit`, read off self.gdl) as a pre-download sleep on real media
downloads only (skips don't pace); pace /api/posts page fetches with the
per-source sleep_request override, defaulting to max(0.5, rate_limit/4) —
the same API-pacing default gallery-dl used for `sleep-request`.
- 429 backoff (ride out transient limits): PatreonClient._fetch retries a
429 with backoff (honor Retry-After, else exponential 2·2^(n-1), capped
30s, ≤3 tries); only a PERSISTENT 429 propagates as terminal
RATE_LIMITED. Light 2-retry on a media-GET 429 too.
Threaded via PatreonIngester(rate_limit=, request_sleep=) →
PatreonClient/PatreonDownloader; download_service sources them. Injected
test client/downloader are unaffected (carry their own pacing).
Tests mock time.sleep (no real sleeping): retry-then-success, persistent
429 raises after N, Retry-After honored, request_sleep paces, media pacing
per real download, skips don't pace, media 429 retried.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -10,12 +10,14 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from backend.app.services import patreon_client as pc_mod
|
||||
from backend.app.services.patreon_client import (
|
||||
MediaItem,
|
||||
PatreonAPIError,
|
||||
PatreonAuthError,
|
||||
PatreonClient,
|
||||
PatreonDriftError,
|
||||
_retry_after_seconds,
|
||||
parse_cursor_from_url,
|
||||
)
|
||||
|
||||
@@ -192,10 +194,11 @@ def test_media_referenced_but_absent_raises_drift(client):
|
||||
|
||||
|
||||
class _FakeResp:
|
||||
def __init__(self, status_code, *, json_data=None, raise_json=False):
|
||||
def __init__(self, status_code, *, json_data=None, raise_json=False, headers=None):
|
||||
self.status_code = status_code
|
||||
self._json_data = json_data
|
||||
self._raise_json = raise_json
|
||||
self.headers = headers or {}
|
||||
|
||||
def json(self):
|
||||
if self._raise_json:
|
||||
@@ -275,3 +278,62 @@ def test_verify_auth_inconclusive_on_network(monkeypatch):
|
||||
monkeypatch.setattr(client._session, "get", _raise)
|
||||
ok, _msg = client.verify_auth("5555")
|
||||
assert ok is None
|
||||
|
||||
|
||||
# -- 429 backoff + pacing (plan #703) --------------------------------------
|
||||
|
||||
|
||||
def test_retry_after_seconds_honors_numeric_header():
|
||||
assert _retry_after_seconds(_FakeResp(429, headers={"Retry-After": "7"}), 1) == 7.0
|
||||
|
||||
|
||||
def test_retry_after_seconds_exponential_without_header():
|
||||
resp = _FakeResp(429)
|
||||
assert _retry_after_seconds(resp, 1) == 2.0
|
||||
assert _retry_after_seconds(resp, 2) == 4.0
|
||||
assert _retry_after_seconds(resp, 3) == 8.0
|
||||
|
||||
|
||||
def test_retry_after_seconds_caps():
|
||||
assert _retry_after_seconds(_FakeResp(429), 10) == 30.0
|
||||
assert _retry_after_seconds(_FakeResp(429, headers={"Retry-After": "999"}), 1) == 30.0
|
||||
|
||||
|
||||
def _client_with_sequence(monkeypatch, responses):
|
||||
"""A client whose session yields `responses` in order; time.sleep stubbed so
|
||||
backoff never really sleeps. Returns (client, slept_list)."""
|
||||
client = PatreonClient(cookies_path=None)
|
||||
it = iter(responses)
|
||||
monkeypatch.setattr(client._session, "get", lambda *a, **k: next(it))
|
||||
slept: list[float] = []
|
||||
monkeypatch.setattr(pc_mod.time, "sleep", lambda s: slept.append(s))
|
||||
return client, slept
|
||||
|
||||
|
||||
def test_fetch_retries_then_succeeds_on_429(monkeypatch):
|
||||
client, slept = _client_with_sequence(monkeypatch, [
|
||||
_FakeResp(429, headers={"Retry-After": "5"}),
|
||||
_FakeResp(200, json_data={"data": []}),
|
||||
])
|
||||
assert client._fetch("5555", None) == {"data": []}
|
||||
assert 5.0 in slept # backed off once before the retry
|
||||
|
||||
|
||||
def test_fetch_persistent_429_raises_after_retries(monkeypatch):
|
||||
client, _slept = _client_with_sequence(monkeypatch, [_FakeResp(429)] * 10)
|
||||
with pytest.raises(PatreonAPIError) as ei:
|
||||
client._fetch("5555", None)
|
||||
assert ei.value.status_code == 429
|
||||
assert not isinstance(ei.value, PatreonAuthError)
|
||||
|
||||
|
||||
def test_request_sleep_paces_before_fetch(monkeypatch):
|
||||
client = PatreonClient(cookies_path=None, request_sleep=1.5)
|
||||
monkeypatch.setattr(
|
||||
client._session, "get",
|
||||
lambda *a, **k: _FakeResp(200, json_data={"data": []}),
|
||||
)
|
||||
slept: list[float] = []
|
||||
monkeypatch.setattr(pc_mod.time, "sleep", lambda s: slept.append(s))
|
||||
client._fetch("5555", None)
|
||||
assert slept == [1.5]
|
||||
|
||||
Reference in New Issue
Block a user