feat(patreon): within-pass transient retry for media GETs — #8 (was overstated as done)
Honest completion of roadmap #8. Previously a NON-429 media failure (a connection reset, a timeout, a truncated stream, a 5xx) was an immediate terminal "error" for the pass — only retried on the NEXT walk. Now _fetch_to_file retries TRANSIENT failures in-place with backoff (transport blips incl. mid-download, 429 honoring Retry-After, and 5xx; up to 3 tries), while PERMANENT failures (404 gone / 403 forbidden) fail fast straight to the error → dead-letter path — re-fetching them is pointless. This makes the transient-vs-permanent split explicit instead of leaning on the next-tick cycle. (#1's 429 backoff + #7's dead-letter covered most of #8's value; this is the missing in-pass transient piece I'd loosely marked "folded".) Tests: a connection blip / a 5xx is retried then succeeds; a 404 errors with NO retry; an exhausted transient becomes a terminal error. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -56,15 +56,19 @@ class _FakeSession:
|
||||
|
||||
|
||||
class _SeqSession:
|
||||
"""Returns the queued responses in order (for retry tests)."""
|
||||
"""Returns the queued items in order (for retry tests). An Exception item is
|
||||
RAISED by get() instead of returned (simulates a transport error)."""
|
||||
|
||||
def __init__(self, responses: list[_FakeResponse]):
|
||||
def __init__(self, responses):
|
||||
self._responses = list(responses)
|
||||
self.calls: list[str] = []
|
||||
|
||||
def get(self, url, stream=False, timeout=None):
|
||||
self.calls.append(url)
|
||||
return self._responses.pop(0)
|
||||
item = self._responses.pop(0)
|
||||
if isinstance(item, Exception):
|
||||
raise item
|
||||
return item
|
||||
|
||||
|
||||
def _post(post_id="1001", title="My Post Title", published="2026-05-01T12:00:00Z"):
|
||||
@@ -314,3 +318,54 @@ def test_media_429_retried_then_succeeds(tmp_path, monkeypatch):
|
||||
assert outcomes[0].status == "downloaded"
|
||||
assert 4.0 in slept # backed off before the retry
|
||||
assert len(session.calls) == 2
|
||||
|
||||
|
||||
def _dl(tmp_path, session, monkeypatch):
|
||||
monkeypatch.setattr(pd_mod.time, "sleep", lambda s: None)
|
||||
return PatreonDownloader(
|
||||
images_root=tmp_path, cookies_path=None, validate=False, session=session,
|
||||
)
|
||||
|
||||
|
||||
def test_transient_transport_error_retried_within_pass(tmp_path, monkeypatch):
|
||||
"""plan #705 #8: a connection blip on the GET is retried in-place, not a
|
||||
terminal error that waits for the next walk."""
|
||||
session = _SeqSession([
|
||||
requests.ConnectionError("connection reset"),
|
||||
_FakeResponse(_PNG_BYTES, status_code=200),
|
||||
])
|
||||
dl = _dl(tmp_path, session, monkeypatch)
|
||||
outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x")
|
||||
assert outcomes[0].status == "downloaded"
|
||||
assert len(session.calls) == 2
|
||||
|
||||
|
||||
def test_5xx_retried_within_pass(tmp_path, monkeypatch):
|
||||
session = _SeqSession([
|
||||
_FakeResponse(b"", status_code=503),
|
||||
_FakeResponse(_PNG_BYTES, status_code=200),
|
||||
])
|
||||
dl = _dl(tmp_path, session, monkeypatch)
|
||||
outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x")
|
||||
assert outcomes[0].status == "downloaded"
|
||||
assert len(session.calls) == 2
|
||||
|
||||
|
||||
def test_404_fails_fast_no_retry(tmp_path, monkeypatch):
|
||||
"""A permanent 4xx (gone/forbidden) is NOT retried — it errors immediately
|
||||
(and the ingester's dead-letter ledger takes over across walks)."""
|
||||
session = _SeqSession([
|
||||
_FakeResponse(b"", status_code=404),
|
||||
_FakeResponse(_PNG_BYTES, status_code=200), # would succeed if it retried
|
||||
])
|
||||
dl = _dl(tmp_path, session, monkeypatch)
|
||||
outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x")
|
||||
assert outcomes[0].status == "error"
|
||||
assert len(session.calls) == 1 # no retry on a permanent failure
|
||||
|
||||
|
||||
def test_exhausted_transient_becomes_error(tmp_path, monkeypatch):
|
||||
session = _SeqSession([requests.Timeout("t")] * 10) # always times out
|
||||
dl = _dl(tmp_path, session, monkeypatch)
|
||||
outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x")
|
||||
assert outcomes[0].status == "error"
|
||||
|
||||
Reference in New Issue
Block a user