feat(patreon): resume partial media downloads via HTTP Range — B5 (plan #708)
CI / lint (push) Successful in 3s
CI / frontend-build (push) Successful in 17s
CI / backend-lint-and-test (push) Failing after 26s
CI / integration (push) Successful in 3m1s

Owning the media fetch means a mid-download transport cut no longer refetches
from zero. _fetch_to_file now resumes: on a transient retry, if bytes already
landed in the .part, it requests Range: bytes=<have>- and appends on a 206;
falls back to a clean truncate-and-restart if the server ignores Range (200) or
the range is past EOF (416). The .part staging means a non-range server never
corrupts the output — worst case is the old behavior (refetch from zero).

Tests: mid-stream cut resumes from the offset (asserts the Range header);
a Range-ignoring server refetches clean (no double-write). Test session fakes
updated to accept the new headers kwarg.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-06 11:38:11 -04:00
parent fb7383eea7
commit 402086c34c
2 changed files with 99 additions and 5 deletions
+77 -2
View File
@@ -50,7 +50,7 @@ class _FakeSession:
self.payloads = payloads or {}
self.calls: list[str] = []
def get(self, url, stream=False, timeout=None):
def get(self, url, stream=False, timeout=None, headers=None):
self.calls.append(url)
return _FakeResponse(self.payloads.get(url, _PNG_BYTES))
@@ -63,7 +63,7 @@ class _SeqSession:
self._responses = list(responses)
self.calls: list[str] = []
def get(self, url, stream=False, timeout=None):
def get(self, url, stream=False, timeout=None, headers=None):
self.calls.append(url)
item = self._responses.pop(0)
if isinstance(item, Exception):
@@ -71,6 +71,26 @@ class _SeqSession:
return item
class _FlakyResp:
"""A streaming response that yields its payload then optionally raises a
mid-stream transport error (for the Range-resume tests, plan #708 B5)."""
headers: dict = {}
def __init__(self, payload, status_code=200, die=False):
self._payload = payload
self.status_code = status_code
self._die = die
def raise_for_status(self):
return None
def iter_content(self, chunk_size=65536):
yield self._payload
if self._die:
raise requests.exceptions.ChunkedEncodingError("cut mid-stream")
def _post(post_id="1001", title="My Post Title", published="2026-05-01T12:00:00Z"):
return {
"id": post_id,
@@ -442,3 +462,58 @@ def test_exhausted_transient_becomes_error(tmp_path, monkeypatch):
dl = _dl(tmp_path, session, monkeypatch)
outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x")
assert outcomes[0].status == "error"
def test_partial_download_resumes_via_range(tmp_path, monkeypatch):
"""plan #708 B5: a mid-download transport cut resumes from the bytes already
on disk via a Range request, not a refetch from zero."""
full = bytes(i % 256 for i in range(1000))
split = 400
class _ResumeSession:
def __init__(self):
self.headers_seen = []
def get(self, url, stream=False, timeout=None, headers=None):
self.headers_seen.append(headers)
if len(self.headers_seen) == 1:
return _FlakyResp(full[:split], die=True) # partial then cut
start = int(headers["Range"].split("=")[1].rstrip("-"))
return _FlakyResp(full[start:], status_code=206)
monkeypatch.setattr(pd_mod.time, "sleep", lambda *_a: None)
session = _ResumeSession()
dl = PatreonDownloader(
images_root=tmp_path, cookies_path=None, validate=False, session=session,
)
outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x")
assert outcomes[0].status == "downloaded"
assert outcomes[0].path.read_bytes() == full
assert session.headers_seen[0] is None # first GET: no Range
assert session.headers_seen[1] == {"Range": "bytes=400-"} # resume from offset
def test_range_ignored_by_server_refetches_clean(tmp_path, monkeypatch):
"""plan #708 B5: if the server ignores Range (200, not 206) on the resume, we
truncate and start clean — never append into a corrupt double-write."""
full = b"ABCD" * 250 # 1000 bytes
split = 400
class _NoRangeSession:
def __init__(self):
self.n = 0
def get(self, url, stream=False, timeout=None, headers=None):
self.n += 1
if self.n == 1:
return _FlakyResp(full[:split], die=True)
return _FlakyResp(full, status_code=200) # ignores Range, whole file
monkeypatch.setattr(pd_mod.time, "sleep", lambda *_a: None)
dl = PatreonDownloader(
images_root=tmp_path, cookies_path=None, validate=False,
session=_NoRangeSession(),
)
outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x")
assert outcomes[0].status == "downloaded"
assert outcomes[0].path.read_bytes() == full # clean, not full[:400] + full