"""Unit tests for PatreonDownloader (build step 2b). No network, no real subprocess. The HTTP layer is stubbed via the `session` seam (a fake session whose `.get` returns a fake streaming response) and yt-dlp via monkeypatching `_run_ytdlp`. PURE module → no DB, so unmarked (fast lane). """ from __future__ import annotations import json from pathlib import Path import pytest import requests from backend.app.services import patreon_downloader as pd_mod from backend.app.services.native_ingest_common import ( MediaOutcome, PostRecordOutcome, post_dir_name, sanitize_segment, ) from backend.app.services.patreon_client import MediaItem from backend.app.services.patreon_downloader import PatreonDownloader from backend.app.utils.sidecar import find_sidecar # Minimal valid PNG so the file_validator passes on the happy path. _PNG_HEAD = b"\x89PNG\r\n\x1a\n" _PNG_TAIL = b"\x00\x00\x00\x00IEND\xaeB`\x82" _PNG_BYTES = _PNG_HEAD + (b"\x00" * 16) + _PNG_TAIL class _FakeResponse: def __init__(self, payload: bytes, status_code: int = 200, headers=None): self._payload = payload self.status_code = status_code self.headers = headers or {} def raise_for_status(self): if self.status_code >= 400: raise requests.HTTPError(f"HTTP {self.status_code}") return None def iter_content(self, chunk_size=65536): for i in range(0, len(self._payload), chunk_size): yield self._payload[i : i + chunk_size] class _FakeSession: """Records GETs and returns canned bytes (per-URL, default _PNG_BYTES).""" def __init__(self, payloads: dict[str, bytes] | None = None): self.payloads = payloads or {} self.calls: list[str] = [] def get(self, url, stream=False, timeout=None, headers=None): self.calls.append(url) return _FakeResponse(self.payloads.get(url, _PNG_BYTES)) class _SeqSession: """Returns the queued items in order (for retry tests). An Exception item is RAISED by get() instead of returned (simulates a transport error).""" def __init__(self, responses): self._responses = list(responses) self.calls: list[str] = [] def get(self, url, stream=False, timeout=None, headers=None): self.calls.append(url) item = self._responses.pop(0) if isinstance(item, Exception): raise item return item class _FlakyResp: """A streaming response that yields its payload then optionally raises a mid-stream transport error (for the Range-resume tests, plan #708 B5).""" headers: dict = {} def __init__(self, payload, status_code=200, die=False): self._payload = payload self.status_code = status_code self._die = die def raise_for_status(self): return None def iter_content(self, chunk_size=65536): yield self._payload if self._die: raise requests.exceptions.ChunkedEncodingError("cut mid-stream") def _post(post_id="1001", title="My Post Title", published="2026-05-01T12:00:00Z"): return { "id": post_id, "attributes": { "title": title, "content": "

body

", "published_at": published, "url": f"https://www.patreon.com/posts/{post_id}", }, } def _img(name, post_id="1001", url=None): return MediaItem( url=url or f"https://cdn.patreon.com/{name}", filename=name, kind="images", filehash=None, post_id=post_id, ) def _downloader(tmp_path: Path, session=None, validate=True): return PatreonDownloader( images_root=tmp_path, cookies_path=None, validate=validate, session=session or _FakeSession(), ) def test_path_layout_two_images(tmp_path): dl = _downloader(tmp_path) items = [_img("media1.png"), _img("media2.png")] outcomes = dl.download_post(_post(), items, "artist-x") post_dir = tmp_path / "artist-x" / "patreon" / "2026-05-01_1001_My Post Title" p1 = post_dir / "01_media1.png" p2 = post_dir / "02_media2.png" assert p1.is_file() assert p2.is_file() assert [o.status for o in outcomes] == ["downloaded", "downloaded"] assert outcomes[0].path == p1 assert p1.read_bytes() == _PNG_BYTES def test_dir_sanitized(tmp_path): dl = _downloader(tmp_path) post = _post(title='bad/name:with*chars?') dl.download_post(post, [_img("a.png")], "artist-x") patreon_dir = tmp_path / "artist-x" / "patreon" children = list(patreon_dir.iterdir()) assert len(children) == 1 name = children[0].name assert "/" not in name assert not any(c in name for c in '<>:"\\|?*') def test_no_date_prefix_when_unparseable(tmp_path): dl = _downloader(tmp_path) post = _post(published="not-a-date") dl.download_post(post, [_img("a.png")], "artist-x") patreon_dir = tmp_path / "artist-x" / "patreon" children = [c.name for c in patreon_dir.iterdir()] assert children == ["1001_My Post Title"] def test_sidecar_written_and_findable(tmp_path): dl = _downloader(tmp_path) outcomes = dl.download_post(_post(), [_img("media1.png")], "artist-x") media_path = outcomes[0].path sidecar = find_sidecar(media_path) assert sidecar is not None assert sidecar.name == "01_media1.json" import json data = json.loads(sidecar.read_text()) assert data["category"] == "patreon" assert data["id"] == "1001" # Post-first (#856): the per-media sidecar carries image identity ONLY — no # post body/title/url. The post-record (`_post.json`) owns those now. assert "title" not in data assert "url" not in data assert "content" not in data # #830 Phase 2: the per-media sidecar records THIS file's CDN URL so the # importer can persist its filehash for inline-image localization. assert data["source_url"] == "https://cdn.patreon.com/media1.png" def test_skip_seen(tmp_path): session = _FakeSession() dl = _downloader(tmp_path, session=session) outcomes = dl.download_post( _post(), [_img("media1.png")], "artist-x", is_seen=lambda m: True ) assert outcomes[0].status == "skipped_seen" assert outcomes[0].path is None assert session.calls == [] # No file written anywhere. assert not (tmp_path / "artist-x").exists() def test_skip_disk(tmp_path): dl = _downloader(tmp_path) post_dir = tmp_path / "artist-x" / "patreon" / "2026-05-01_1001_My Post Title" post_dir.mkdir(parents=True) existing = post_dir / "01_media1.png" existing.write_bytes(b"already here") outcomes = dl.download_post(_post(), [_img("media1.png")], "artist-x") assert outcomes[0].status == "skipped_disk" assert outcomes[0].path == existing assert existing.read_bytes() == b"already here" # untouched def test_recapture_surfaces_seen_on_disk_without_download(tmp_path): """#830 recapture: a seen item that IS on disk is surfaced as skipped_disk (with its path, for source_filehash backfill) — NOT tier-1 short-circuited, and never re-downloaded.""" session = _FakeSession() dl = _downloader(tmp_path, session=session) post_dir = tmp_path / "artist-x" / "patreon" / "2026-05-01_1001_My Post Title" post_dir.mkdir(parents=True) existing = post_dir / "01_media1.png" existing.write_bytes(b"already here") outcomes = dl.download_post( _post(), [_img("media1.png")], "artist-x", is_seen=lambda m: True, recapture=True, ) assert outcomes[0].status == "skipped_disk" assert outcomes[0].path == existing assert session.calls == [] # no re-download assert existing.read_bytes() == b"already here" # untouched def test_recapture_does_not_refetch_seen_missing(tmp_path): """#830 recapture: a seen item that's NOT on disk is left alone (skipped_seen, no download) — refetching is recovery's job, not recapture's.""" session = _FakeSession() dl = _downloader(tmp_path, session=session) outcomes = dl.download_post( _post(), [_img("media1.png")], "artist-x", is_seen=lambda m: True, recapture=True, ) assert outcomes[0].status == "skipped_seen" assert outcomes[0].path is None assert session.calls == [] # not refetched def test_recapture_still_downloads_genuinely_new(tmp_path): """#830 recapture: media that is neither seen nor on disk still downloads — a recapture walk is thorough, it just doesn't re-pull existing files.""" session = _FakeSession() dl = _downloader(tmp_path, session=session) outcomes = dl.download_post( _post(), [_img("media1.png")], "artist-x", is_seen=lambda m: False, recapture=True, ) assert outcomes[0].status == "downloaded" assert outcomes[0].path is not None def test_video_routes_to_ytdlp(tmp_path, monkeypatch): session = _FakeSession() dl = _downloader(tmp_path, session=session) captured = {} def fake_ytdlp(url, dest, headers): captured["url"] = url captured["headers"] = headers out = Path(dest).with_suffix(".mp4") out.parent.mkdir(parents=True, exist_ok=True) out.write_bytes(b"video-bytes") return out monkeypatch.setattr(dl, "_run_ytdlp", fake_ytdlp) video = MediaItem( url="https://stream.mux.com/abc123.m3u8", filename="clip.mp4", kind="postfile", filehash=None, post_id="1001", ) outcomes = dl.download_post(_post(), [video], "artist-x") assert captured["url"] == "https://stream.mux.com/abc123.m3u8" assert captured["headers"]["Referer"] == "https://www.patreon.com/" assert captured["headers"]["Origin"] == "https://www.patreon.com" assert session.calls == [] # NOT the plain-GET path assert outcomes[0].status == "downloaded" assert outcomes[0].path.read_bytes() == b"video-bytes" # Sidecar written next to the actual (remuxed) output. assert find_sidecar(outcomes[0].path) is not None def _video_item(): return MediaItem( url="https://stream.mux.com/abc123.m3u8", filename="clip.mp4", kind="postfile", filehash=None, post_id="1001", ) def test_video_transient_failure_retried_then_succeeds(tmp_path, monkeypatch): """plan #705 #8: a TRANSIENT yt-dlp failure (TimeoutExpired) is retried within the pass; a later success yields a downloaded outcome. (The GET path already retried transients; the video path didn't until #8 closed the gap.)""" calls = {"n": 0} def fake_run(cmd, **kwargs): calls["n"] += 1 if calls["n"] == 1: raise pd_mod.subprocess.TimeoutExpired(cmd, kwargs.get("timeout")) out = Path(cmd[cmd.index("-o") + 1].replace(".%(ext)s", ".mp4")) out.parent.mkdir(parents=True, exist_ok=True) out.write_bytes(b"video-bytes") return pd_mod.subprocess.CompletedProcess(cmd, 0, "", "") monkeypatch.setattr(pd_mod.subprocess, "run", fake_run) monkeypatch.setattr(pd_mod.time, "sleep", lambda *_a, **_k: None) dl = _downloader(tmp_path, session=_FakeSession()) outcomes = dl.download_post(_post(), [_video_item()], "artist-x") assert calls["n"] == 2 # retried once after the transient blip assert outcomes[0].status == "downloaded" assert outcomes[0].path.read_bytes() == b"video-bytes" def test_video_permanent_failure_fails_fast(tmp_path, monkeypatch): """plan #705 #8: a non-zero yt-dlp exit (CalledProcessError) is PERMANENT — yt-dlp already did its own network retries — so we don't re-spawn; the item errors straight to the per-item/dead-letter path.""" calls = {"n": 0} def fake_run(cmd, **kwargs): calls["n"] += 1 raise pd_mod.subprocess.CalledProcessError(1, cmd, stderr="ERROR: gone") monkeypatch.setattr(pd_mod.subprocess, "run", fake_run) monkeypatch.setattr(pd_mod.time, "sleep", lambda *_a, **_k: None) dl = _downloader(tmp_path, session=_FakeSession()) outcomes = dl.download_post(_post(), [_video_item()], "artist-x") assert calls["n"] == 1 # no retry on a permanent failure assert outcomes[0].status == "error" def test_one_failure_isolated(tmp_path): class _BoomSession(_FakeSession): def get(self, url, stream=False, timeout=None, headers=None): self.calls.append(url) if "bad" in url: raise RuntimeError("network boom") return _FakeResponse(_PNG_BYTES) dl = _downloader(tmp_path, session=_BoomSession()) items = [ _img("good.png", url="https://cdn.patreon.com/good.png"), _img("bad.png", url="https://cdn.patreon.com/bad.png"), ] outcomes = dl.download_post(_post(), items, "artist-x") assert outcomes[0].status == "downloaded" assert outcomes[1].status == "error" assert "boom" in outcomes[1].error def test_validation_quarantines_corrupt(tmp_path): # A .png whose bytes are not a valid PNG → validator fails → "quarantined" # outcome (distinct from a download error) carrying the dest path + # quarantine move; with validate=False it would pass. plan #704 (#4). bad = b"not a real png" session = _FakeSession({"https://cdn.patreon.com/corrupt.png": bad}) dl = _downloader(tmp_path, session=session, validate=True) item = _img("corrupt.png", url="https://cdn.patreon.com/corrupt.png") outcomes = dl.download_post(_post(), [item], "artist-x") assert outcomes[0].status == "quarantined" assert outcomes[0].path is not None assert "_quarantine" in str(outcomes[0].path) post_dir = tmp_path / "artist-x" / "patreon" / "2026-05-01_1001_My Post Title" assert not (post_dir / "01_corrupt.png").exists() quarantine = tmp_path / "_quarantine" / "artist-x" / "patreon" assert quarantine.is_dir() assert any(quarantine.iterdir()) def test_quarantine_writes_provenance_sidecar(tmp_path): """A1 parity fix: the native path now writes the same `.quarantine.json` provenance sidecar gallery-dl writes (it skipped it before the shared file_validator.quarantine_file helper).""" import json bad = b"not a real png" url = "https://cdn.patreon.com/corrupt.png" session = _FakeSession({url: bad}) dl = _downloader(tmp_path, session=session, validate=True) outcomes = dl.download_post(_post(), [_img("corrupt.png", url=url)], "artist-x") dest = outcomes[0].path sidecar = dest.with_suffix(dest.suffix + ".quarantine.json") assert sidecar.is_file() meta = json.loads(sidecar.read_text()) assert meta["source_url"] == url assert meta["platform"] == "patreon" assert meta["artist_slug"] == "artist-x" assert meta["reason"] def test_validation_off_keeps_bytes(tmp_path): bad = b"not a real png" session = _FakeSession({"https://cdn.patreon.com/x.png": bad}) dl = _downloader(tmp_path, session=session, validate=False) item = _img("x.png", url="https://cdn.patreon.com/x.png") outcomes = dl.download_post(_post(), [item], "artist-x") assert outcomes[0].status == "downloaded" assert outcomes[0].path.read_bytes() == bad def test_sanitize_helper(): assert sanitize_segment('a/b') == "a_b" assert sanitize_segment('x<>:"|?*y') == "x_______y" assert sanitize_segment("trail. ") == "trail" assert sanitize_segment("") == "_" def test_media_outcome_shape(): o = MediaOutcome(media=None, status="downloaded", path=Path("/tmp/x"), error=None) assert o.status == "downloaded" assert o.path == Path("/tmp/x") # -- pacing + 429 backoff (plan #703) -------------------------------------- def test_rate_limit_paces_each_real_download(tmp_path, monkeypatch): slept: list[float] = [] monkeypatch.setattr(pd_mod.time, "sleep", lambda s: slept.append(s)) dl = PatreonDownloader( images_root=tmp_path, cookies_path=None, validate=False, rate_limit=2.0, session=_FakeSession(), ) dl.download_post(_post(), [_img("a.png"), _img("b.png")], "artist-x") # One pacing sleep per actual download (two media downloaded). assert slept == [2.0, 2.0] def test_skips_do_not_pace(tmp_path, monkeypatch): slept: list[float] = [] monkeypatch.setattr(pd_mod.time, "sleep", lambda s: slept.append(s)) dl = PatreonDownloader( images_root=tmp_path, cookies_path=None, validate=False, rate_limit=2.0, session=_FakeSession(), ) # is_seen → skipped_seen, never reaches the download/pacing path. dl.download_post(_post(), [_img("a.png")], "artist-x", is_seen=lambda m: True) assert slept == [] def test_download_post_honors_should_stop_mid_post(tmp_path): """The time-box is polled BEFORE each media item, so a heavy post can't run the whole chunk past its budget (Pocketacer soft-limit, 2026-06-07).""" dl = PatreonDownloader( images_root=tmp_path, cookies_path=None, validate=False, session=_FakeSession(), ) items = [_img("a.png"), _img("b.png"), _img("c.png")] polls = {"n": 0} def _stop(): polls["n"] += 1 return polls["n"] > 1 # allow the first item, stop before the second outcomes = dl.download_post(_post(), items, "artist-x", should_stop=_stop) assert len(outcomes) == 1 # only the first item ran; the rest re-fetch later assert outcomes[0].status == "downloaded" def test_media_429_retried_then_succeeds(tmp_path, monkeypatch): slept: list[float] = [] monkeypatch.setattr(pd_mod.time, "sleep", lambda s: slept.append(s)) session = _SeqSession([ _FakeResponse(b"", status_code=429, headers={"Retry-After": "4"}), _FakeResponse(_PNG_BYTES, status_code=200), ]) dl = PatreonDownloader( images_root=tmp_path, cookies_path=None, validate=False, session=session, ) outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x") assert outcomes[0].status == "downloaded" assert 4.0 in slept # backed off before the retry assert len(session.calls) == 2 def _dl(tmp_path, session, monkeypatch): monkeypatch.setattr(pd_mod.time, "sleep", lambda s: None) return PatreonDownloader( images_root=tmp_path, cookies_path=None, validate=False, session=session, ) def test_transient_transport_error_retried_within_pass(tmp_path, monkeypatch): """plan #705 #8: a connection blip on the GET is retried in-place, not a terminal error that waits for the next walk.""" session = _SeqSession([ requests.ConnectionError("connection reset"), _FakeResponse(_PNG_BYTES, status_code=200), ]) dl = _dl(tmp_path, session, monkeypatch) outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x") assert outcomes[0].status == "downloaded" assert len(session.calls) == 2 def test_5xx_retried_within_pass(tmp_path, monkeypatch): session = _SeqSession([ _FakeResponse(b"", status_code=503), _FakeResponse(_PNG_BYTES, status_code=200), ]) dl = _dl(tmp_path, session, monkeypatch) outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x") assert outcomes[0].status == "downloaded" assert len(session.calls) == 2 def test_404_fails_fast_no_retry(tmp_path, monkeypatch): """A permanent 4xx (gone/forbidden) is NOT retried — it errors immediately (and the ingester's dead-letter ledger takes over across walks).""" session = _SeqSession([ _FakeResponse(b"", status_code=404), _FakeResponse(_PNG_BYTES, status_code=200), # would succeed if it retried ]) dl = _dl(tmp_path, session, monkeypatch) outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x") assert outcomes[0].status == "error" assert len(session.calls) == 1 # no retry on a permanent failure def test_exhausted_transient_becomes_error(tmp_path, monkeypatch): session = _SeqSession([requests.Timeout("t")] * 10) # always times out dl = _dl(tmp_path, session, monkeypatch) outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x") assert outcomes[0].status == "error" def test_partial_download_resumes_via_range(tmp_path, monkeypatch): """plan #708 B5: a mid-download transport cut resumes from the bytes already on disk via a Range request, not a refetch from zero.""" full = bytes(i % 256 for i in range(1000)) split = 400 class _ResumeSession: def __init__(self): self.headers_seen = [] def get(self, url, stream=False, timeout=None, headers=None): self.headers_seen.append(headers) if len(self.headers_seen) == 1: return _FlakyResp(full[:split], die=True) # partial then cut start = int(headers["Range"].split("=")[1].rstrip("-")) return _FlakyResp(full[start:], status_code=206) monkeypatch.setattr(pd_mod.time, "sleep", lambda *_a: None) session = _ResumeSession() dl = PatreonDownloader( images_root=tmp_path, cookies_path=None, validate=False, session=session, ) outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x") assert outcomes[0].status == "downloaded" assert outcomes[0].path.read_bytes() == full assert session.headers_seen[0] is None # first GET: no Range assert session.headers_seen[1] == {"Range": "bytes=400-"} # resume from offset def test_range_ignored_by_server_refetches_clean(tmp_path, monkeypatch): """plan #708 B5: if the server ignores Range (200, not 206) on the resume, we truncate and start clean — never append into a corrupt double-write.""" full = b"ABCD" * 250 # 1000 bytes split = 400 class _NoRangeSession: def __init__(self): self.n = 0 def get(self, url, stream=False, timeout=None, headers=None): self.n += 1 if self.n == 1: return _FlakyResp(full[:split], die=True) return _FlakyResp(full, status_code=200) # ignores Range, whole file monkeypatch.setattr(pd_mod.time, "sleep", lambda *_a: None) dl = PatreonDownloader( images_root=tmp_path, cookies_path=None, validate=False, session=_NoRangeSession(), ) outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x") assert outcomes[0].status == "downloaded" assert outcomes[0].path.read_bytes() == full # clean, not full[:400] + full # -- content enrichment (adaptive detail-fetch of an empty feed body) ------- def test_media_sidecar_is_minimal_post_first(tmp_path): """Post-first (#856): the per-media sidecar carries ONLY image identity (category/id/source_url), NOT the post body — the post-record (`_post.json`) owns the body and is written first in the walk. So download_post never hits the detail fetcher (that's write_post_record's job), and the body is no longer duplicated next to every image.""" calls: list[str] = [] dl = PatreonDownloader( images_root=tmp_path, cookies_path=None, validate=False, session=_FakeSession(), content_fetcher=lambda pid: calls.append(pid) or "

full body

", ) post = _post() post["attributes"]["content"] = "" # even an empty feed body isn't fetched here dl.download_post(post, [_img("a.png"), _img("b.png")], "artist-x") post_dir = tmp_path / "artist-x" / "patreon" / post_dir_name(post) sc = find_sidecar(post_dir / "01_a.png") assert sc is not None data = json.loads(sc.read_text()) assert data == { "category": "patreon", "id": "1001", "source_url": "https://cdn.patreon.com/a.png", } # No post body/title next to the image; the detail endpoint is never hit here. assert "content" not in data assert "title" not in data assert calls == [] # -- write_post_record (media-less / text-only post capture) ---------------- def test_write_post_record_writes_enriched_post_only_sidecar(tmp_path): calls: list[str] = [] def _fetcher(post_id: str) -> str: calls.append(post_id) return "

text post body

" dl = PatreonDownloader( images_root=tmp_path, cookies_path=None, validate=False, session=_FakeSession(), content_fetcher=_fetcher, ) post = _post() post["attributes"]["content"] = "" # media-less post, empty feed body rec = dl.write_post_record(post, "artist-x") assert isinstance(rec, PostRecordOutcome) assert rec.path is not None assert rec.path.name == "_post.json" # can't collide with a media `NN_*.json` data = json.loads(rec.path.read_text()) assert data["id"] == "1001" assert data["content"] == "

text post body

" assert calls == ["1001"] # Outcome reports the captured body's shape (detail-fetched here). assert rec.body_chars == len("

text post body

") def test_write_post_record_none_without_post_id(tmp_path): dl = PatreonDownloader( images_root=tmp_path, cookies_path=None, validate=False, session=_FakeSession(), ) rec = dl.write_post_record({"id": "", "attributes": {}}, "artist-x") assert rec.path is None assert rec.body_chars == 0