FabledCurator/tests/test_patreon_downloader.py

"""Unit tests for PatreonDownloader (build step 2b).

No network, no real subprocess. The HTTP layer is stubbed via the `session`
seam (a fake session whose `.get` returns a fake streaming response) and yt-dlp
via monkeypatching `_run_ytdlp`. PURE module → no DB, so unmarked (fast lane).
"""

from __future__ import annotations

import json
from pathlib import Path

import pytest
import requests

from backend.app.services import patreon_downloader as pd_mod
from backend.app.services.patreon_client import MediaItem
from backend.app.services.patreon_downloader import (
    MediaOutcome,
    PatreonDownloader,
    _sanitize,
)
from backend.app.utils.sidecar import find_sidecar

# Minimal valid PNG so the file_validator passes on the happy path.
_PNG_HEAD = b"\x89PNG\r\n\x1a\n"
_PNG_TAIL = b"\x00\x00\x00\x00IEND\xaeB`\x82"
_PNG_BYTES = _PNG_HEAD + (b"\x00" * 16) + _PNG_TAIL


class _FakeResponse:
    def __init__(self, payload: bytes, status_code: int = 200, headers=None):
        self._payload = payload
        self.status_code = status_code
        self.headers = headers or {}

    def raise_for_status(self):
        if self.status_code >= 400:
            raise requests.HTTPError(f"HTTP {self.status_code}")
        return None

    def iter_content(self, chunk_size=65536):
        for i in range(0, len(self._payload), chunk_size):
            yield self._payload[i : i + chunk_size]


class _FakeSession:
    """Records GETs and returns canned bytes (per-URL, default _PNG_BYTES)."""

    def __init__(self, payloads: dict[str, bytes] | None = None):
        self.payloads = payloads or {}
        self.calls: list[str] = []

    def get(self, url, stream=False, timeout=None, headers=None):
        self.calls.append(url)
        return _FakeResponse(self.payloads.get(url, _PNG_BYTES))


class _SeqSession:
    """Returns the queued items in order (for retry tests). An Exception item is
    RAISED by get() instead of returned (simulates a transport error)."""

    def __init__(self, responses):
        self._responses = list(responses)
        self.calls: list[str] = []

    def get(self, url, stream=False, timeout=None, headers=None):
        self.calls.append(url)
        item = self._responses.pop(0)
        if isinstance(item, Exception):
            raise item
        return item


class _FlakyResp:
    """A streaming response that yields its payload then optionally raises a
    mid-stream transport error (for the Range-resume tests, plan #708 B5)."""

    headers: dict = {}

    def __init__(self, payload, status_code=200, die=False):
        self._payload = payload
        self.status_code = status_code
        self._die = die

    def raise_for_status(self):
        return None

    def iter_content(self, chunk_size=65536):
        yield self._payload
        if self._die:
            raise requests.exceptions.ChunkedEncodingError("cut mid-stream")


def _post(post_id="1001", title="My Post Title", published="2026-05-01T12:00:00Z"):
    return {
        "id": post_id,
        "attributes": {
            "title": title,
            "content": "<p>body</p>",
            "published_at": published,
            "url": f"https://www.patreon.com/posts/{post_id}",
        },
    }


def _img(name, post_id="1001", url=None):
    return MediaItem(
        url=url or f"https://cdn.patreon.com/{name}",
        filename=name,
        kind="images",
        filehash=None,
        post_id=post_id,
    )


def _downloader(tmp_path: Path, session=None, validate=True):
    return PatreonDownloader(
        images_root=tmp_path,
        cookies_path=None,
        validate=validate,
        session=session or _FakeSession(),
    )


def test_path_layout_two_images(tmp_path):
    dl = _downloader(tmp_path)
    items = [_img("media1.png"), _img("media2.png")]
    outcomes = dl.download_post(_post(), items, "artist-x")

    post_dir = tmp_path / "artist-x" / "patreon" / "2026-05-01_1001_My Post Title"
    p1 = post_dir / "01_media1.png"
    p2 = post_dir / "02_media2.png"
    assert p1.is_file()
    assert p2.is_file()
    assert [o.status for o in outcomes] == ["downloaded", "downloaded"]
    assert outcomes[0].path == p1
    assert p1.read_bytes() == _PNG_BYTES


def test_dir_sanitized(tmp_path):
    dl = _downloader(tmp_path)
    post = _post(title='bad/name:with*chars?')
    dl.download_post(post, [_img("a.png")], "artist-x")
    patreon_dir = tmp_path / "artist-x" / "patreon"
    children = list(patreon_dir.iterdir())
    assert len(children) == 1
    name = children[0].name
    assert "/" not in name
    assert not any(c in name for c in '<>:"\\|?*')


def test_no_date_prefix_when_unparseable(tmp_path):
    dl = _downloader(tmp_path)
    post = _post(published="not-a-date")
    dl.download_post(post, [_img("a.png")], "artist-x")
    patreon_dir = tmp_path / "artist-x" / "patreon"
    children = [c.name for c in patreon_dir.iterdir()]
    assert children == ["1001_My Post Title"]


def test_sidecar_written_and_findable(tmp_path):
    dl = _downloader(tmp_path)
    outcomes = dl.download_post(_post(), [_img("media1.png")], "artist-x")
    media_path = outcomes[0].path

    sidecar = find_sidecar(media_path)
    assert sidecar is not None
    assert sidecar.name == "01_media1.json"

    import json

    data = json.loads(sidecar.read_text())
    assert data["category"] == "patreon"
    assert data["id"] == "1001"
    assert data["title"] == "My Post Title"
    assert data["url"] == "https://www.patreon.com/posts/1001"
    # #830 Phase 2: the per-media sidecar records THIS file's CDN URL so the
    # importer can persist its filehash for inline-image localization.
    assert data["source_url"] == "https://cdn.patreon.com/media1.png"


def test_skip_seen(tmp_path):
    session = _FakeSession()
    dl = _downloader(tmp_path, session=session)
    outcomes = dl.download_post(
        _post(), [_img("media1.png")], "artist-x", is_seen=lambda m: True
    )
    assert outcomes[0].status == "skipped_seen"
    assert outcomes[0].path is None
    assert session.calls == []
    # No file written anywhere.
    assert not (tmp_path / "artist-x").exists()


def test_skip_disk(tmp_path):
    dl = _downloader(tmp_path)
    post_dir = tmp_path / "artist-x" / "patreon" / "2026-05-01_1001_My Post Title"
    post_dir.mkdir(parents=True)
    existing = post_dir / "01_media1.png"
    existing.write_bytes(b"already here")

    outcomes = dl.download_post(_post(), [_img("media1.png")], "artist-x")
    assert outcomes[0].status == "skipped_disk"
    assert outcomes[0].path == existing
    assert existing.read_bytes() == b"already here"  # untouched


def test_video_routes_to_ytdlp(tmp_path, monkeypatch):
    session = _FakeSession()
    dl = _downloader(tmp_path, session=session)
    captured = {}

    def fake_ytdlp(url, dest, headers):
        captured["url"] = url
        captured["headers"] = headers
        out = Path(dest).with_suffix(".mp4")
        out.parent.mkdir(parents=True, exist_ok=True)
        out.write_bytes(b"video-bytes")
        return out

    monkeypatch.setattr(dl, "_run_ytdlp", fake_ytdlp)

    video = MediaItem(
        url="https://stream.mux.com/abc123.m3u8",
        filename="clip.mp4",
        kind="postfile",
        filehash=None,
        post_id="1001",
    )
    outcomes = dl.download_post(_post(), [video], "artist-x")

    assert captured["url"] == "https://stream.mux.com/abc123.m3u8"
    assert captured["headers"]["Referer"] == "https://www.patreon.com/"
    assert captured["headers"]["Origin"] == "https://www.patreon.com"
    assert session.calls == []  # NOT the plain-GET path
    assert outcomes[0].status == "downloaded"
    assert outcomes[0].path.read_bytes() == b"video-bytes"
    # Sidecar written next to the actual (remuxed) output.
    assert find_sidecar(outcomes[0].path) is not None


def _video_item():
    return MediaItem(
        url="https://stream.mux.com/abc123.m3u8", filename="clip.mp4",
        kind="postfile", filehash=None, post_id="1001",
    )


def test_video_transient_failure_retried_then_succeeds(tmp_path, monkeypatch):
    """plan #705 #8: a TRANSIENT yt-dlp failure (TimeoutExpired) is retried within
    the pass; a later success yields a downloaded outcome. (The GET path already
    retried transients; the video path didn't until #8 closed the gap.)"""
    calls = {"n": 0}

    def fake_run(cmd, **kwargs):
        calls["n"] += 1
        if calls["n"] == 1:
            raise pd_mod.subprocess.TimeoutExpired(cmd, kwargs.get("timeout"))
        out = Path(cmd[cmd.index("-o") + 1].replace(".%(ext)s", ".mp4"))
        out.parent.mkdir(parents=True, exist_ok=True)
        out.write_bytes(b"video-bytes")
        return pd_mod.subprocess.CompletedProcess(cmd, 0, "", "")

    monkeypatch.setattr(pd_mod.subprocess, "run", fake_run)
    monkeypatch.setattr(pd_mod.time, "sleep", lambda *_a, **_k: None)

    dl = _downloader(tmp_path, session=_FakeSession())
    outcomes = dl.download_post(_post(), [_video_item()], "artist-x")
    assert calls["n"] == 2  # retried once after the transient blip
    assert outcomes[0].status == "downloaded"
    assert outcomes[0].path.read_bytes() == b"video-bytes"


def test_video_permanent_failure_fails_fast(tmp_path, monkeypatch):
    """plan #705 #8: a non-zero yt-dlp exit (CalledProcessError) is PERMANENT —
    yt-dlp already did its own network retries — so we don't re-spawn; the item
    errors straight to the per-item/dead-letter path."""
    calls = {"n": 0}

    def fake_run(cmd, **kwargs):
        calls["n"] += 1
        raise pd_mod.subprocess.CalledProcessError(1, cmd, stderr="ERROR: gone")

    monkeypatch.setattr(pd_mod.subprocess, "run", fake_run)
    monkeypatch.setattr(pd_mod.time, "sleep", lambda *_a, **_k: None)

    dl = _downloader(tmp_path, session=_FakeSession())
    outcomes = dl.download_post(_post(), [_video_item()], "artist-x")
    assert calls["n"] == 1  # no retry on a permanent failure
    assert outcomes[0].status == "error"


def test_one_failure_isolated(tmp_path):
    class _BoomSession(_FakeSession):
        def get(self, url, stream=False, timeout=None, headers=None):
            self.calls.append(url)
            if "bad" in url:
                raise RuntimeError("network boom")
            return _FakeResponse(_PNG_BYTES)

    dl = _downloader(tmp_path, session=_BoomSession())
    items = [
        _img("good.png", url="https://cdn.patreon.com/good.png"),
        _img("bad.png", url="https://cdn.patreon.com/bad.png"),
    ]
    outcomes = dl.download_post(_post(), items, "artist-x")
    assert outcomes[0].status == "downloaded"
    assert outcomes[1].status == "error"
    assert "boom" in outcomes[1].error


def test_validation_quarantines_corrupt(tmp_path):
    # A .png whose bytes are not a valid PNG → validator fails → "quarantined"
    # outcome (distinct from a download error) carrying the dest path +
    # quarantine move; with validate=False it would pass. plan #704 (#4).
    bad = b"not a real png"
    session = _FakeSession({"https://cdn.patreon.com/corrupt.png": bad})
    dl = _downloader(tmp_path, session=session, validate=True)
    item = _img("corrupt.png", url="https://cdn.patreon.com/corrupt.png")
    outcomes = dl.download_post(_post(), [item], "artist-x")
    assert outcomes[0].status == "quarantined"
    assert outcomes[0].path is not None
    assert "_quarantine" in str(outcomes[0].path)
    post_dir = tmp_path / "artist-x" / "patreon" / "2026-05-01_1001_My Post Title"
    assert not (post_dir / "01_corrupt.png").exists()
    quarantine = tmp_path / "_quarantine" / "artist-x" / "patreon"
    assert quarantine.is_dir()
    assert any(quarantine.iterdir())


def test_quarantine_writes_provenance_sidecar(tmp_path):
    """A1 parity fix: the native path now writes the same `.quarantine.json`
    provenance sidecar gallery-dl writes (it skipped it before the shared
    file_validator.quarantine_file helper)."""
    import json

    bad = b"not a real png"
    url = "https://cdn.patreon.com/corrupt.png"
    session = _FakeSession({url: bad})
    dl = _downloader(tmp_path, session=session, validate=True)
    outcomes = dl.download_post(_post(), [_img("corrupt.png", url=url)], "artist-x")

    dest = outcomes[0].path
    sidecar = dest.with_suffix(dest.suffix + ".quarantine.json")
    assert sidecar.is_file()
    meta = json.loads(sidecar.read_text())
    assert meta["source_url"] == url
    assert meta["platform"] == "patreon"
    assert meta["artist_slug"] == "artist-x"
    assert meta["reason"]


def test_validation_off_keeps_bytes(tmp_path):
    bad = b"not a real png"
    session = _FakeSession({"https://cdn.patreon.com/x.png": bad})
    dl = _downloader(tmp_path, session=session, validate=False)
    item = _img("x.png", url="https://cdn.patreon.com/x.png")
    outcomes = dl.download_post(_post(), [item], "artist-x")
    assert outcomes[0].status == "downloaded"
    assert outcomes[0].path.read_bytes() == bad


def test_sanitize_helper():
    assert _sanitize('a/b') == "a_b"
    assert _sanitize('x<>:"|?*y') == "x_______y"
    assert _sanitize("trail.  ") == "trail"
    assert _sanitize("") == "_"


def test_media_outcome_shape():
    o = MediaOutcome(media=None, status="downloaded", path=Path("/tmp/x"), error=None)
    assert o.status == "downloaded"
    assert o.path == Path("/tmp/x")


# -- pacing + 429 backoff (plan #703) --------------------------------------


def test_rate_limit_paces_each_real_download(tmp_path, monkeypatch):
    slept: list[float] = []
    monkeypatch.setattr(pd_mod.time, "sleep", lambda s: slept.append(s))
    dl = PatreonDownloader(
        images_root=tmp_path, cookies_path=None, validate=False,
        rate_limit=2.0, session=_FakeSession(),
    )
    dl.download_post(_post(), [_img("a.png"), _img("b.png")], "artist-x")
    # One pacing sleep per actual download (two media downloaded).
    assert slept == [2.0, 2.0]


def test_skips_do_not_pace(tmp_path, monkeypatch):
    slept: list[float] = []
    monkeypatch.setattr(pd_mod.time, "sleep", lambda s: slept.append(s))
    dl = PatreonDownloader(
        images_root=tmp_path, cookies_path=None, validate=False,
        rate_limit=2.0, session=_FakeSession(),
    )
    # is_seen → skipped_seen, never reaches the download/pacing path.
    dl.download_post(_post(), [_img("a.png")], "artist-x", is_seen=lambda m: True)
    assert slept == []


def test_download_post_honors_should_stop_mid_post(tmp_path):
    """The time-box is polled BEFORE each media item, so a heavy post can't run
    the whole chunk past its budget (Pocketacer soft-limit, 2026-06-07)."""
    dl = PatreonDownloader(
        images_root=tmp_path, cookies_path=None, validate=False,
        session=_FakeSession(),
    )
    items = [_img("a.png"), _img("b.png"), _img("c.png")]
    polls = {"n": 0}

    def _stop():
        polls["n"] += 1
        return polls["n"] > 1  # allow the first item, stop before the second

    outcomes = dl.download_post(_post(), items, "artist-x", should_stop=_stop)
    assert len(outcomes) == 1  # only the first item ran; the rest re-fetch later
    assert outcomes[0].status == "downloaded"


def test_media_429_retried_then_succeeds(tmp_path, monkeypatch):
    slept: list[float] = []
    monkeypatch.setattr(pd_mod.time, "sleep", lambda s: slept.append(s))
    session = _SeqSession([
        _FakeResponse(b"", status_code=429, headers={"Retry-After": "4"}),
        _FakeResponse(_PNG_BYTES, status_code=200),
    ])
    dl = PatreonDownloader(
        images_root=tmp_path, cookies_path=None, validate=False, session=session,
    )
    outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x")
    assert outcomes[0].status == "downloaded"
    assert 4.0 in slept  # backed off before the retry
    assert len(session.calls) == 2


def _dl(tmp_path, session, monkeypatch):
    monkeypatch.setattr(pd_mod.time, "sleep", lambda s: None)
    return PatreonDownloader(
        images_root=tmp_path, cookies_path=None, validate=False, session=session,
    )


def test_transient_transport_error_retried_within_pass(tmp_path, monkeypatch):
    """plan #705 #8: a connection blip on the GET is retried in-place, not a
    terminal error that waits for the next walk."""
    session = _SeqSession([
        requests.ConnectionError("connection reset"),
        _FakeResponse(_PNG_BYTES, status_code=200),
    ])
    dl = _dl(tmp_path, session, monkeypatch)
    outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x")
    assert outcomes[0].status == "downloaded"
    assert len(session.calls) == 2


def test_5xx_retried_within_pass(tmp_path, monkeypatch):
    session = _SeqSession([
        _FakeResponse(b"", status_code=503),
        _FakeResponse(_PNG_BYTES, status_code=200),
    ])
    dl = _dl(tmp_path, session, monkeypatch)
    outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x")
    assert outcomes[0].status == "downloaded"
    assert len(session.calls) == 2


def test_404_fails_fast_no_retry(tmp_path, monkeypatch):
    """A permanent 4xx (gone/forbidden) is NOT retried — it errors immediately
    (and the ingester's dead-letter ledger takes over across walks)."""
    session = _SeqSession([
        _FakeResponse(b"", status_code=404),
        _FakeResponse(_PNG_BYTES, status_code=200),  # would succeed if it retried
    ])
    dl = _dl(tmp_path, session, monkeypatch)
    outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x")
    assert outcomes[0].status == "error"
    assert len(session.calls) == 1  # no retry on a permanent failure


def test_exhausted_transient_becomes_error(tmp_path, monkeypatch):
    session = _SeqSession([requests.Timeout("t")] * 10)  # always times out
    dl = _dl(tmp_path, session, monkeypatch)
    outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x")
    assert outcomes[0].status == "error"


def test_partial_download_resumes_via_range(tmp_path, monkeypatch):
    """plan #708 B5: a mid-download transport cut resumes from the bytes already
    on disk via a Range request, not a refetch from zero."""
    full = bytes(i % 256 for i in range(1000))
    split = 400

    class _ResumeSession:
        def __init__(self):
            self.headers_seen = []

        def get(self, url, stream=False, timeout=None, headers=None):
            self.headers_seen.append(headers)
            if len(self.headers_seen) == 1:
                return _FlakyResp(full[:split], die=True)  # partial then cut
            start = int(headers["Range"].split("=")[1].rstrip("-"))
            return _FlakyResp(full[start:], status_code=206)

    monkeypatch.setattr(pd_mod.time, "sleep", lambda *_a: None)
    session = _ResumeSession()
    dl = PatreonDownloader(
        images_root=tmp_path, cookies_path=None, validate=False, session=session,
    )
    outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x")
    assert outcomes[0].status == "downloaded"
    assert outcomes[0].path.read_bytes() == full
    assert session.headers_seen[0] is None                      # first GET: no Range
    assert session.headers_seen[1] == {"Range": "bytes=400-"}   # resume from offset


def test_range_ignored_by_server_refetches_clean(tmp_path, monkeypatch):
    """plan #708 B5: if the server ignores Range (200, not 206) on the resume, we
    truncate and start clean — never append into a corrupt double-write."""
    full = b"ABCD" * 250  # 1000 bytes
    split = 400

    class _NoRangeSession:
        def __init__(self):
            self.n = 0

        def get(self, url, stream=False, timeout=None, headers=None):
            self.n += 1
            if self.n == 1:
                return _FlakyResp(full[:split], die=True)
            return _FlakyResp(full, status_code=200)  # ignores Range, whole file

    monkeypatch.setattr(pd_mod.time, "sleep", lambda *_a: None)
    dl = PatreonDownloader(
        images_root=tmp_path, cookies_path=None, validate=False,
        session=_NoRangeSession(),
    )
    outcomes = dl.download_post(_post(), [_img("a.png")], "artist-x")
    assert outcomes[0].status == "downloaded"
    assert outcomes[0].path.read_bytes() == full  # clean, not full[:400] + full


# -- content enrichment (adaptive detail-fetch of an empty feed body) -------


def test_enriches_empty_content_via_fetcher(tmp_path):
    """An empty feed body is backfilled from the detail-fetch seam — once per
    post (memoized on the shared dict) — and lands in the importer sidecar."""
    calls: list[str] = []

    def _fetcher(post_id: str) -> str:
        calls.append(post_id)
        return "<p>full body</p>"

    dl = PatreonDownloader(
        images_root=tmp_path, cookies_path=None, validate=False,
        session=_FakeSession(), content_fetcher=_fetcher,
    )
    post = _post()
    post["attributes"]["content"] = ""  # feed returned an empty body
    dl.download_post(post, [_img("a.png"), _img("b.png")], "artist-x")

    post_dir = tmp_path / "artist-x" / "patreon" / pd_mod._post_dir_name(post)
    sc = find_sidecar(post_dir / "01_a.png")
    assert sc is not None
    assert json.loads(sc.read_text())["content"] == "<p>full body</p>"
    # Fetched ONCE for the post, not once per media item.
    assert calls == ["1001"]


def test_no_enrichment_when_feed_body_present(tmp_path):
    """A non-empty feed body is trusted as-is; the detail endpoint isn't hit."""
    calls: list[str] = []
    dl = PatreonDownloader(
        images_root=tmp_path, cookies_path=None, validate=False,
        session=_FakeSession(),
        content_fetcher=lambda pid: calls.append(pid) or "x",
    )
    dl.download_post(_post(), [_img("a.png")], "artist-x")  # _post body is non-empty
    assert calls == []


# -- write_post_record (media-less / text-only post capture) ----------------


def test_write_post_record_writes_enriched_post_only_sidecar(tmp_path):
    calls: list[str] = []

    def _fetcher(post_id: str) -> str:
        calls.append(post_id)
        return "<p>text post body</p>"

    dl = PatreonDownloader(
        images_root=tmp_path, cookies_path=None, validate=False,
        session=_FakeSession(), content_fetcher=_fetcher,
    )
    post = _post()
    post["attributes"]["content"] = ""  # media-less post, empty feed body
    sc = dl.write_post_record(post, "artist-x")

    assert sc is not None
    assert sc.name == "_post.json"  # can't collide with a media `NN_*.json`
    data = json.loads(sc.read_text())
    assert data["id"] == "1001"
    assert data["content"] == "<p>text post body</p>"
    assert calls == ["1001"]


def test_write_post_record_none_without_post_id(tmp_path):
    dl = PatreonDownloader(
        images_root=tmp_path, cookies_path=None, validate=False,
        session=_FakeSession(),
    )
    assert dl.write_post_record({"id": "", "attributes": {}}, "artist-x") is None