FabledCurator/tests/test_patreon_ingester.py

"""PatreonIngester tests (native ingester build step 3).

The HTTP client and the media downloader are stubbed (injected seams), so these
exercise the ingester's own logic — mode behavior, the two-tier skip, cursor
emission, the budget cutoff, and the Postgres seen-ledger — without network or a
real CDN. The ledger is real (a sync sessionmaker bound to the test engine), so
the tier-1 skip and the idempotent mark-seen run against actual rows.
"""

import pytest
from sqlalchemy import func, select
from sqlalchemy.orm import sessionmaker

from backend.app.models import Artist, PatreonSeenMedia, Source
from backend.app.services.gallery_dl import ErrorType
from backend.app.services.patreon_client import MediaItem, PatreonDriftError
from backend.app.services.patreon_downloader import MediaOutcome
from backend.app.services.patreon_ingester import PatreonIngester, _ledger_key

pytestmark = pytest.mark.integration


# --- fakes ----------------------------------------------------------------


def _media(post_id, n, *, filehash=None, kind="images"):
    # Deterministic 32-hex-ish key, unique per (post_id, n).
    fh = filehash if filehash is not None else f"{post_id}{n}".encode().hex().ljust(32, "0")[:32]
    return MediaItem(
        url=f"https://cdn.patreon.com/{fh}/{n}.jpg",
        filename=f"{n}.jpg",
        kind=kind,
        filehash=fh,
        post_id=post_id,
    )


class _FakeClient:
    """Stub PatreonClient. `pages` is a list of (page_cursor, [posts]); each post
    is (post_id, [MediaItem]). `raise_on_first` lets a test trip drift."""

    def __init__(self, pages, raise_exc=None):
        self._pages = pages
        self._raise_exc = raise_exc
        self.consumed_posts = 0

    def iter_posts(self, campaign_id, cursor=None):
        if self._raise_exc is not None:
            raise self._raise_exc
        for page_cursor, posts in self._pages:
            for post_id, media in posts:
                self.consumed_posts += 1
                yield {"id": post_id, "_media": media}, {}, page_cursor

    def extract_media(self, post, included_index):
        return post["_media"]


class _FakeDownloader:
    """Stub PatreonDownloader. Honors the injected is_seen (tier-1); media in
    `on_disk` report skipped_disk (tier-2); everything else downloads."""

    def __init__(self, tmp_path, on_disk=None):
        self.tmp_path = tmp_path
        self.on_disk = set(on_disk or ())
        self.download_calls = 0

    def download_post(self, post, media_items, artist_slug, *, is_seen):
        outcomes = []
        for m in media_items:
            if is_seen(m):
                outcomes.append(MediaOutcome(media=m, status="skipped_seen", path=None, error=None))
            elif _ledger_key(m) in self.on_disk:
                p = self.tmp_path / f"{m.post_id}_{m.filename}"
                outcomes.append(MediaOutcome(media=m, status="skipped_disk", path=p, error=None))
            else:
                self.download_calls += 1
                p = self.tmp_path / f"{m.post_id}_{m.filename}"
                p.write_bytes(b"x")
                outcomes.append(MediaOutcome(media=m, status="downloaded", path=p, error=None))
        return outcomes


@pytest.fixture
async def source_id(db):
    artist = Artist(name="Ingest", slug="ingest")
    db.add(artist)
    await db.flush()
    source = Source(
        artist_id=artist.id, platform="patreon",
        url="https://patreon.com/ingest", enabled=True, config_overrides={},
    )
    db.add(source)
    await db.commit()
    return source.id


def _ingester(sync_engine, tmp_path, client, downloader):
    factory = sessionmaker(sync_engine, expire_on_commit=False)
    return PatreonIngester(
        images_root=tmp_path, cookies_path=None, session_factory=factory,
        client=client, downloader=downloader,
    )


def _count_ledger(sync_engine, source_id):
    factory = sessionmaker(sync_engine, expire_on_commit=False)
    with factory() as s:
        return s.execute(
            select(func.count(PatreonSeenMedia.id)).where(
                PatreonSeenMedia.source_id == source_id
            )
        ).scalar_one()


# --- tick -----------------------------------------------------------------


@pytest.mark.asyncio
async def test_tick_downloads_unseen_and_marks_seen(source_id, sync_engine, tmp_path):
    m1, m2 = _media("p1", 1), _media("p1", 2)
    client = _FakeClient([(None, [("p1", [m1, m2])])])
    downloader = _FakeDownloader(tmp_path)
    ing = _ingester(sync_engine, tmp_path, client, downloader)

    result = ing.run(
        source_id=source_id, campaign_id="c1", artist_slug="ingest",
        url="https://patreon.com/ingest", mode="tick",
    )

    assert result.success is True
    assert result.return_code == 0
    assert result.files_downloaded == 2
    assert len(result.written_paths) == 2
    assert _count_ledger(sync_engine, source_id) == 2


@pytest.mark.asyncio
async def test_tick_skips_seen_via_ledger(source_id, sync_engine, tmp_path):
    m1 = _media("p1", 1)
    # Pre-seed the ledger with m1's key.
    factory = sessionmaker(sync_engine, expire_on_commit=False)
    with factory() as s:
        s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m1), post_id="p1"))
        s.commit()

    client = _FakeClient([(None, [("p1", [m1])])])
    downloader = _FakeDownloader(tmp_path)
    ing = _ingester(sync_engine, tmp_path, client, downloader)

    result = ing.run(
        source_id=source_id, campaign_id="c1", artist_slug="ingest",
        url="https://patreon.com/ingest", mode="tick",
    )
    assert result.files_downloaded == 0
    assert downloader.download_calls == 0


@pytest.mark.asyncio
async def test_tick_early_out_after_threshold(source_id, sync_engine, tmp_path):
    # Three posts, one already-seen media each; threshold 2 → stop before the 3rd.
    seen_media = [_media(f"p{i}", 1) for i in range(1, 4)]
    factory = sessionmaker(sync_engine, expire_on_commit=False)
    with factory() as s:
        for m in seen_media:
            s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m), post_id=m.post_id))
        s.commit()

    pages = [(None, [(m.post_id, [m]) for m in seen_media])]
    client = _FakeClient(pages)
    downloader = _FakeDownloader(tmp_path)
    ing = _ingester(sync_engine, tmp_path, client, downloader)

    result = ing.run(
        source_id=source_id, campaign_id="c1", artist_slug="ingest",
        url="https://patreon.com/ingest", mode="tick", seen_threshold=2,
    )
    assert result.success is True
    # Stopped after the 2nd contiguous seen item — never consumed the 3rd post.
    assert client.consumed_posts == 2


# --- backfill -------------------------------------------------------------


@pytest.mark.asyncio
async def test_backfill_reaches_bottom_signals_complete(source_id, sync_engine, tmp_path):
    pages = [
        (None, [("p1", [_media("p1", 1)])]),
        ("CUR2", [("p2", [_media("p2", 1)])]),
    ]
    client = _FakeClient(pages)
    downloader = _FakeDownloader(tmp_path)
    ing = _ingester(sync_engine, tmp_path, client, downloader)

    result = ing.run(
        source_id=source_id, campaign_id="c1", artist_slug="ingest",
        url="https://patreon.com/ingest", mode="backfill",
    )
    # rc 0 + error_type None is what _apply_backfill_lifecycle reads as "complete".
    assert result.return_code == 0
    assert result.error_type is None
    assert result.files_downloaded == 2
    # The page-2 cursor was emitted for checkpointing.
    assert "Cursor: CUR2" in result.stdout


@pytest.mark.asyncio
async def test_backfill_budget_cut_returns_partial_with_progress(
    source_id, sync_engine, tmp_path, monkeypatch,
):
    # Deterministic clock: start=0, post1 check=10 (ok), post2 check=200 (>budget).
    import backend.app.services.patreon_ingester as mod
    ticks = iter([0.0, 10.0, 200.0, 250.0])
    last = [0.0]

    def fake_monotonic():
        try:
            last[0] = next(ticks)
        except StopIteration:
            pass
        return last[0]

    monkeypatch.setattr(mod.time, "monotonic", fake_monotonic)

    pages = [
        ("CUR1", [("p1", [_media("p1", 1)])]),
        ("CUR2", [("p2", [_media("p2", 1)])]),
    ]
    client = _FakeClient(pages)
    downloader = _FakeDownloader(tmp_path)
    ing = _ingester(sync_engine, tmp_path, client, downloader)

    result = ing.run(
        source_id=source_id, campaign_id="c1", artist_slug="ingest",
        url="https://patreon.com/ingest", mode="backfill",
        resume_cursor=None, time_budget_seconds=100.0,
    )
    assert result.error_type == ErrorType.PARTIAL
    assert result.return_code == -1
    assert result.files_downloaded == 1  # post1 downloaded before the cut
    assert downloader.download_calls == 1  # post2's body never ran
    assert "Cursor: CUR1" in result.stdout


# --- recovery -------------------------------------------------------------


@pytest.mark.asyncio
async def test_recovery_bypasses_seen_ledger(source_id, sync_engine, tmp_path):
    m1 = _media("p1", 1)
    factory = sessionmaker(sync_engine, expire_on_commit=False)
    with factory() as s:
        s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m1), post_id="p1"))
        s.commit()

    client = _FakeClient([(None, [("p1", [m1])])])
    downloader = _FakeDownloader(tmp_path)
    ing = _ingester(sync_engine, tmp_path, client, downloader)

    result = ing.run(
        source_id=source_id, campaign_id="c1", artist_slug="ingest",
        url="https://patreon.com/ingest", mode="recovery",
    )
    # Ledger says seen, but recovery bypasses tier-1 → re-downloaded for a fresh
    # pHash evaluation under the current threshold.
    assert result.files_downloaded == 1
    assert downloader.download_calls == 1


@pytest.mark.asyncio
async def test_recovery_tier2_disk_still_skips(source_id, sync_engine, tmp_path):
    m1 = _media("p1", 1)
    client = _FakeClient([(None, [("p1", [m1])])])
    # File still on disk (a kept image) → tier-2 spares it even under recovery.
    downloader = _FakeDownloader(tmp_path, on_disk={_ledger_key(m1)})
    ing = _ingester(sync_engine, tmp_path, client, downloader)

    result = ing.run(
        source_id=source_id, campaign_id="c1", artist_slug="ingest",
        url="https://patreon.com/ingest", mode="recovery",
    )
    assert result.files_downloaded == 0
    assert downloader.download_calls == 0
    # Disk-skip still reconciles the ledger so a later tick skips at tier-1.
    assert _count_ledger(sync_engine, source_id) == 1


# --- ledger + drift -------------------------------------------------------


@pytest.mark.asyncio
async def test_mark_seen_is_idempotent(source_id, sync_engine, tmp_path):
    m1 = _media("p1", 1)
    client = _FakeClient([(None, [("p1", [m1])])])
    downloader = _FakeDownloader(tmp_path)
    ing = _ingester(sync_engine, tmp_path, client, downloader)

    # Same media downloaded twice across two runs — the second marks seen again.
    ing._mark_seen(source_id, [(_ledger_key(m1), "p1")])
    ing._mark_seen(source_id, [(_ledger_key(m1), "p1")])
    assert _count_ledger(sync_engine, source_id) == 1


@pytest.mark.asyncio
async def test_drift_fails_loud(source_id, sync_engine, tmp_path):
    client = _FakeClient([], raise_exc=PatreonDriftError("missing top-level 'data' key"))
    downloader = _FakeDownloader(tmp_path)
    ing = _ingester(sync_engine, tmp_path, client, downloader)

    result = ing.run(
        source_id=source_id, campaign_id="c1", artist_slug="ingest",
        url="https://patreon.com/ingest", mode="tick",
    )
    assert result.success is False
    assert "Patreon API changed" in (result.error_message or "")


def test_ledger_key_video_has_no_filehash():
    video = MediaItem(
        url="https://stream.mux.com/abc.m3u8", filename="clip.mp4",
        kind="postfile", filehash=None, post_id="p9",
    )
    assert _ledger_key(video) == "p9:clip.mp4"