FabledCurator/tests/test_patreon_ingester.py

"""PatreonIngester tests (native ingester build step 3).

The HTTP client and the media downloader are stubbed (injected seams), so these
exercise the ingester's own logic — mode behavior, the two-tier skip, cursor
emission, the budget cutoff, and the Postgres seen-ledger — without network or a
real CDN. The ledger is real (a sync sessionmaker bound to the test engine), so
the tier-1 skip and the idempotent mark-seen run against actual rows.
"""

import pytest
from sqlalchemy import func, select
from sqlalchemy.orm import sessionmaker

from backend.app.models import (
    Artist,
    PatreonFailedMedia,
    PatreonSeenMedia,
    Source,
)
from backend.app.services.gallery_dl import ErrorType
from backend.app.services.patreon_client import (
    MediaItem,
    PatreonAPIError,
    PatreonAuthError,
    PatreonDriftError,
)
from backend.app.services.patreon_downloader import MediaOutcome
from backend.app.services.patreon_ingester import PatreonIngester, _ledger_key

pytestmark = pytest.mark.integration


# --- fakes ----------------------------------------------------------------


def _media(post_id, n, *, filehash=None, kind="images"):
    # Deterministic 32-hex-ish key, unique per (post_id, n).
    fh = filehash if filehash is not None else f"{post_id}{n}".encode().hex().ljust(32, "0")[:32]
    return MediaItem(
        url=f"https://cdn.patreon.com/{fh}/{n}.jpg",
        filename=f"{n}.jpg",
        kind=kind,
        filehash=fh,
        post_id=post_id,
    )


class _FakeClient:
    """Stub PatreonClient. `pages` is a list of (page_cursor, [posts]); each post
    is (post_id, [MediaItem]). `raise_on_first` lets a test trip drift."""

    def __init__(self, pages, raise_exc=None):
        self._pages = pages
        self._raise_exc = raise_exc
        self.consumed_posts = 0

    def iter_posts(self, campaign_id, cursor=None):
        if self._raise_exc is not None:
            raise self._raise_exc
        for page_cursor, posts in self._pages:
            for post_id, media in posts:
                self.consumed_posts += 1
                yield {"id": post_id, "_media": media}, {}, page_cursor

    def extract_media(self, post, included_index):
        return post["_media"]


class _FakeDownloader:
    """Stub PatreonDownloader. Honors the injected is_seen (tier-1); media in
    `on_disk` report skipped_disk (tier-2); media in `quarantine` report
    quarantined; everything else downloads."""

    def __init__(self, tmp_path, on_disk=None, quarantine=None, error=None):
        self.tmp_path = tmp_path
        self.on_disk = set(on_disk or ())
        self.quarantine = set(quarantine or ())
        self.error = set(error or ())
        self.download_calls = 0

    def download_post(self, post, media_items, artist_slug, *, is_seen):
        outcomes = []
        for m in media_items:
            if is_seen(m):
                outcomes.append(MediaOutcome(media=m, status="skipped_seen", path=None, error=None))
            elif _ledger_key(m) in self.on_disk:
                p = self.tmp_path / f"{m.post_id}_{m.filename}"
                outcomes.append(MediaOutcome(media=m, status="skipped_disk", path=p, error=None))
            elif _ledger_key(m) in self.error:
                self.download_calls += 1
                outcomes.append(MediaOutcome(media=m, status="error", path=None, error="boom"))
            elif _ledger_key(m) in self.quarantine:
                self.download_calls += 1
                q = self.tmp_path / "_quarantine" / f"{m.post_id}_{m.filename}"
                outcomes.append(MediaOutcome(media=m, status="quarantined", path=q, error="bad image"))
            else:
                self.download_calls += 1
                p = self.tmp_path / f"{m.post_id}_{m.filename}"
                p.write_bytes(b"x")
                outcomes.append(MediaOutcome(media=m, status="downloaded", path=p, error=None))
        return outcomes


@pytest.fixture
async def source_id(db):
    artist = Artist(name="Ingest", slug="ingest")
    db.add(artist)
    await db.flush()
    source = Source(
        artist_id=artist.id, platform="patreon",
        url="https://patreon.com/ingest", enabled=True, config_overrides={},
    )
    db.add(source)
    await db.commit()
    return source.id


def _ingester(sync_engine, tmp_path, client, downloader):
    factory = sessionmaker(sync_engine, expire_on_commit=False)
    return PatreonIngester(
        images_root=tmp_path, cookies_path=None, session_factory=factory,
        client=client, downloader=downloader,
    )


def _count_ledger(sync_engine, source_id):
    factory = sessionmaker(sync_engine, expire_on_commit=False)
    with factory() as s:
        return s.execute(
            select(func.count(PatreonSeenMedia.id)).where(
                PatreonSeenMedia.source_id == source_id
            )
        ).scalar_one()


# --- tick -----------------------------------------------------------------


@pytest.mark.asyncio
async def test_tick_downloads_unseen_and_marks_seen(source_id, sync_engine, tmp_path):
    m1, m2 = _media("p1", 1), _media("p1", 2)
    client = _FakeClient([(None, [("p1", [m1, m2])])])
    downloader = _FakeDownloader(tmp_path)
    ing = _ingester(sync_engine, tmp_path, client, downloader)

    result = ing.run(
        source_id=source_id, campaign_id="c1", artist_slug="ingest",
        url="https://patreon.com/ingest", mode="tick",
    )

    assert result.success is True
    assert result.return_code == 0
    assert result.files_downloaded == 2
    assert len(result.written_paths) == 2
    # plan #704: structured run_stats carry the real counts.
    assert result.run_stats["downloaded_count"] == 2
    assert result.posts_processed == 1
    assert _count_ledger(sync_engine, source_id) == 2


@pytest.mark.asyncio
async def test_quarantined_media_surfaced_in_result(source_id, sync_engine, tmp_path):
    """plan #704 (#4): a media that fails validation is reported as quarantined —
    a real files_quarantined count + paths + run_stats — not a blank 0, and is
    NOT written or marked seen."""
    m1, m2 = _media("p1", 1), _media("p1", 2)
    client = _FakeClient([(None, [("p1", [m1, m2])])])
    downloader = _FakeDownloader(tmp_path, quarantine={_ledger_key(m2)})
    ing = _ingester(sync_engine, tmp_path, client, downloader)

    result = ing.run(
        source_id=source_id, campaign_id="c1", artist_slug="ingest",
        url="https://patreon.com/ingest", mode="tick",
    )
    assert result.files_downloaded == 1          # only m1
    assert result.files_quarantined == 1         # m2
    assert len(result.quarantined_paths) == 1
    assert result.run_stats["quarantined_count"] == 1
    assert result.run_stats["downloaded_count"] == 1
    assert len(result.written_paths) == 1        # quarantined NOT written
    # Quarantined media is NOT marked seen (a fixed file may be re-fetched).
    assert _count_ledger(sync_engine, source_id) == 1


@pytest.mark.asyncio
async def test_tick_skips_seen_via_ledger(source_id, sync_engine, tmp_path):
    m1 = _media("p1", 1)
    # Pre-seed the ledger with m1's key.
    factory = sessionmaker(sync_engine, expire_on_commit=False)
    with factory() as s:
        s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m1), post_id="p1"))
        s.commit()

    client = _FakeClient([(None, [("p1", [m1])])])
    downloader = _FakeDownloader(tmp_path)
    ing = _ingester(sync_engine, tmp_path, client, downloader)

    result = ing.run(
        source_id=source_id, campaign_id="c1", artist_slug="ingest",
        url="https://patreon.com/ingest", mode="tick",
    )
    assert result.files_downloaded == 0
    assert downloader.download_calls == 0


@pytest.mark.asyncio
async def test_tick_early_out_after_threshold(source_id, sync_engine, tmp_path):
    # Three posts, one already-seen media each; threshold 2 → stop before the 3rd.
    seen_media = [_media(f"p{i}", 1) for i in range(1, 4)]
    factory = sessionmaker(sync_engine, expire_on_commit=False)
    with factory() as s:
        for m in seen_media:
            s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m), post_id=m.post_id))
        s.commit()

    pages = [(None, [(m.post_id, [m]) for m in seen_media])]
    client = _FakeClient(pages)
    downloader = _FakeDownloader(tmp_path)
    ing = _ingester(sync_engine, tmp_path, client, downloader)

    result = ing.run(
        source_id=source_id, campaign_id="c1", artist_slug="ingest",
        url="https://patreon.com/ingest", mode="tick", seen_threshold=2,
    )
    assert result.success is True
    # Stopped after the 2nd contiguous seen item — never consumed the 3rd post.
    assert client.consumed_posts == 2


# --- backfill -------------------------------------------------------------


@pytest.mark.asyncio
async def test_backfill_reaches_bottom_signals_complete(source_id, sync_engine, tmp_path):
    pages = [
        (None, [("p1", [_media("p1", 1)])]),
        ("CUR2", [("p2", [_media("p2", 1)])]),
    ]
    client = _FakeClient(pages)
    downloader = _FakeDownloader(tmp_path)
    ing = _ingester(sync_engine, tmp_path, client, downloader)

    result = ing.run(
        source_id=source_id, campaign_id="c1", artist_slug="ingest",
        url="https://patreon.com/ingest", mode="backfill",
    )
    # rc 0 + error_type None is what _apply_backfill_lifecycle reads as "complete".
    assert result.return_code == 0
    assert result.error_type is None
    assert result.files_downloaded == 2
    # The page-2 cursor is carried structurally for checkpointing (plan #704).
    assert result.cursor == "CUR2"
    assert result.posts_processed == 2
    assert result.run_stats["downloaded_count"] == 2


@pytest.mark.asyncio
async def test_backfill_checkpoints_cursor_mid_walk(source_id, sync_engine, tmp_path, db):
    """plan #705 #6: the cursor is persisted to the DB at each page boundary
    DURING the walk (not just at the end via phase 3), so a worker SIGKILL
    mid-chunk resumes near the crash. After the walk the source carries the last
    page's cursor — written by the ingester, not phase 3 (which the unit run
    never reaches)."""
    pages = [
        (None, [("p1", [_media("p1", 1)])]),
        ("CUR2", [("p2", [_media("p2", 1)])]),
        ("CUR3", [("p3", [_media("p3", 1)])]),
    ]
    ing = _ingester(sync_engine, tmp_path, _FakeClient(pages), _FakeDownloader(tmp_path))
    ing.run(
        source_id=source_id, campaign_id="c1", artist_slug="ingest",
        url="https://patreon.com/ingest", mode="backfill",
    )
    co = (await db.execute(
        select(Source.config_overrides).where(Source.id == source_id)
    )).scalar_one()
    assert co.get("_backfill_cursor") == "CUR3"


@pytest.mark.asyncio
async def test_backfill_persists_live_posts_count(source_id, sync_engine, tmp_path, db):
    """plan #704 #5: the ingester writes a monotonic _backfill_posts absolute
    DURING the walk (seeded from prior chunks via posts_base), EXCLUDING the
    re-walked resume page so the count doesn't inflate across chunks."""
    # Resume from CUR2 (a prior chunk left off here): its page is re-walked and
    # must NOT be re-counted. posts_base=4 stands in for the prior chunks.
    pages = [
        ("CUR2", [("p1", [_media("p1", 1)])]),   # the resumed page — not counted
        ("CUR3", [("p2", [_media("p2", 1)])]),   # net-new
        ("CUR4", [("p3", [_media("p3", 1)])]),   # net-new
    ]
    ing = _ingester(sync_engine, tmp_path, _FakeClient(pages), _FakeDownloader(tmp_path))
    ing.run(
        source_id=source_id, campaign_id="c1", artist_slug="ingest",
        url="https://patreon.com/ingest", mode="backfill",
        resume_cursor="CUR2", posts_base=4,
    )
    co = (await db.execute(
        select(Source.config_overrides).where(Source.id == source_id)
    )).scalar_one()
    # 4 prior + 2 net-new (p2, p3); p1 on the resumed CUR2 page is excluded.
    assert co.get("_backfill_posts") == 6


@pytest.mark.asyncio
async def test_tick_does_not_persist_posts(source_id, sync_engine, tmp_path, db):
    """A tick has no resumable backfill state, so it must not write _backfill_posts
    (the badge is a backfill/recovery concept)."""
    ing = _ingester(
        sync_engine, tmp_path,
        _FakeClient([("CUR2", [("p1", [_media("p1", 1)])])]),
        _FakeDownloader(tmp_path),
    )
    ing.run(
        source_id=source_id, campaign_id="c1", artist_slug="ingest",
        url="https://patreon.com/ingest", mode="tick",
    )
    co = (await db.execute(
        select(Source.config_overrides).where(Source.id == source_id)
    )).scalar_one()
    assert "_backfill_posts" not in (co or {})


@pytest.mark.asyncio
async def test_tick_does_not_checkpoint_cursor(source_id, sync_engine, tmp_path, db):
    """A tick has no resumable backfill state, so it must not write a cursor."""
    ing = _ingester(
        sync_engine, tmp_path,
        _FakeClient([("CUR2", [("p1", [_media("p1", 1)])])]),
        _FakeDownloader(tmp_path),
    )
    ing.run(
        source_id=source_id, campaign_id="c1", artist_slug="ingest",
        url="https://patreon.com/ingest", mode="tick",
    )
    co = (await db.execute(
        select(Source.config_overrides).where(Source.id == source_id)
    )).scalar_one()
    assert "_backfill_cursor" not in (co or {})


@pytest.mark.asyncio
async def test_backfill_budget_cut_returns_partial_with_progress(
    source_id, sync_engine, tmp_path, monkeypatch,
):
    # Deterministic clock: start=0, post1 check=10 (ok), post2 check=200 (>budget).
    # run() lives in ingest_core now, so patch the clock there (the same global
    # time module object, but we reference it through the module that uses it).
    import backend.app.services.ingest_core as core
    ticks = iter([0.0, 10.0, 200.0, 250.0])
    last = [0.0]

    def fake_monotonic():
        try:
            last[0] = next(ticks)
        except StopIteration:
            pass
        return last[0]

    monkeypatch.setattr(core.time, "monotonic", fake_monotonic)

    pages = [
        ("CUR1", [("p1", [_media("p1", 1)])]),
        ("CUR2", [("p2", [_media("p2", 1)])]),
    ]
    client = _FakeClient(pages)
    downloader = _FakeDownloader(tmp_path)
    ing = _ingester(sync_engine, tmp_path, client, downloader)

    result = ing.run(
        source_id=source_id, campaign_id="c1", artist_slug="ingest",
        url="https://patreon.com/ingest", mode="backfill",
        resume_cursor=None, time_budget_seconds=100.0,
    )
    assert result.error_type == ErrorType.PARTIAL
    assert result.return_code == -1
    assert result.files_downloaded == 1  # post1 downloaded before the cut
    assert downloader.download_calls == 1  # post2's body never ran
    # The checkpoint is the page we were CUT ON (CUR2 — entered, cursor emitted,
    # then the budget check broke before processing it); the next chunk resumes
    # there. Structured (plan #704), matching the old parse_last_cursor(last).
    assert result.cursor == "CUR2"


# --- recovery -------------------------------------------------------------


@pytest.mark.asyncio
async def test_recovery_bypasses_seen_ledger(source_id, sync_engine, tmp_path):
    m1 = _media("p1", 1)
    factory = sessionmaker(sync_engine, expire_on_commit=False)
    with factory() as s:
        s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m1), post_id="p1"))
        s.commit()

    client = _FakeClient([(None, [("p1", [m1])])])
    downloader = _FakeDownloader(tmp_path)
    ing = _ingester(sync_engine, tmp_path, client, downloader)

    result = ing.run(
        source_id=source_id, campaign_id="c1", artist_slug="ingest",
        url="https://patreon.com/ingest", mode="recovery",
    )
    # Ledger says seen, but recovery bypasses tier-1 → re-downloaded for a fresh
    # pHash evaluation under the current threshold.
    assert result.files_downloaded == 1
    assert downloader.download_calls == 1


@pytest.mark.asyncio
async def test_recovery_tier2_disk_still_skips(source_id, sync_engine, tmp_path):
    m1 = _media("p1", 1)
    client = _FakeClient([(None, [("p1", [m1])])])
    # File still on disk (a kept image) → tier-2 spares it even under recovery.
    downloader = _FakeDownloader(tmp_path, on_disk={_ledger_key(m1)})
    ing = _ingester(sync_engine, tmp_path, client, downloader)

    result = ing.run(
        source_id=source_id, campaign_id="c1", artist_slug="ingest",
        url="https://patreon.com/ingest", mode="recovery",
    )
    assert result.files_downloaded == 0
    assert downloader.download_calls == 0
    # Disk-skip still reconciles the ledger so a later tick skips at tier-1.
    assert _count_ledger(sync_engine, source_id) == 1


# --- dead-letter ledger (plan #705 #7) ------------------------------------


def _count_failed(sync_engine, source_id):
    factory = sessionmaker(sync_engine, expire_on_commit=False)
    with factory() as s:
        return s.execute(
            select(func.count(PatreonFailedMedia.id)).where(
                PatreonFailedMedia.source_id == source_id
            )
        ).scalar_one()


def _seed_failed(sync_engine, source_id, key, attempts):
    factory = sessionmaker(sync_engine, expire_on_commit=False)
    with factory() as s:
        s.add(PatreonFailedMedia(source_id=source_id, filehash=key, attempts=attempts))
        s.commit()


@pytest.mark.asyncio
async def test_repeated_failures_dead_letter_then_skip(source_id, sync_engine, tmp_path):
    """A media that errors DEAD_LETTER_THRESHOLD times is recorded; the next walk
    skips it (no download attempt) and counts it dead-lettered."""
    from backend.app.services.patreon_ingester import DEAD_LETTER_THRESHOLD

    m1 = _media("p1", 1)

    def _run():
        ing = _ingester(
            sync_engine, tmp_path,
            _FakeClient([(None, [("p1", [m1])])]),
            _FakeDownloader(tmp_path, error={_ledger_key(m1)}),
        )
        result = ing.run(
            source_id=source_id, campaign_id="c1", artist_slug="ingest",
            url="https://patreon.com/ingest", mode="tick",
        )
        return result, ing.downloader

    # Fail it up to the threshold — each run attempts the download and errors.
    for _ in range(DEAD_LETTER_THRESHOLD):
        result, dl = _run()
        assert dl.download_calls == 1
    assert _count_failed(sync_engine, source_id) == 1

    # Now attempts == threshold → dead: the next walk skips it without trying.
    result, dl = _run()
    assert dl.download_calls == 0
    assert result.run_stats["dead_lettered_count"] == 1


@pytest.mark.asyncio
async def test_recovery_reattempts_dead_lettered_and_clears_on_success(
    source_id, sync_engine, tmp_path,
):
    """Recovery bypasses the dead-letter ledger (re-attempts dead media); a clean
    download clears the row (it recovered)."""
    from backend.app.services.patreon_ingester import DEAD_LETTER_THRESHOLD

    m1 = _media("p1", 1)
    _seed_failed(sync_engine, source_id, _ledger_key(m1), DEAD_LETTER_THRESHOLD)

    downloader = _FakeDownloader(tmp_path)  # succeeds
    ing = _ingester(sync_engine, tmp_path, _FakeClient([(None, [("p1", [m1])])]), downloader)
    result = ing.run(
        source_id=source_id, campaign_id="c1", artist_slug="ingest",
        url="https://patreon.com/ingest", mode="recovery",
    )
    assert downloader.download_calls == 1   # re-attempted despite being dead
    assert result.files_downloaded == 1
    assert _count_failed(sync_engine, source_id) == 0  # cleared on success


@pytest.mark.asyncio
async def test_clean_download_clears_below_threshold_failure(
    source_id, sync_engine, tmp_path,
):
    """A media with a sub-threshold failure that now downloads cleanly (tick) has
    its dead-letter row cleared."""
    m1 = _media("p1", 1)
    _seed_failed(sync_engine, source_id, _ledger_key(m1), 1)  # not yet dead

    downloader = _FakeDownloader(tmp_path)
    ing = _ingester(sync_engine, tmp_path, _FakeClient([(None, [("p1", [m1])])]), downloader)
    result = ing.run(
        source_id=source_id, campaign_id="c1", artist_slug="ingest",
        url="https://patreon.com/ingest", mode="tick",
    )
    assert result.files_downloaded == 1
    assert _count_failed(sync_engine, source_id) == 0


# --- ledger + drift -------------------------------------------------------


@pytest.mark.asyncio
async def test_mark_seen_is_idempotent(source_id, sync_engine, tmp_path):
    m1 = _media("p1", 1)
    client = _FakeClient([(None, [("p1", [m1])])])
    downloader = _FakeDownloader(tmp_path)
    ing = _ingester(sync_engine, tmp_path, client, downloader)

    # Same media downloaded twice across two runs — the second marks seen again.
    ing._mark_seen(source_id, [(_ledger_key(m1), "p1")])
    ing._mark_seen(source_id, [(_ledger_key(m1), "p1")])
    assert _count_ledger(sync_engine, source_id) == 1


def _run_with_client_error(source_id, sync_engine, tmp_path, exc):
    client = _FakeClient([], raise_exc=exc)
    ing = _ingester(sync_engine, tmp_path, client, _FakeDownloader(tmp_path))
    return ing.run(
        source_id=source_id, campaign_id="c1", artist_slug="ingest",
        url="https://patreon.com/ingest", mode="tick",
    )


@pytest.mark.asyncio
async def test_drift_maps_to_api_drift_and_fails_loud(source_id, sync_engine, tmp_path):
    result = _run_with_client_error(
        source_id, sync_engine, tmp_path,
        PatreonDriftError("missing top-level 'data' key"),
    )
    assert result.success is False
    assert result.error_type == ErrorType.API_DRIFT
    assert "Patreon API changed" in (result.error_message or "")


@pytest.mark.asyncio
async def test_auth_error_maps_to_auth_error(source_id, sync_engine, tmp_path):
    result = _run_with_client_error(
        source_id, sync_engine, tmp_path,
        PatreonAuthError("HTTP 403 — auth rejected", status_code=403),
    )
    assert result.success is False
    assert result.error_type == ErrorType.AUTH_ERROR


@pytest.mark.asyncio
async def test_rate_limit_status_maps_to_rate_limited(source_id, sync_engine, tmp_path):
    result = _run_with_client_error(
        source_id, sync_engine, tmp_path,
        PatreonAPIError("HTTP 429", status_code=429),
    )
    assert result.error_type == ErrorType.RATE_LIMITED


@pytest.mark.asyncio
async def test_not_found_status_maps_to_not_found(source_id, sync_engine, tmp_path):
    result = _run_with_client_error(
        source_id, sync_engine, tmp_path,
        PatreonAPIError("HTTP 404", status_code=404),
    )
    assert result.error_type == ErrorType.NOT_FOUND


@pytest.mark.asyncio
async def test_transport_error_maps_to_network_error(source_id, sync_engine, tmp_path):
    result = _run_with_client_error(
        source_id, sync_engine, tmp_path,
        PatreonAPIError("connection reset"),  # no status_code → transport
    )
    assert result.error_type == ErrorType.NETWORK_ERROR


def test_ledger_key_video_has_no_filehash():
    video = MediaItem(
        url="https://stream.mux.com/abc.m3u8", filename="clip.mp4",
        kind="postfile", filehash=None, post_id="p9",
    )
    assert _ledger_key(video) == "p9:clip.mp4"


# --- verify_patreon_credential (the verify counterpart, plan #697) ---------


@pytest.mark.asyncio
async def test_verify_patreon_credential_unresolvable_is_inconclusive():
    from backend.app.services.patreon_ingester import verify_patreon_credential

    # No campaign id resolvable from a non-Patreon URL → can't test → None.
    ok, msg = await verify_patreon_credential("not-a-patreon-url", None, {})
    assert ok is None
    assert "campaign id" in msg.lower()


@pytest.mark.asyncio
async def test_verify_patreon_credential_delegates_to_client(monkeypatch):
    import backend.app.services.patreon_ingester as mod

    async def _fake_resolve(url, cookies_path, overrides):
        return "123", None
    monkeypatch.setattr(mod, "resolve_campaign_id_for_source", _fake_resolve)
    monkeypatch.setattr(
        mod.PatreonClient, "verify_auth",
        lambda self, campaign_id: (True, f"ok:{campaign_id}"),
    )
    ok, msg = await mod.verify_patreon_credential("https://patreon.com/x", None, {})
    assert ok is True
    assert msg == "ok:123"