feat(patreon): dead-letter ledger for permanently-failing media — #705 step 2 (#7)

A media that fails every walk (404'd CDN, deleted post, geo-blocked Mux,
persistently-corrupt bytes) used to re-error forever and re-burn chunks.
New `patreon_failed_media` table (alembic 0038, chains 0037) records
per-media attempts; once attempts reach DEAD_LETTER_THRESHOLD (3) the
ingester skips it on routine tick/backfill walks (tier-1.5, folded into the
seen/skip predicate). Recovery BYPASSES it (the operator's "try everything
again" re-attempts dead media). A clean download clears the row (recovered);
errors/quarantines upsert-increment it. Surfaced as
run_stats.dead_lettered_count.

- New PatreonFailedMedia model + migration; ingester _dead_keys /
  _record_failures (on_conflict increment) / _clear_failures.
- skip = seen | dead (empty in recovery); failures recorded post-fetch on
  short sessions (same pattern as the seen-ledger).

Tests: a media erroring 3× is dead-lettered + skipped (no download attempt);
recovery re-attempts a dead media and clears it on success; a clean download
clears a sub-threshold failure.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

tests/test_patreon_ingester.py

@@ -11,7 +11,12 @@ import pytest
 from sqlalchemy import func, select
 from sqlalchemy.orm import sessionmaker
 
-from backend.app.models import Artist, PatreonSeenMedia, Source
+from backend.app.models import (
+    Artist,
+    PatreonFailedMedia,
+    PatreonSeenMedia,
+    Source,
+)
 from backend.app.services.gallery_dl import ErrorType
 from backend.app.services.patreon_client import (
     MediaItem,
@@ -66,10 +71,11 @@ class _FakeDownloader:
     `on_disk` report skipped_disk (tier-2); media in `quarantine` report
     quarantined; everything else downloads."""
 
-    def __init__(self, tmp_path, on_disk=None, quarantine=None):
+    def __init__(self, tmp_path, on_disk=None, quarantine=None, error=None):
         self.tmp_path = tmp_path
         self.on_disk = set(on_disk or ())
         self.quarantine = set(quarantine or ())
+        self.error = set(error or ())
         self.download_calls = 0
 
     def download_post(self, post, media_items, artist_slug, *, is_seen):
@@ -80,6 +86,9 @@ class _FakeDownloader:
             elif _ledger_key(m) in self.on_disk:
                 p = self.tmp_path / f"{m.post_id}_{m.filename}"
                 outcomes.append(MediaOutcome(media=m, status="skipped_disk", path=p, error=None))
+            elif _ledger_key(m) in self.error:
+                self.download_calls += 1
+                outcomes.append(MediaOutcome(media=m, status="error", path=None, error="boom"))
             elif _ledger_key(m) in self.quarantine:
                 self.download_calls += 1
                 q = self.tmp_path / "_quarantine" / f"{m.post_id}_{m.filename}"
@@ -370,6 +379,99 @@ async def test_recovery_tier2_disk_still_skips(source_id, sync_engine, tmp_path)
     assert _count_ledger(sync_engine, source_id) == 1
 
 
+# --- dead-letter ledger (plan #705 #7) ------------------------------------
+
+
+def _count_failed(sync_engine, source_id):
+    factory = sessionmaker(sync_engine, expire_on_commit=False)
+    with factory() as s:
+        return s.execute(
+            select(func.count(PatreonFailedMedia.id)).where(
+                PatreonFailedMedia.source_id == source_id
+            )
+        ).scalar_one()
+
+
+def _seed_failed(sync_engine, source_id, key, attempts):
+    factory = sessionmaker(sync_engine, expire_on_commit=False)
+    with factory() as s:
+        s.add(PatreonFailedMedia(source_id=source_id, filehash=key, attempts=attempts))
+        s.commit()
+
+
+@pytest.mark.asyncio
+async def test_repeated_failures_dead_letter_then_skip(source_id, sync_engine, tmp_path):
+    """A media that errors DEAD_LETTER_THRESHOLD times is recorded; the next walk
+    skips it (no download attempt) and counts it dead-lettered."""
+    from backend.app.services.patreon_ingester import DEAD_LETTER_THRESHOLD
+
+    m1 = _media("p1", 1)
+
+    def _run():
+        ing = _ingester(
+            sync_engine, tmp_path,
+            _FakeClient([(None, [("p1", [m1])])]),
+            _FakeDownloader(tmp_path, error={_ledger_key(m1)}),
+        )
+        result = ing.run(
+            source_id=source_id, campaign_id="c1", artist_slug="ingest",
+            url="https://patreon.com/ingest", mode="tick",
+        )
+        return result, ing.downloader
+
+    # Fail it up to the threshold — each run attempts the download and errors.
+    for _ in range(DEAD_LETTER_THRESHOLD):
+        result, dl = _run()
+        assert dl.download_calls == 1
+    assert _count_failed(sync_engine, source_id) == 1
+
+    # Now attempts == threshold → dead: the next walk skips it without trying.
+    result, dl = _run()
+    assert dl.download_calls == 0
+    assert result.run_stats["dead_lettered_count"] == 1
+
+
+@pytest.mark.asyncio
+async def test_recovery_reattempts_dead_lettered_and_clears_on_success(
+    source_id, sync_engine, tmp_path,
+):
+    """Recovery bypasses the dead-letter ledger (re-attempts dead media); a clean
+    download clears the row (it recovered)."""
+    from backend.app.services.patreon_ingester import DEAD_LETTER_THRESHOLD
+
+    m1 = _media("p1", 1)
+    _seed_failed(sync_engine, source_id, _ledger_key(m1), DEAD_LETTER_THRESHOLD)
+
+    downloader = _FakeDownloader(tmp_path)  # succeeds
+    ing = _ingester(sync_engine, tmp_path, _FakeClient([(None, [("p1", [m1])])]), downloader)
+    result = ing.run(
+        source_id=source_id, campaign_id="c1", artist_slug="ingest",
+        url="https://patreon.com/ingest", mode="recovery",
+    )
+    assert downloader.download_calls == 1   # re-attempted despite being dead
+    assert result.files_downloaded == 1
+    assert _count_failed(sync_engine, source_id) == 0  # cleared on success
+
+
+@pytest.mark.asyncio
+async def test_clean_download_clears_below_threshold_failure(
+    source_id, sync_engine, tmp_path,
+):
+    """A media with a sub-threshold failure that now downloads cleanly (tick) has
+    its dead-letter row cleared."""
+    m1 = _media("p1", 1)
+    _seed_failed(sync_engine, source_id, _ledger_key(m1), 1)  # not yet dead
+
+    downloader = _FakeDownloader(tmp_path)
+    ing = _ingester(sync_engine, tmp_path, _FakeClient([(None, [("p1", [m1])])]), downloader)
+    result = ing.run(
+        source_id=source_id, campaign_id="c1", artist_slug="ingest",
+        url="https://patreon.com/ingest", mode="tick",
+    )
+    assert result.files_downloaded == 1
+    assert _count_failed(sync_engine, source_id) == 0
+
+
 # --- ledger + drift -------------------------------------------------------