fix(ingester): close #5 within-chunk live posts + #8 video transient retry

Review of the #1–#9 ingester roadmap found two real-but-small gaps; this closes
both.

#5 (live posts progress) shipped at per-chunk granularity — _apply_backfill_
lifecycle accumulated DownloadResult.posts_processed AFTER each chunk, so the
badge didn't move during a chunk (up to ~14.5 min) and over-counted the
re-walked resume page. The plan called for within-chunk live updates. Move
ownership of _backfill_posts into the ingester: ingest_core writes a monotonic
absolute (posts_base + net-new) via _checkpoint_posts at each page boundary and
once at the end, EXCLUDING the resumed page so it no longer inflates across
chunks. download_service seeds posts_base from prior chunks and stops touching
the key (the lifecycle now carries the ingester's committed value forward).

#8 (per-media transient/permanent retry) covered only the plain-GET path
(_fetch_to_file); the Mux/video path returned None on any yt-dlp failure with no
retry. Give _run_ytdlp the same split: TimeoutExpired/OSError are transient
(back off + retry up to _MAX_MEDIA_RETRIES), a non-zero exit (CalledProcessError)
is permanent (yt-dlp already did its own network retries) → fail fast to the
per-item/dead-letter path.

Tests: live-posts absolute + resume-page exclusion + tick-doesn't-persist
(test_patreon_ingester); lifecycle-leaves-posts-to-ingester rewrite
(test_download_service); video transient-retry + permanent-fail-fast
(test_patreon_downloader).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

tests/test_download_service.py

@@ -649,17 +649,18 @@ async def test_backfill_stuck_guard_stalls_after_two_no_advance(
 
 
 @pytest.mark.asyncio
-async def test_backfill_accumulates_posts_processed(
+async def test_backfill_lifecycle_leaves_posts_to_ingester(
     db, db_sync, tmp_path, seed_artist_and_source,
 ):
-    """plan #704 (#5): the lifecycle accumulates DownloadResult.posts_processed
-    into _backfill_posts across chunks for the live progress badge."""
+    """plan #704 (#5), revised: _backfill_posts is OWNED by the ingester now (it
+    writes a monotonic absolute live mid-walk), so the post-chunk lifecycle must
+    NOT touch it — it carries the ingester's committed value forward unchanged."""
     from unittest.mock import MagicMock
 
     from backend.app.services.download_service import DownloadService
 
     _artist, source = seed_artist_and_source
-    source.config_overrides = {"_backfill_state": "running", "_backfill_posts": 5}
+    source.config_overrides = {"_backfill_state": "running", "_backfill_posts": 12}
     source.backfill_runs_remaining = 5
     await db.commit()
 
@@ -669,7 +670,7 @@ async def test_backfill_accumulates_posts_processed(
     )
     ctx = {
         "source_id": source.id, "platform": "patreon",
-        "config_overrides": {"_backfill_state": "running", "_backfill_posts": 5},
+        "config_overrides": {"_backfill_state": "running", "_backfill_posts": 12},
         "backfill_runs_remaining": 5,
     }
     await svc._apply_backfill_lifecycle(
@@ -679,7 +680,7 @@ async def test_backfill_accumulates_posts_processed(
     co = (await db.execute(
         select(Source.config_overrides).where(Source.id == source.id)
     )).scalar_one()
-    assert co.get("_backfill_posts") == 15  # 5 (prior) + 10 (this chunk)
+    assert co.get("_backfill_posts") == 12  # untouched — the ingester owns it
 
 
 @pytest.mark.asyncio

tests/test_patreon_downloader.py

@@ -216,6 +216,57 @@ def test_video_routes_to_ytdlp(tmp_path, monkeypatch):
     assert find_sidecar(outcomes[0].path) is not None
 
 
+def _video_item():
+    return MediaItem(
+        url="https://stream.mux.com/abc123.m3u8", filename="clip.mp4",
+        kind="postfile", filehash=None, post_id="1001",
+    )
+
+
+def test_video_transient_failure_retried_then_succeeds(tmp_path, monkeypatch):
+    """plan #705 #8: a TRANSIENT yt-dlp failure (TimeoutExpired) is retried within
+    the pass; a later success yields a downloaded outcome. (The GET path already
+    retried transients; the video path didn't until #8 closed the gap.)"""
+    calls = {"n": 0}
+
+    def fake_run(cmd, **kwargs):
+        calls["n"] += 1
+        if calls["n"] == 1:
+            raise pd_mod.subprocess.TimeoutExpired(cmd, kwargs.get("timeout"))
+        out = Path(cmd[cmd.index("-o") + 1].replace(".%(ext)s", ".mp4"))
+        out.parent.mkdir(parents=True, exist_ok=True)
+        out.write_bytes(b"video-bytes")
+        return pd_mod.subprocess.CompletedProcess(cmd, 0, "", "")
+
+    monkeypatch.setattr(pd_mod.subprocess, "run", fake_run)
+    monkeypatch.setattr(pd_mod.time, "sleep", lambda *_a, **_k: None)
+
+    dl = _downloader(tmp_path, session=_FakeSession())
+    outcomes = dl.download_post(_post(), [_video_item()], "artist-x")
+    assert calls["n"] == 2  # retried once after the transient blip
+    assert outcomes[0].status == "downloaded"
+    assert outcomes[0].path.read_bytes() == b"video-bytes"
+
+
+def test_video_permanent_failure_fails_fast(tmp_path, monkeypatch):
+    """plan #705 #8: a non-zero yt-dlp exit (CalledProcessError) is PERMANENT —
+    yt-dlp already did its own network retries — so we don't re-spawn; the item
+    errors straight to the per-item/dead-letter path."""
+    calls = {"n": 0}
+
+    def fake_run(cmd, **kwargs):
+        calls["n"] += 1
+        raise pd_mod.subprocess.CalledProcessError(1, cmd, stderr="ERROR: gone")
+
+    monkeypatch.setattr(pd_mod.subprocess, "run", fake_run)
+    monkeypatch.setattr(pd_mod.time, "sleep", lambda *_a, **_k: None)
+
+    dl = _downloader(tmp_path, session=_FakeSession())
+    outcomes = dl.download_post(_post(), [_video_item()], "artist-x")
+    assert calls["n"] == 1  # no retry on a permanent failure
+    assert outcomes[0].status == "error"
+
+
 def test_one_failure_isolated(tmp_path):
     class _BoomSession(_FakeSession):
         def get(self, url, stream=False, timeout=None):

tests/test_patreon_ingester.py

@@ -277,6 +277,50 @@ async def test_backfill_checkpoints_cursor_mid_walk(source_id, sync_engine, tmp_
     assert co.get("_backfill_cursor") == "CUR3"
 
 
+@pytest.mark.asyncio
+async def test_backfill_persists_live_posts_count(source_id, sync_engine, tmp_path, db):
+    """plan #704 #5: the ingester writes a monotonic _backfill_posts absolute
+    DURING the walk (seeded from prior chunks via posts_base), EXCLUDING the
+    re-walked resume page so the count doesn't inflate across chunks."""
+    # Resume from CUR2 (a prior chunk left off here): its page is re-walked and
+    # must NOT be re-counted. posts_base=4 stands in for the prior chunks.
+    pages = [
+        ("CUR2", [("p1", [_media("p1", 1)])]),   # the resumed page — not counted
+        ("CUR3", [("p2", [_media("p2", 1)])]),   # net-new
+        ("CUR4", [("p3", [_media("p3", 1)])]),   # net-new
+    ]
+    ing = _ingester(sync_engine, tmp_path, _FakeClient(pages), _FakeDownloader(tmp_path))
+    ing.run(
+        source_id=source_id, campaign_id="c1", artist_slug="ingest",
+        url="https://patreon.com/ingest", mode="backfill",
+        resume_cursor="CUR2", posts_base=4,
+    )
+    co = (await db.execute(
+        select(Source.config_overrides).where(Source.id == source_id)
+    )).scalar_one()
+    # 4 prior + 2 net-new (p2, p3); p1 on the resumed CUR2 page is excluded.
+    assert co.get("_backfill_posts") == 6
+
+
+@pytest.mark.asyncio
+async def test_tick_does_not_persist_posts(source_id, sync_engine, tmp_path, db):
+    """A tick has no resumable backfill state, so it must not write _backfill_posts
+    (the badge is a backfill/recovery concept)."""
+    ing = _ingester(
+        sync_engine, tmp_path,
+        _FakeClient([("CUR2", [("p1", [_media("p1", 1)])])]),
+        _FakeDownloader(tmp_path),
+    )
+    ing.run(
+        source_id=source_id, campaign_id="c1", artist_slug="ingest",
+        url="https://patreon.com/ingest", mode="tick",
+    )
+    co = (await db.execute(
+        select(Source.config_overrides).where(Source.id == source_id)
+    )).scalar_one()
+    assert "_backfill_posts" not in (co or {})
+
+
 @pytest.mark.asyncio
 async def test_tick_does_not_checkpoint_cursor(source_id, sync_engine, tmp_path, db):
     """A tick has no resumable backfill state, so it must not write a cursor."""