Review of the #1–#9 ingester roadmap found two real-but-small gaps; this closes both. #5 (live posts progress) shipped at per-chunk granularity — _apply_backfill_ lifecycle accumulated DownloadResult.posts_processed AFTER each chunk, so the badge didn't move during a chunk (up to ~14.5 min) and over-counted the re-walked resume page. The plan called for within-chunk live updates. Move ownership of _backfill_posts into the ingester: ingest_core writes a monotonic absolute (posts_base + net-new) via _checkpoint_posts at each page boundary and once at the end, EXCLUDING the resumed page so it no longer inflates across chunks. download_service seeds posts_base from prior chunks and stops touching the key (the lifecycle now carries the ingester's committed value forward). #8 (per-media transient/permanent retry) covered only the plain-GET path (_fetch_to_file); the Mux/video path returned None on any yt-dlp failure with no retry. Give _run_ytdlp the same split: TimeoutExpired/OSError are transient (back off + retry up to _MAX_MEDIA_RETRIES), a non-zero exit (CalledProcessError) is permanent (yt-dlp already did its own network retries) → fail fast to the per-item/dead-letter path. Tests: live-posts absolute + resume-page exclusion + tick-doesn't-persist (test_patreon_ingester); lifecycle-leaves-posts-to-ingester rewrite (test_download_service); video transient-retry + permanent-fail-fast (test_patreon_downloader). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -216,6 +216,57 @@ def test_video_routes_to_ytdlp(tmp_path, monkeypatch):
|
||||
assert find_sidecar(outcomes[0].path) is not None
|
||||
|
||||
|
||||
def _video_item():
|
||||
return MediaItem(
|
||||
url="https://stream.mux.com/abc123.m3u8", filename="clip.mp4",
|
||||
kind="postfile", filehash=None, post_id="1001",
|
||||
)
|
||||
|
||||
|
||||
def test_video_transient_failure_retried_then_succeeds(tmp_path, monkeypatch):
|
||||
"""plan #705 #8: a TRANSIENT yt-dlp failure (TimeoutExpired) is retried within
|
||||
the pass; a later success yields a downloaded outcome. (The GET path already
|
||||
retried transients; the video path didn't until #8 closed the gap.)"""
|
||||
calls = {"n": 0}
|
||||
|
||||
def fake_run(cmd, **kwargs):
|
||||
calls["n"] += 1
|
||||
if calls["n"] == 1:
|
||||
raise pd_mod.subprocess.TimeoutExpired(cmd, kwargs.get("timeout"))
|
||||
out = Path(cmd[cmd.index("-o") + 1].replace(".%(ext)s", ".mp4"))
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_bytes(b"video-bytes")
|
||||
return pd_mod.subprocess.CompletedProcess(cmd, 0, "", "")
|
||||
|
||||
monkeypatch.setattr(pd_mod.subprocess, "run", fake_run)
|
||||
monkeypatch.setattr(pd_mod.time, "sleep", lambda *_a, **_k: None)
|
||||
|
||||
dl = _downloader(tmp_path, session=_FakeSession())
|
||||
outcomes = dl.download_post(_post(), [_video_item()], "artist-x")
|
||||
assert calls["n"] == 2 # retried once after the transient blip
|
||||
assert outcomes[0].status == "downloaded"
|
||||
assert outcomes[0].path.read_bytes() == b"video-bytes"
|
||||
|
||||
|
||||
def test_video_permanent_failure_fails_fast(tmp_path, monkeypatch):
|
||||
"""plan #705 #8: a non-zero yt-dlp exit (CalledProcessError) is PERMANENT —
|
||||
yt-dlp already did its own network retries — so we don't re-spawn; the item
|
||||
errors straight to the per-item/dead-letter path."""
|
||||
calls = {"n": 0}
|
||||
|
||||
def fake_run(cmd, **kwargs):
|
||||
calls["n"] += 1
|
||||
raise pd_mod.subprocess.CalledProcessError(1, cmd, stderr="ERROR: gone")
|
||||
|
||||
monkeypatch.setattr(pd_mod.subprocess, "run", fake_run)
|
||||
monkeypatch.setattr(pd_mod.time, "sleep", lambda *_a, **_k: None)
|
||||
|
||||
dl = _downloader(tmp_path, session=_FakeSession())
|
||||
outcomes = dl.download_post(_post(), [_video_item()], "artist-x")
|
||||
assert calls["n"] == 1 # no retry on a permanent failure
|
||||
assert outcomes[0].status == "error"
|
||||
|
||||
|
||||
def test_one_failure_isolated(tmp_path):
|
||||
class _BoomSession(_FakeSession):
|
||||
def get(self, url, stream=False, timeout=None):
|
||||
|
||||
Reference in New Issue
Block a user