Native Patreon ingester + download-engine ownership (plans #697, #703–#708) #72

Merged
bvandeusen merged 26 commits from dev into main 2026-06-06 12:57:31 -04:00
2 changed files with 10 additions and 20 deletions
Showing only changes of commit e42a86d995 - Show all commits
+6 -19
View File
@@ -165,22 +165,6 @@ def test_skip_seen(tmp_path):
assert not (tmp_path / "artist-x").exists()
def test_invalid_file_is_quarantined(tmp_path):
"""plan #704 (#4): a downloaded file that fails validation is moved to
_quarantine and reported as a 'quarantined' outcome carrying that path —
distinct from a download 'error'."""
bad_url = "https://cdn.patreon.com/bad.png"
session = _FakeSession({bad_url: b"this is not a valid PNG"})
dl = _downloader(tmp_path, session=session, validate=True)
outcomes = dl.download_post(
_post(), [_img("bad.png", url=bad_url)], "artist-x"
)
assert outcomes[0].status == "quarantined"
assert outcomes[0].path is not None
assert "_quarantine" in str(outcomes[0].path)
assert outcomes[0].path.is_file()
def test_skip_disk(tmp_path):
dl = _downloader(tmp_path)
post_dir = tmp_path / "artist-x" / "patreon" / "2026-05-01_1001_My Post Title"
@@ -248,14 +232,17 @@ def test_one_failure_isolated(tmp_path):
def test_validation_quarantines_corrupt(tmp_path):
# A .png whose bytes are not a valid PNG → validator fails → error +
# quarantine move; with validate=False it would pass.
# A .png whose bytes are not a valid PNG → validator fails → "quarantined"
# outcome (distinct from a download error) carrying the dest path +
# quarantine move; with validate=False it would pass. plan #704 (#4).
bad = b"not a real png"
session = _FakeSession({"https://cdn.patreon.com/corrupt.png": bad})
dl = _downloader(tmp_path, session=session, validate=True)
item = _img("corrupt.png", url="https://cdn.patreon.com/corrupt.png")
outcomes = dl.download_post(_post(), [item], "artist-x")
assert outcomes[0].status == "error"
assert outcomes[0].status == "quarantined"
assert outcomes[0].path is not None
assert "_quarantine" in str(outcomes[0].path)
post_dir = tmp_path / "artist-x" / "patreon" / "2026-05-01_1001_My Post Title"
assert not (post_dir / "01_corrupt.png").exists()
quarantine = tmp_path / "_quarantine" / "artist-x" / "patreon"
+4 -1
View File
@@ -280,7 +280,10 @@ async def test_backfill_budget_cut_returns_partial_with_progress(
assert result.return_code == -1
assert result.files_downloaded == 1 # post1 downloaded before the cut
assert downloader.download_calls == 1 # post2's body never ran
assert result.cursor == "CUR1" # structured checkpoint (plan #704)
# The checkpoint is the page we were CUT ON (CUR2 — entered, cursor emitted,
# then the budget check broke before processing it); the next chunk resumes
# there. Structured (plan #704), matching the old parse_last_cursor(last).
assert result.cursor == "CUR2"
# --- recovery -------------------------------------------------------------