feat(patreon): structured ingester results + quarantine surfacing — #704 step 1
CI / lint (push) Successful in 2s
CI / backend-lint-and-test (push) Failing after 25s
CI / frontend-build (push) Successful in 37s
CI / integration (push) Failing after 3m0s

The native ingester faked gallery-dl stdout (`Cursor:` lines, summary) and
phase 3 regex-scraped it back — so Patreon run-stats were mostly zero and
quarantine stats blank. We own the ingester, so it now RETURNS structured
data and phase 3 reads it directly.

- DownloadResult gains run_stats/cursor/posts_processed (None/0 on the
  gallery-dl path, which keeps the text route).
- Ingester builds real run_stats from per-media outcome counts, sets the
  checkpoint cursor structurally (no fake `Cursor:` stdout), and counts
  posts processed. download_service phase 3 uses dl_result.run_stats when
  present; the backfill lifecycle + TIMEOUT→PARTIAL block checkpoint
  dl_result.cursor instead of parse_last_cursor(stdout).
- #4 quarantine: PatreonDownloader reports a distinct "quarantined"
  MediaOutcome (with the _quarantine dest); the ingester surfaces a real
  files_quarantined + quarantined_paths + run_stats.quarantined_count
  (was hardcoded 0). Quarantined media isn't written or marked seen.
- Cleanup: parse_last_cursor + _CURSOR_RE (and the now-unused `import re`)
  removed from gallery_dl — the structured cursor replaced the scrape.

Tests: ingester result carries real run_stats/cursor/posts_processed +
quarantine counts; downloader quarantines an invalid file as "quarantined";
backfill cursor tests pass cursor= structurally; dropped the
parse_last_cursor tests.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-05 23:39:28 -04:00
parent 5b615b7ded
commit e53f8959af
8 changed files with 162 additions and 86 deletions
+40 -5
View File
@@ -63,11 +63,13 @@ class _FakeClient:
class _FakeDownloader:
"""Stub PatreonDownloader. Honors the injected is_seen (tier-1); media in
`on_disk` report skipped_disk (tier-2); everything else downloads."""
`on_disk` report skipped_disk (tier-2); media in `quarantine` report
quarantined; everything else downloads."""
def __init__(self, tmp_path, on_disk=None):
def __init__(self, tmp_path, on_disk=None, quarantine=None):
self.tmp_path = tmp_path
self.on_disk = set(on_disk or ())
self.quarantine = set(quarantine or ())
self.download_calls = 0
def download_post(self, post, media_items, artist_slug, *, is_seen):
@@ -78,6 +80,10 @@ class _FakeDownloader:
elif _ledger_key(m) in self.on_disk:
p = self.tmp_path / f"{m.post_id}_{m.filename}"
outcomes.append(MediaOutcome(media=m, status="skipped_disk", path=p, error=None))
elif _ledger_key(m) in self.quarantine:
self.download_calls += 1
q = self.tmp_path / "_quarantine" / f"{m.post_id}_{m.filename}"
outcomes.append(MediaOutcome(media=m, status="quarantined", path=q, error="bad image"))
else:
self.download_calls += 1
p = self.tmp_path / f"{m.post_id}_{m.filename}"
@@ -137,9 +143,36 @@ async def test_tick_downloads_unseen_and_marks_seen(source_id, sync_engine, tmp_
assert result.return_code == 0
assert result.files_downloaded == 2
assert len(result.written_paths) == 2
# plan #704: structured run_stats carry the real counts.
assert result.run_stats["downloaded_count"] == 2
assert result.posts_processed == 1
assert _count_ledger(sync_engine, source_id) == 2
@pytest.mark.asyncio
async def test_quarantined_media_surfaced_in_result(source_id, sync_engine, tmp_path):
"""plan #704 (#4): a media that fails validation is reported as quarantined —
a real files_quarantined count + paths + run_stats — not a blank 0, and is
NOT written or marked seen."""
m1, m2 = _media("p1", 1), _media("p1", 2)
client = _FakeClient([(None, [("p1", [m1, m2])])])
downloader = _FakeDownloader(tmp_path, quarantine={_ledger_key(m2)})
ing = _ingester(sync_engine, tmp_path, client, downloader)
result = ing.run(
source_id=source_id, campaign_id="c1", artist_slug="ingest",
url="https://patreon.com/ingest", mode="tick",
)
assert result.files_downloaded == 1 # only m1
assert result.files_quarantined == 1 # m2
assert len(result.quarantined_paths) == 1
assert result.run_stats["quarantined_count"] == 1
assert result.run_stats["downloaded_count"] == 1
assert len(result.written_paths) == 1 # quarantined NOT written
# Quarantined media is NOT marked seen (a fixed file may be re-fetched).
assert _count_ledger(sync_engine, source_id) == 1
@pytest.mark.asyncio
async def test_tick_skips_seen_via_ledger(source_id, sync_engine, tmp_path):
m1 = _media("p1", 1)
@@ -206,8 +239,10 @@ async def test_backfill_reaches_bottom_signals_complete(source_id, sync_engine,
assert result.return_code == 0
assert result.error_type is None
assert result.files_downloaded == 2
# The page-2 cursor was emitted for checkpointing.
assert "Cursor: CUR2" in result.stdout
# The page-2 cursor is carried structurally for checkpointing (plan #704).
assert result.cursor == "CUR2"
assert result.posts_processed == 2
assert result.run_stats["downloaded_count"] == 2
@pytest.mark.asyncio
@@ -245,7 +280,7 @@ async def test_backfill_budget_cut_returns_partial_with_progress(
assert result.return_code == -1
assert result.files_downloaded == 1 # post1 downloaded before the cut
assert downloader.download_calls == 1 # post2's body never ran
assert "Cursor: CUR1" in result.stdout
assert result.cursor == "CUR1" # structured checkpoint (plan #704)
# --- recovery -------------------------------------------------------------