feat(patreon): structured ingester results + quarantine surfacing — #704 step 1
The native ingester faked gallery-dl stdout (`Cursor:` lines, summary) and phase 3 regex-scraped it back — so Patreon run-stats were mostly zero and quarantine stats blank. We own the ingester, so it now RETURNS structured data and phase 3 reads it directly. - DownloadResult gains run_stats/cursor/posts_processed (None/0 on the gallery-dl path, which keeps the text route). - Ingester builds real run_stats from per-media outcome counts, sets the checkpoint cursor structurally (no fake `Cursor:` stdout), and counts posts processed. download_service phase 3 uses dl_result.run_stats when present; the backfill lifecycle + TIMEOUT→PARTIAL block checkpoint dl_result.cursor instead of parse_last_cursor(stdout). - #4 quarantine: PatreonDownloader reports a distinct "quarantined" MediaOutcome (with the _quarantine dest); the ingester surfaces a real files_quarantined + quarantined_paths + run_stats.quarantined_count (was hardcoded 0). Quarantined media isn't written or marked seen. - Cleanup: parse_last_cursor + _CURSOR_RE (and the now-unused `import re`) removed from gallery_dl — the structured cursor replaced the scrape. Tests: ingester result carries real run_stats/cursor/posts_processed + quarantine counts; downloader quarantines an invalid file as "quarantined"; backfill cursor tests pass cursor= structurally; dropped the parse_last_cursor tests. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -59,7 +59,7 @@ def _make_jpg(path: Path, split: str = "h"):
|
||||
def _make_fake_dl_result(
|
||||
*, success=True, written_paths=None, quarantined_paths=None,
|
||||
files_downloaded=0, error_type=None, error_message=None,
|
||||
stdout="", stderr="",
|
||||
stdout="", stderr="", cursor=None, run_stats=None, posts_processed=0,
|
||||
):
|
||||
return SimpleNamespace(
|
||||
success=success,
|
||||
@@ -78,6 +78,11 @@ def _make_fake_dl_result(
|
||||
duration_seconds=1.23,
|
||||
started_at="2026-05-20T14:00:00+00:00",
|
||||
completed_at="2026-05-20T14:01:00+00:00",
|
||||
# plan #704: native ingester now returns the cursor + run_stats
|
||||
# structurally (no more stderr `Cursor:` scraping).
|
||||
cursor=cursor,
|
||||
run_stats=run_stats,
|
||||
posts_processed=posts_processed,
|
||||
)
|
||||
|
||||
|
||||
@@ -505,7 +510,7 @@ async def test_backfill_chunk_progress_advances_cursor(
|
||||
|
||||
svc, _ = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result(
|
||||
success=False, written_paths=[], files_downloaded=0,
|
||||
stderr="[patreon][debug] Cursor: 03:PAGE2:xyz\n",
|
||||
cursor="03:PAGE2:xyz",
|
||||
))
|
||||
await svc.download_source(source.id)
|
||||
|
||||
@@ -534,7 +539,7 @@ async def test_backfill_state_running_selects_backfill_mode_and_resumes(
|
||||
|
||||
svc, calls = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result(
|
||||
success=False, written_paths=[],
|
||||
stderr="[patreon][debug] Cursor: 03:RESUME2:next\n",
|
||||
cursor="03:RESUME2:next",
|
||||
))
|
||||
await svc.download_source(source.id)
|
||||
|
||||
@@ -587,7 +592,7 @@ async def test_backfill_cap_exhaustion_stalls(
|
||||
|
||||
svc, _ = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result(
|
||||
success=False, written_paths=[],
|
||||
stderr="[patreon][debug] Cursor: 03:MORE:left\n",
|
||||
cursor="03:MORE:left",
|
||||
))
|
||||
await svc.download_source(source.id)
|
||||
|
||||
@@ -623,7 +628,7 @@ async def test_backfill_stuck_guard_stalls_after_two_no_advance(
|
||||
"config_overrides": {"_backfill_state": "running", "_backfill_cursor": "stuck"},
|
||||
"backfill_runs_remaining": 5,
|
||||
}
|
||||
stuck = _make_fake_dl_result(success=False, stderr="Cursor: stuck\n")
|
||||
stuck = _make_fake_dl_result(success=False, cursor="stuck")
|
||||
|
||||
await svc._apply_backfill_lifecycle(ctx, stuck)
|
||||
await db.commit()
|
||||
@@ -661,7 +666,7 @@ async def test_backfill_timeout_chunk_reclassified_to_ok(
|
||||
success=False, files_downloaded=3,
|
||||
error_type=ErrorType.TIMEOUT,
|
||||
error_message="Download timed out",
|
||||
stderr="[patreon][debug] Cursor: 03:NEXT:page\n",
|
||||
cursor="03:NEXT:page",
|
||||
))
|
||||
event_id = await svc.download_source(source.id)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user