feat(patreon): structured ingester results + quarantine surfacing — #704 step 1
The native ingester faked gallery-dl stdout (`Cursor:` lines, summary) and phase 3 regex-scraped it back — so Patreon run-stats were mostly zero and quarantine stats blank. We own the ingester, so it now RETURNS structured data and phase 3 reads it directly. - DownloadResult gains run_stats/cursor/posts_processed (None/0 on the gallery-dl path, which keeps the text route). - Ingester builds real run_stats from per-media outcome counts, sets the checkpoint cursor structurally (no fake `Cursor:` stdout), and counts posts processed. download_service phase 3 uses dl_result.run_stats when present; the backfill lifecycle + TIMEOUT→PARTIAL block checkpoint dl_result.cursor instead of parse_last_cursor(stdout). - #4 quarantine: PatreonDownloader reports a distinct "quarantined" MediaOutcome (with the _quarantine dest); the ingester surfaces a real files_quarantined + quarantined_paths + run_stats.quarantined_count (was hardcoded 0). Quarantined media isn't written or marked seen. - Cleanup: parse_last_cursor + _CURSOR_RE (and the now-unused `import re`) removed from gallery_dl — the structured cursor replaced the scrape. Tests: ingester result carries real run_stats/cursor/posts_processed + quarantine counts; downloader quarantines an invalid file as "quarantined"; backfill cursor tests pass cursor= structurally; dropped the parse_last_cursor tests. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -156,8 +156,12 @@ class PatreonIngester:
|
||||
start = time.monotonic()
|
||||
log_lines: list[str] = []
|
||||
written: list[str] = []
|
||||
quarantined_paths: list[str] = []
|
||||
downloaded = 0
|
||||
errors = 0
|
||||
quarantined = 0
|
||||
skipped_count = 0
|
||||
posts_processed = 0
|
||||
consecutive_seen = 0
|
||||
emitted_cursor: str | None = None
|
||||
reached_bottom = False
|
||||
@@ -168,14 +172,17 @@ class PatreonIngester:
|
||||
*, success: bool, return_code: int,
|
||||
error_type: ErrorType | None, error_message: str | None,
|
||||
) -> DownloadResult:
|
||||
# plan #704: return STRUCTURED data — phase 3 reads run_stats/cursor
|
||||
# directly instead of regex-scraping a reconstructed stdout. stdout
|
||||
# stays a human-readable summary (no fake `Cursor:` lines).
|
||||
return DownloadResult(
|
||||
success=success,
|
||||
url=url,
|
||||
artist_slug=artist_slug,
|
||||
platform="patreon",
|
||||
files_downloaded=downloaded,
|
||||
files_quarantined=0,
|
||||
quarantined_paths=[],
|
||||
files_quarantined=quarantined,
|
||||
quarantined_paths=list(quarantined_paths),
|
||||
written_paths=written,
|
||||
stdout="\n".join(log_lines),
|
||||
stderr="",
|
||||
@@ -183,18 +190,28 @@ class PatreonIngester:
|
||||
error_type=error_type,
|
||||
error_message=error_message,
|
||||
duration_seconds=time.monotonic() - start,
|
||||
cursor=emitted_cursor,
|
||||
posts_processed=posts_processed,
|
||||
run_stats={
|
||||
"exit_code": return_code,
|
||||
"downloaded_count": downloaded,
|
||||
"skipped_count": skipped_count,
|
||||
"per_item_failures": errors,
|
||||
"warning_count": 0,
|
||||
"tier_gated_count": 0,
|
||||
"quarantined_count": quarantined,
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
for post, included, page_cursor in self.client.iter_posts(
|
||||
campaign_id, cursor=resume_cursor
|
||||
):
|
||||
# Checkpoint: emit the cursor that FETCHED this page once, the
|
||||
# moment we START it — so a chunk cut mid-page resumes the page,
|
||||
# not the one after it (matches the gallery-dl cursor semantics
|
||||
# download_service's lifecycle already depends on).
|
||||
# Checkpoint the cursor that FETCHED this page the moment we
|
||||
# START it — so a chunk cut mid-page resumes the page, not the one
|
||||
# after it. Carried as DownloadResult.cursor (plan #704); no fake
|
||||
# `Cursor:` stdout line to regex back out.
|
||||
if page_cursor and page_cursor != emitted_cursor:
|
||||
log_lines.append(f"Cursor: {page_cursor}")
|
||||
emitted_cursor = page_cursor
|
||||
|
||||
# Time-box check at the post boundary (coarse, like a gallery-dl
|
||||
@@ -203,6 +220,7 @@ class PatreonIngester:
|
||||
budget_hit = True
|
||||
break
|
||||
|
||||
posts_processed += 1
|
||||
media = self.client.extract_media(post, included)
|
||||
if not media:
|
||||
continue
|
||||
@@ -236,9 +254,20 @@ class PatreonIngester:
|
||||
# do NOT re-feed it to phase 3 — attach_in_place would see
|
||||
# the duplicate sha256 and unlink the on-disk copy.
|
||||
to_mark.append((key, media_item.post_id))
|
||||
skipped_count += 1
|
||||
consecutive_seen += 1
|
||||
elif outcome.status == "skipped_seen":
|
||||
skipped_count += 1
|
||||
consecutive_seen += 1
|
||||
elif outcome.status == "quarantined":
|
||||
# New content that failed validation (corrupt) — counted
|
||||
# distinctly so the run surfaces a real quarantined total.
|
||||
# Not marked seen (a later walk may re-fetch a fixed file);
|
||||
# it IS new content, so it breaks the run-of-seen.
|
||||
quarantined += 1
|
||||
if outcome.path is not None:
|
||||
quarantined_paths.append(str(outcome.path))
|
||||
consecutive_seen = 0
|
||||
elif outcome.status == "error":
|
||||
errors += 1
|
||||
# An error neither advances nor resets the run-of-seen.
|
||||
@@ -262,9 +291,12 @@ class PatreonIngester:
|
||||
|
||||
if errors:
|
||||
log_lines.append(f"{errors} media item(s) failed")
|
||||
if quarantined:
|
||||
log_lines.append(f"{quarantined} media item(s) quarantined (invalid)")
|
||||
log_lines.append(
|
||||
f"Patreon ingest ({mode}): {downloaded} downloaded, "
|
||||
f"{errors} error(s)"
|
||||
f"{skipped_count} skipped, {quarantined} quarantined, "
|
||||
f"{errors} error(s), {posts_processed} post(s)"
|
||||
+ (", reached end" if reached_bottom else "")
|
||||
+ (", time-boxed" if budget_hit else "")
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user