feat(patreon): structured ingester results + quarantine surfacing — #704 step 1
The native ingester faked gallery-dl stdout (`Cursor:` lines, summary) and phase 3 regex-scraped it back — so Patreon run-stats were mostly zero and quarantine stats blank. We own the ingester, so it now RETURNS structured data and phase 3 reads it directly. - DownloadResult gains run_stats/cursor/posts_processed (None/0 on the gallery-dl path, which keeps the text route). - Ingester builds real run_stats from per-media outcome counts, sets the checkpoint cursor structurally (no fake `Cursor:` stdout), and counts posts processed. download_service phase 3 uses dl_result.run_stats when present; the backfill lifecycle + TIMEOUT→PARTIAL block checkpoint dl_result.cursor instead of parse_last_cursor(stdout). - #4 quarantine: PatreonDownloader reports a distinct "quarantined" MediaOutcome (with the _quarantine dest); the ingester surfaces a real files_quarantined + quarantined_paths + run_stats.quarantined_count (was hardcoded 0). Quarantined media isn't written or marked seen. - Cleanup: parse_last_cursor + _CURSOR_RE (and the now-unused `import re`) removed from gallery_dl — the structured cursor replaced the scrape. Tests: ingester result carries real run_stats/cursor/posts_processed + quarantine counts; downloader quarantines an invalid file as "quarantined"; backfill cursor tests pass cursor= structurally; dropped the parse_last_cursor tests. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -124,11 +124,12 @@ def _post_dir_name(post: dict) -> str:
|
||||
class MediaOutcome:
|
||||
"""Per-media result of a download_post pass.
|
||||
|
||||
status is one of: "downloaded", "skipped_seen", "skipped_disk", "error".
|
||||
`path` is the final on-disk path for "downloaded" (the actual yt-dlp output
|
||||
for video), or the path that already existed for "skipped_disk"; None for
|
||||
"skipped_seen" and (usually) "error". `error` carries the failure reason for
|
||||
"error", else None.
|
||||
status is one of: "downloaded", "skipped_seen", "skipped_disk",
|
||||
"quarantined", "error". `path` is the final on-disk path for "downloaded"
|
||||
(the actual yt-dlp output for video), the path that already existed for
|
||||
"skipped_disk", or the _quarantine destination for "quarantined"; None for
|
||||
"skipped_seen" and (usually) "error". `error` carries the failure/validation
|
||||
reason for "error"/"quarantined", else None.
|
||||
"""
|
||||
|
||||
media: object # MediaItem (avoid importing the name for a bare annotation)
|
||||
@@ -253,10 +254,13 @@ class PatreonDownloader:
|
||||
out_path = Path(out_path)
|
||||
else:
|
||||
out_path = self._fetch_get(media.url, media_path)
|
||||
invalid = self._validate_path(out_path, artist_slug)
|
||||
if invalid is not None:
|
||||
reason, quarantine_dest = self._validate_path(out_path, artist_slug)
|
||||
if reason is not None:
|
||||
# Quarantined (corrupt/invalid) — distinct from a download error
|
||||
# so the run can report a real files_quarantined count + paths.
|
||||
return MediaOutcome(
|
||||
media=media, status="error", path=None, error=invalid
|
||||
media=media, status="quarantined",
|
||||
path=quarantine_dest, error=reason,
|
||||
)
|
||||
|
||||
self._write_sidecar(post, out_path)
|
||||
@@ -358,23 +362,29 @@ class PatreonDownloader:
|
||||
|
||||
# -- validation --------------------------------------------------------
|
||||
|
||||
def _validate_path(self, path: Path, artist_slug: str) -> str | None:
|
||||
"""Validate a freshly-written file; quarantine + return reason if bad.
|
||||
def _validate_path(
|
||||
self, path: Path, artist_slug: str
|
||||
) -> tuple[str | None, Path | None]:
|
||||
"""Validate a freshly-written file; quarantine if bad.
|
||||
|
||||
Mirrors gallery_dl._validate_and_quarantine: fail-open for unknown
|
||||
formats, move the corrupt file to _quarantine/<slug>/patreon, and return
|
||||
the failure reason string (None when ok / not validatable / disabled).
|
||||
formats, move the corrupt file to _quarantine/<slug>/patreon. Returns
|
||||
`(reason, quarantine_dest)` when quarantined (dest is the original path if
|
||||
the move itself failed), else `(None, None)` (ok / not validatable /
|
||||
disabled). plan #704: the dest is surfaced so the run reports a real
|
||||
quarantined-paths list instead of a blank count.
|
||||
"""
|
||||
if not self._validate or not is_validatable(path):
|
||||
return None
|
||||
return None, None
|
||||
try:
|
||||
result = validate_file(path)
|
||||
except Exception as exc:
|
||||
log.warning("Validator raised on %s: %s", path, exc)
|
||||
return None
|
||||
return None, None
|
||||
if result.ok:
|
||||
return None
|
||||
return None, None
|
||||
quarantine_root = self.images_root / "_quarantine" / artist_slug / "patreon"
|
||||
dest = path
|
||||
try:
|
||||
quarantine_root.mkdir(parents=True, exist_ok=True)
|
||||
dest = quarantine_root / path.name
|
||||
@@ -385,7 +395,8 @@ class PatreonDownloader:
|
||||
shutil.move(str(path), str(dest))
|
||||
except OSError as exc:
|
||||
log.error("Failed to quarantine %s: %s. File left in place.", path, exc)
|
||||
return result.reason or "validation failed"
|
||||
dest = path
|
||||
return (result.reason or "validation failed"), dest
|
||||
|
||||
# -- sidecar -----------------------------------------------------------
|
||||
|
||||
|
||||
Reference in New Issue
Block a user