refactor(downloads): DRY the ingester/gallery-dl seam — A1–A4 (plan #707)
CI / lint (push) Successful in 3s
CI / frontend-build (push) Successful in 17s
CI / backend-lint-and-test (push) Successful in 25s
CI / integration (push) Successful in 3m0s

Consolidates duplication that owning the native ingester left against the still-
live gallery-dl path, and fixes a parity gap the duplication hid.

A1 — shared quarantine: extract file_validator.quarantine_file (move to
_quarantine/<slug>/<platform> + write the .quarantine.json provenance sidecar).
gallery_dl._validate_and_quarantine and patreon_downloader._validate_path both
call it. PARITY FIX: the native path now writes the provenance sidecar it
previously skipped — threads the media url through for source_url.

A2 — make_run_stats(**counts) factory in gallery_dl for the canonical run_stats
key set; gallery_dl._compute_run_stats and ingest_core both build through it so
the shape can't drift (gallery-dl path gains a benign dead_lettered_count=0).

A3 — one safe_ext in utils/paths.py; importer._safe_ext (thin wrapper, kept for
the Path call sites + memory pointer) and patreon_client both use it. Closes the
double-impl of the URL-encoded-basename ext gotcha.

A4 — promote gallery_dl._truncate_log/_extract_errors_warnings to module-level
truncate_log/extract_errors_warnings; download_service calls them directly
instead of reaching through self.gdl for native-result log shaping. The
staticmethods stay as thin delegators for existing callers/tests.

Behavior-preserving except the A1 sidecar parity fix. Test: native quarantine
writes a .quarantine.json (test_patreon_downloader).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-06 11:11:25 -04:00
parent 697a86d31c
commit b211900390
9 changed files with 222 additions and 137 deletions
+83 -62
View File
@@ -22,7 +22,7 @@ from datetime import UTC, datetime
from enum import StrEnum
from pathlib import Path
from .file_validator import is_validatable, validate_file
from .file_validator import is_validatable, quarantine_file, validate_file
log = logging.getLogger(__name__)
@@ -165,6 +165,39 @@ class DownloadResult:
posts_processed: int = 0
def extract_errors_warnings(stderr: str) -> str:
"""Keep only the `[error]`/`[warning]` lines from a captured stderr blob.
A generic download-log helper (module-level so the native-ingester result
path can shape its logs without reaching through a GalleryDLService instance).
"""
if not stderr:
return ""
kept = [
line for line in stderr.splitlines()
if "][error]" in line.lower() or "][warning]" in line.lower()
]
return "\n".join(kept)
def truncate_log(text: str, max_bytes: int = 500_000) -> str:
"""Cap a captured log to `max_bytes`, eliding the middle (head + tail kept)."""
if not text:
return text
encoded = text.encode("utf-8")
if len(encoded) <= max_bytes:
return text
half = max_bytes // 2
head = encoded[:half].decode("utf-8", errors="ignore")
tail = encoded[-half:].decode("utf-8", errors="ignore")
head_lines = head.count("\n")
tail_lines = tail.count("\n")
total_lines = text.count("\n")
elided = max(0, total_lines - head_lines - tail_lines)
marker = f"\n\n... [{elided} lines elided, {len(encoded) - max_bytes} bytes] ...\n\n"
return head + marker + tail
def _summarize_validation_failures(failures: list[dict]) -> str:
if not failures:
return "Validation failed"
@@ -184,6 +217,35 @@ def _summarize_validation_failures(failures: list[dict]) -> str:
# no log text to scrape — and gallery-dl platforms never had a cursor.)
def make_run_stats(
*,
exit_code: int = 0,
downloaded_count: int = 0,
skipped_count: int = 0,
per_item_failures: int = 0,
warning_count: int = 0,
tier_gated_count: int = 0,
quarantined_count: int = 0,
dead_lettered_count: int = 0,
) -> dict:
"""The canonical `run_stats` dict shape, in ONE place.
Both result producers — gallery-dl's `_compute_run_stats` (log scrape) and the
native ingester's per-outcome tally (ingest_core) — build through this so the
key set can't drift between backends. Phase 3 + the Logs UI read these keys.
"""
return {
"exit_code": exit_code,
"downloaded_count": downloaded_count,
"skipped_count": skipped_count,
"per_item_failures": per_item_failures,
"warning_count": warning_count,
"tier_gated_count": tier_gated_count,
"quarantined_count": quarantined_count,
"dead_lettered_count": dead_lettered_count,
}
class GalleryDLService:
"""Service for executing gallery-dl downloads."""
@@ -521,10 +583,6 @@ class GalleryDLService:
if not written_paths:
return quarantined_relpaths, failures
quarantine_root = (
self.images_root / "_quarantine" / artist_slug / platform
)
for path in written_paths:
if not is_validatable(path):
continue
@@ -535,34 +593,14 @@ class GalleryDLService:
continue
if result.ok:
continue
try:
quarantine_root.mkdir(parents=True, exist_ok=True)
dest = quarantine_root / path.name
counter = 1
while dest.exists():
dest = quarantine_root / f"{path.stem}.{counter}{path.suffix}"
counter += 1
path.rename(dest)
sidecar = dest.with_suffix(dest.suffix + ".quarantine.json")
sidecar.write_text(
json.dumps(
{
"original_path": str(path),
"source_url": url,
"artist_slug": artist_slug,
"platform": platform,
"format": result.format,
"reason": result.reason,
"size": result.size,
"quarantined_at": datetime.now(UTC).isoformat(),
},
indent=2,
)
)
except OSError as exc:
log.error("Failed to quarantine %s: %s. File left in place.", path, exc)
continue
# Shared move+sidecar (file_validator.quarantine_file) — same impl the
# native ingester uses, so the layout + provenance sidecar can't drift.
dest = quarantine_file(
self.images_root, path, artist_slug, platform,
url=url, result=result,
)
if dest is None:
continue # move failed → left in place, not counted
log.warning(
"Quarantined corrupt file: %s%s (%s: %s)",
path, dest, result.format, result.reason,
@@ -600,41 +638,24 @@ class GalleryDLService:
if "][warning]" in line.lower() and "not allowed to view post" in line.lower()
)
return {
"exit_code": return_code,
"downloaded_count": self._count_downloaded_files(stdout),
"skipped_count": skipped_stdout + skipped_stderr,
"per_item_failures": per_item_failures,
"warning_count": warning_count,
"tier_gated_count": tier_gated_count,
}
return make_run_stats(
exit_code=return_code,
downloaded_count=self._count_downloaded_files(stdout),
skipped_count=skipped_stdout + skipped_stderr,
per_item_failures=per_item_failures,
warning_count=warning_count,
tier_gated_count=tier_gated_count,
)
@staticmethod
def _extract_errors_warnings(stderr: str) -> str:
if not stderr:
return ""
kept = [
line for line in stderr.splitlines()
if "][error]" in line.lower() or "][warning]" in line.lower()
]
return "\n".join(kept)
# Thin delegator to the module-level helper (kept for existing callers).
return extract_errors_warnings(stderr)
@staticmethod
def _truncate_log(text: str, max_bytes: int = 500_000) -> str:
if not text:
return text
encoded = text.encode("utf-8")
if len(encoded) <= max_bytes:
return text
half = max_bytes // 2
head = encoded[:half].decode("utf-8", errors="ignore")
tail = encoded[-half:].decode("utf-8", errors="ignore")
head_lines = head.count("\n")
tail_lines = tail.count("\n")
total_lines = text.count("\n")
elided = max(0, total_lines - head_lines - tail_lines)
marker = f"\n\n... [{elided} lines elided, {len(encoded) - max_bytes} bytes] ...\n\n"
return head + marker + tail
# Thin delegator to the module-level helper (kept for existing callers).
return truncate_log(text, max_bytes)
async def download(
self,