refactor(downloads): DRY the ingester/gallery-dl seam — A1–A4 (plan #707)
Consolidates duplication that owning the native ingester left against the still- live gallery-dl path, and fixes a parity gap the duplication hid. A1 — shared quarantine: extract file_validator.quarantine_file (move to _quarantine/<slug>/<platform> + write the .quarantine.json provenance sidecar). gallery_dl._validate_and_quarantine and patreon_downloader._validate_path both call it. PARITY FIX: the native path now writes the provenance sidecar it previously skipped — threads the media url through for source_url. A2 — make_run_stats(**counts) factory in gallery_dl for the canonical run_stats key set; gallery_dl._compute_run_stats and ingest_core both build through it so the shape can't drift (gallery-dl path gains a benign dead_lettered_count=0). A3 — one safe_ext in utils/paths.py; importer._safe_ext (thin wrapper, kept for the Path call sites + memory pointer) and patreon_client both use it. Closes the double-impl of the URL-encoded-basename ext gotcha. A4 — promote gallery_dl._truncate_log/_extract_errors_warnings to module-level truncate_log/extract_errors_warnings; download_service calls them directly instead of reaching through self.gdl for native-result log shaping. The staticmethods stay as thin delegators for existing callers/tests. Behavior-preserving except the A1 sidecar parity fix. Test: native quarantine writes a .quarantine.json (test_patreon_downloader). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -22,7 +22,7 @@ from datetime import UTC, datetime
|
||||
from enum import StrEnum
|
||||
from pathlib import Path
|
||||
|
||||
from .file_validator import is_validatable, validate_file
|
||||
from .file_validator import is_validatable, quarantine_file, validate_file
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -165,6 +165,39 @@ class DownloadResult:
|
||||
posts_processed: int = 0
|
||||
|
||||
|
||||
def extract_errors_warnings(stderr: str) -> str:
|
||||
"""Keep only the `[error]`/`[warning]` lines from a captured stderr blob.
|
||||
|
||||
A generic download-log helper (module-level so the native-ingester result
|
||||
path can shape its logs without reaching through a GalleryDLService instance).
|
||||
"""
|
||||
if not stderr:
|
||||
return ""
|
||||
kept = [
|
||||
line for line in stderr.splitlines()
|
||||
if "][error]" in line.lower() or "][warning]" in line.lower()
|
||||
]
|
||||
return "\n".join(kept)
|
||||
|
||||
|
||||
def truncate_log(text: str, max_bytes: int = 500_000) -> str:
|
||||
"""Cap a captured log to `max_bytes`, eliding the middle (head + tail kept)."""
|
||||
if not text:
|
||||
return text
|
||||
encoded = text.encode("utf-8")
|
||||
if len(encoded) <= max_bytes:
|
||||
return text
|
||||
half = max_bytes // 2
|
||||
head = encoded[:half].decode("utf-8", errors="ignore")
|
||||
tail = encoded[-half:].decode("utf-8", errors="ignore")
|
||||
head_lines = head.count("\n")
|
||||
tail_lines = tail.count("\n")
|
||||
total_lines = text.count("\n")
|
||||
elided = max(0, total_lines - head_lines - tail_lines)
|
||||
marker = f"\n\n... [{elided} lines elided, {len(encoded) - max_bytes} bytes] ...\n\n"
|
||||
return head + marker + tail
|
||||
|
||||
|
||||
def _summarize_validation_failures(failures: list[dict]) -> str:
|
||||
if not failures:
|
||||
return "Validation failed"
|
||||
@@ -184,6 +217,35 @@ def _summarize_validation_failures(failures: list[dict]) -> str:
|
||||
# no log text to scrape — and gallery-dl platforms never had a cursor.)
|
||||
|
||||
|
||||
def make_run_stats(
|
||||
*,
|
||||
exit_code: int = 0,
|
||||
downloaded_count: int = 0,
|
||||
skipped_count: int = 0,
|
||||
per_item_failures: int = 0,
|
||||
warning_count: int = 0,
|
||||
tier_gated_count: int = 0,
|
||||
quarantined_count: int = 0,
|
||||
dead_lettered_count: int = 0,
|
||||
) -> dict:
|
||||
"""The canonical `run_stats` dict shape, in ONE place.
|
||||
|
||||
Both result producers — gallery-dl's `_compute_run_stats` (log scrape) and the
|
||||
native ingester's per-outcome tally (ingest_core) — build through this so the
|
||||
key set can't drift between backends. Phase 3 + the Logs UI read these keys.
|
||||
"""
|
||||
return {
|
||||
"exit_code": exit_code,
|
||||
"downloaded_count": downloaded_count,
|
||||
"skipped_count": skipped_count,
|
||||
"per_item_failures": per_item_failures,
|
||||
"warning_count": warning_count,
|
||||
"tier_gated_count": tier_gated_count,
|
||||
"quarantined_count": quarantined_count,
|
||||
"dead_lettered_count": dead_lettered_count,
|
||||
}
|
||||
|
||||
|
||||
class GalleryDLService:
|
||||
"""Service for executing gallery-dl downloads."""
|
||||
|
||||
@@ -521,10 +583,6 @@ class GalleryDLService:
|
||||
if not written_paths:
|
||||
return quarantined_relpaths, failures
|
||||
|
||||
quarantine_root = (
|
||||
self.images_root / "_quarantine" / artist_slug / platform
|
||||
)
|
||||
|
||||
for path in written_paths:
|
||||
if not is_validatable(path):
|
||||
continue
|
||||
@@ -535,34 +593,14 @@ class GalleryDLService:
|
||||
continue
|
||||
if result.ok:
|
||||
continue
|
||||
try:
|
||||
quarantine_root.mkdir(parents=True, exist_ok=True)
|
||||
dest = quarantine_root / path.name
|
||||
counter = 1
|
||||
while dest.exists():
|
||||
dest = quarantine_root / f"{path.stem}.{counter}{path.suffix}"
|
||||
counter += 1
|
||||
path.rename(dest)
|
||||
sidecar = dest.with_suffix(dest.suffix + ".quarantine.json")
|
||||
sidecar.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"original_path": str(path),
|
||||
"source_url": url,
|
||||
"artist_slug": artist_slug,
|
||||
"platform": platform,
|
||||
"format": result.format,
|
||||
"reason": result.reason,
|
||||
"size": result.size,
|
||||
"quarantined_at": datetime.now(UTC).isoformat(),
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
except OSError as exc:
|
||||
log.error("Failed to quarantine %s: %s. File left in place.", path, exc)
|
||||
continue
|
||||
|
||||
# Shared move+sidecar (file_validator.quarantine_file) — same impl the
|
||||
# native ingester uses, so the layout + provenance sidecar can't drift.
|
||||
dest = quarantine_file(
|
||||
self.images_root, path, artist_slug, platform,
|
||||
url=url, result=result,
|
||||
)
|
||||
if dest is None:
|
||||
continue # move failed → left in place, not counted
|
||||
log.warning(
|
||||
"Quarantined corrupt file: %s → %s (%s: %s)",
|
||||
path, dest, result.format, result.reason,
|
||||
@@ -600,41 +638,24 @@ class GalleryDLService:
|
||||
if "][warning]" in line.lower() and "not allowed to view post" in line.lower()
|
||||
)
|
||||
|
||||
return {
|
||||
"exit_code": return_code,
|
||||
"downloaded_count": self._count_downloaded_files(stdout),
|
||||
"skipped_count": skipped_stdout + skipped_stderr,
|
||||
"per_item_failures": per_item_failures,
|
||||
"warning_count": warning_count,
|
||||
"tier_gated_count": tier_gated_count,
|
||||
}
|
||||
return make_run_stats(
|
||||
exit_code=return_code,
|
||||
downloaded_count=self._count_downloaded_files(stdout),
|
||||
skipped_count=skipped_stdout + skipped_stderr,
|
||||
per_item_failures=per_item_failures,
|
||||
warning_count=warning_count,
|
||||
tier_gated_count=tier_gated_count,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _extract_errors_warnings(stderr: str) -> str:
|
||||
if not stderr:
|
||||
return ""
|
||||
kept = [
|
||||
line for line in stderr.splitlines()
|
||||
if "][error]" in line.lower() or "][warning]" in line.lower()
|
||||
]
|
||||
return "\n".join(kept)
|
||||
# Thin delegator to the module-level helper (kept for existing callers).
|
||||
return extract_errors_warnings(stderr)
|
||||
|
||||
@staticmethod
|
||||
def _truncate_log(text: str, max_bytes: int = 500_000) -> str:
|
||||
if not text:
|
||||
return text
|
||||
encoded = text.encode("utf-8")
|
||||
if len(encoded) <= max_bytes:
|
||||
return text
|
||||
half = max_bytes // 2
|
||||
head = encoded[:half].decode("utf-8", errors="ignore")
|
||||
tail = encoded[-half:].decode("utf-8", errors="ignore")
|
||||
head_lines = head.count("\n")
|
||||
tail_lines = tail.count("\n")
|
||||
total_lines = text.count("\n")
|
||||
elided = max(0, total_lines - head_lines - tail_lines)
|
||||
marker = f"\n\n... [{elided} lines elided, {len(encoded) - max_bytes} bytes] ...\n\n"
|
||||
return head + marker + tail
|
||||
# Thin delegator to the module-level helper (kept for existing callers).
|
||||
return truncate_log(text, max_bytes)
|
||||
|
||||
async def download(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user