diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py new file mode 100644 index 0000000..eb37a2f --- /dev/null +++ b/backend/app/services/gallery_dl.py @@ -0,0 +1,660 @@ +"""Gallery-dl subprocess wrapper. + +Ported from GallerySubscriber (~/Nextcloud/Projects/GallerySubscriber/ +backend/app/services/gallery_dl.py). FC adaptations: + - `artist_slug` replaces GS's `subscription_name` throughout + - Validation uses backend/app/services/file_validator.py + - Config dir under `/.gallery-dl/`, not a separate config_path + - PLATFORM_DEFAULTS scoped to the GS-6 platforms registered in FC-3b +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import subprocess +import sys +import tempfile +import time +from dataclasses import dataclass, field +from datetime import UTC, datetime +from enum import Enum +from pathlib import Path + +from .file_validator import is_validatable, validate_file + +log = logging.getLogger(__name__) + + +class ErrorType(str, Enum): + AUTH_ERROR = "auth_error" + RATE_LIMITED = "rate_limited" + NOT_FOUND = "not_found" + ACCESS_DENIED = "access_denied" + NETWORK_ERROR = "network_error" + NO_NEW_CONTENT = "no_new_content" + TIER_LIMITED = "tier_limited" + TIMEOUT = "timeout" + HTTP_ERROR = "http_error" + UNSUPPORTED_URL = "unsupported_url" + VALIDATION_FAILED = "validation_failed" + UNKNOWN_ERROR = "unknown_error" + + +@dataclass +class SourceConfig: + content_types: list[str] = field(default_factory=lambda: ["all"]) + sleep: float | None = None + sleep_request: float | None = None + directory_pattern: str | None = None + filename_pattern: str | None = None + skip_existing: bool = True + save_metadata: bool = True + timeout: int = 3600 + + @classmethod + def from_dict(cls, data: dict) -> SourceConfig: + return cls( + content_types=data.get("content_types", ["all"]), + sleep=data.get("sleep"), + sleep_request=data.get("sleep_request"), + directory_pattern=data.get("directory_pattern"), + filename_pattern=data.get("filename_pattern"), + skip_existing=data.get("skip_existing", True), + save_metadata=data.get("save_metadata", True), + timeout=data.get("timeout", 3600), + ) + + +@dataclass +class DownloadResult: + success: bool + url: str + artist_slug: str + platform: str + files_downloaded: int = 0 + files_quarantined: int = 0 + quarantined_paths: list[str] = field(default_factory=list) + written_paths: list[str] = field(default_factory=list) + stdout: str = "" + stderr: str = "" + return_code: int = 0 + error_type: ErrorType | None = None + error_message: str | None = None + duration_seconds: float = 0.0 + started_at: str | None = None + completed_at: str | None = None + + +def _summarize_validation_failures(failures: list[dict]) -> str: + if not failures: + return "Validation failed" + reasons: dict[str, int] = {} + for f in failures: + key = f.get("reason") or "unknown" + reasons[key] = reasons.get(key, 0) + 1 + top_reason, top_count = max(reasons.items(), key=lambda kv: kv[1]) + n = len(failures) + if top_count == n: + return f"{n} file{'s' if n != 1 else ''} quarantined: {top_reason}" + return f"{n} files quarantined ({top_count}× {top_reason}, mixed)" + + +class GalleryDLService: + """Service for executing gallery-dl downloads.""" + + AUTH_ERROR_PATTERNS = [ + "unauthorized", "login required", "please login", "must be logged in", + "authentication required", "authenticationerror", "refresh-token", + "session expired", "invalid cookie", "cookies are expired", + "cookies have expired", "not logged in", + ] + RATE_LIMIT_PATTERNS = [ + '" 429 ', "rate limit", "too many requests", "ratelimit", "throttl", + ] + NOT_FOUND_PATTERNS = [ + '" 404 ', "error 404", "status: 404", "status code: 404", + "404 not found", "page not found", "user not found", + "creator not found", "artist not found", + "content no longer available", "has been deleted", + "account deleted", "profile does not exist", + ] + GENERIC_NOT_FOUND_PATTERNS = ["does not exist", "no longer available"] + NETWORK_ERROR_PATTERNS = [ + "timed out", "read timed out", "connect timed out", + "connection timed out", "connection refused", "connection reset", + "network unreachable", "name resolution failed", + "name or service not known", "nodename nor servname provided", + "ssl: certificate_verify_failed", "ssl: wrong_version_number", + "certificate verify failed", "[errno", + ] + ACCESS_DENIED_PATTERNS = [ + '" 403 ', "error 403", "forbidden", "access denied", + "permission denied", "tier required", "pledge required", + ] + + # Per-platform defaults. Lifted from GS — same six platforms FC supports. + PLATFORM_DEFAULTS = { + "patreon": { + "content_types": ["images", "image_large", "attachments", "postfile", "content"], + "directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"], + "filename": "{num:>02}_{filename}.{extension}", + "videos": True, + "embeds": True, + "cursor": True, + }, + "subscribestar": { + "content_types": ["all"], + "directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"], + "filename": "{num:>02}_{filename}.{extension}", + }, + "hentaifoundry": { + "content_types": ["all"], + "directory": [], + "filename": "{category}_{index:>03}_{title[:50]}.{extension}", + "include": "all", + }, + "discord": { + "content_types": ["all"], + "directory": ["{channel[name]}"], + "filename": "{date:%Y%m%d}_{id}_{filename}.{extension}", + "embeds": "all", + "stickers": True, + "reactions": False, + "threads": True, + }, + "pixiv": { + "content_types": ["all"], + "directory": ["{category}"], + "filename": "{id}_{title[:50]}_{num:>02}.{extension}", + "ugoira": True, + }, + "deviantart": { + "content_types": ["all"], + "directory": [], + "filename": "{index:>03}_{title[:50]}.{extension}", + "flat": True, + "original": True, + "mature": True, + "metadata": True, + }, + } + + def __init__( + self, + images_root: Path, + rate_limit: float = 3.0, + validate_files: bool = True, + ): + self.images_root = Path(images_root) + self._rate_limit = rate_limit + self._validate_files = validate_files + self._config_dir = self.images_root / ".gallery-dl" + self._config_dir.mkdir(parents=True, exist_ok=True, mode=0o700) + (self._config_dir / "temp").mkdir(parents=True, exist_ok=True, mode=0o700) + + def _get_default_config(self) -> dict: + config = { + "extractor": { + "base-directory": str(self.images_root), + "archive": str(self._config_dir / "archive.sqlite3"), + "skip": True, + "sleep": self._rate_limit, + "sleep-request": max(0.5, self._rate_limit / 4), + "retries": 3, + "timeout": 30.0, + "verify": True, + "postprocessors": [ + { + "name": "metadata", + "mode": "json", + "directory": ".", + "filename": "{filename}.json", + } + ], + }, + "downloader": { + "part": True, + "part-directory": str(self._config_dir / "temp"), + "retries": 3, + "timeout": 120.0, + }, + "output": {"progress": True}, + } + for platform_name, defaults in self.PLATFORM_DEFAULTS.items(): + config["extractor"][platform_name] = { + key: value for key, value in defaults.items() if key != "content_types" + } + return config + + def _build_config_for_source( + self, + platform: str, + source_config: SourceConfig, + artist_slug: str, + ) -> dict: + config = json.loads(json.dumps(self._get_default_config())) # deep copy + + destination = str(self.images_root / artist_slug / platform) + config["extractor"]["base-directory"] = destination + + if source_config.sleep is not None: + config["extractor"]["sleep"] = source_config.sleep + if source_config.sleep_request is not None: + config["extractor"]["sleep-request"] = source_config.sleep_request + config["extractor"]["skip"] = source_config.skip_existing + + if source_config.save_metadata: + config["extractor"]["postprocessors"] = [ + { + "name": "metadata", + "mode": "json", + "directory": ".", + "filename": "{filename}.json", + } + ] + else: + config["extractor"].pop("postprocessors", None) + + platform_section = config["extractor"].setdefault(platform, {}) + + if platform == "patreon": + if "all" in source_config.content_types: + platform_section["files"] = [ + "images", "image_large", "attachments", "postfile", "content", + ] + else: + platform_section["files"] = source_config.content_types + elif platform == "hentaifoundry": + if "pictures" in source_config.content_types or "all" in source_config.content_types: + platform_section["include"] = "all" + + if source_config.directory_pattern: + platform_section["directory"] = source_config.directory_pattern + if source_config.filename_pattern: + platform_section["filename"] = source_config.filename_pattern + + platform_section["metadata"] = source_config.save_metadata + + return config + + def _categorize_error( + self, return_code: int, stdout: str, stderr: str, + ) -> tuple[ErrorType, str]: + combined = f"{stdout} {stderr}".lower() + + skip_line_count = len([ + line for line in stdout.split("\n") if line.strip().startswith("#") + ]) + skip_text_indicators = ["skipping", "already exists", "archive"] + has_skip_text = any(ind in combined for ind in skip_text_indicators) + + actual_error_indicators = [ + "][error]", "exception", "traceback", + "download failed", "extraction failed", + ] + per_item_patterns = [ + "not allowed to view", "unable to get post", + "failed to extract campaign id", "failed to download", + "cannot import yt-dlp", "cannot import youtube-dl", + ] + all_lines = combined.split("\n") + error_lines = [ + line for line in all_lines + if any(ind in line for ind in actual_error_indicators) + ] + per_item_error_lines = [ + line for line in error_lines + if any(p in line for p in per_item_patterns) + ] + source_level_error_lines = [ + line for line in error_lines + if line not in set(per_item_error_lines) + ] + has_actual_error = bool(source_level_error_lines) + + if per_item_error_lines: + per_item_set = set(per_item_error_lines) + combined = "\n".join(line for line in all_lines if line not in per_item_set) + + if (skip_line_count > 0 or has_skip_text) and not has_actual_error: + return ErrorType.NO_NEW_CONTENT, "No new content to download" + + if return_code == 0 and not stdout.strip(): + return ErrorType.NO_NEW_CONTENT, "No new content to download" + + has_401_error = ( + '" 401 ' in combined + or "401 unauthorized" in combined + or "error 401" in combined + or "status: 401" in combined + or "status code: 401" in combined + ) + if has_401_error or any(p in combined for p in self.AUTH_ERROR_PATTERNS): + return ErrorType.AUTH_ERROR, "Authentication failed — cookies may be expired or invalid" + + if any(p in combined for p in self.RATE_LIMIT_PATTERNS): + return ErrorType.RATE_LIMITED, "Rate limited by server — try increasing sleep time" + + has_specific_404 = any(p in combined for p in self.NOT_FOUND_PATTERNS) + has_generic_with_error = any( + p in combined and "][error]" in combined + for p in self.GENERIC_NOT_FOUND_PATTERNS + ) + if has_specific_404 or has_generic_with_error: + return ErrorType.NOT_FOUND, "URL not found — artist may have changed username or deleted content" + + if any(p in combined for p in self.NETWORK_ERROR_PATTERNS): + return ErrorType.NETWORK_ERROR, "Network error — check internet connection" + + if any(p in combined for p in self.ACCESS_DENIED_PATTERNS): + return ErrorType.ACCESS_DENIED, "Access denied — may need higher subscription tier" + + if "http error" in combined or "httperror" in combined: + return ErrorType.HTTP_ERROR, "HTTP error occurred" + + if "no suitable" in combined or "unsupported" in combined or "no extractor" in combined: + return ErrorType.UNSUPPORTED_URL, "URL format not supported by gallery-dl" + + if return_code in (1, 4) and (skip_line_count > 0 or has_skip_text) and not has_actual_error: + return ErrorType.NO_NEW_CONTENT, "No new content to download" + + if return_code in (1, 4) and not has_actual_error: + tier_gated_lines = [ + line for line in combined.split("\n") + if "][warning]" in line and "not allowed to view post" in line + ] + if tier_gated_lines: + count = len(tier_gated_lines) + return ( + ErrorType.TIER_LIMITED, + f"Subscription tier does not grant access to {count} post{'s' if count != 1 else ''}", + ) + + return ErrorType.UNKNOWN_ERROR, f"Unknown error (return code: {return_code})" + + def _count_downloaded_files(self, stdout: str) -> int: + if not stdout: + return 0 + return sum(1 for line in stdout.splitlines() if line.strip().startswith("/")) + + def _written_paths(self, stdout: str) -> list[Path]: + if not stdout: + return [] + return [ + Path(stripped) + for line in stdout.splitlines() + if (stripped := line.strip()).startswith("/") + ] + + def _validate_and_quarantine( + self, + written_paths: list[Path], + artist_slug: str, + platform: str, + url: str, + ) -> tuple[list[str], list[dict]]: + quarantined_relpaths: list[str] = [] + failures: list[dict] = [] + if not written_paths: + return quarantined_relpaths, failures + + quarantine_root = ( + self.images_root / "_quarantine" / artist_slug / platform + ) + + for path in written_paths: + if not is_validatable(path): + continue + try: + result = validate_file(path) + except Exception as exc: + log.warning("Validator raised on %s: %s", path, exc) + continue + if result.ok: + continue + try: + quarantine_root.mkdir(parents=True, exist_ok=True) + dest = quarantine_root / path.name + counter = 1 + while dest.exists(): + dest = quarantine_root / f"{path.stem}.{counter}{path.suffix}" + counter += 1 + path.rename(dest) + sidecar = dest.with_suffix(dest.suffix + ".quarantine.json") + sidecar.write_text( + json.dumps( + { + "original_path": str(path), + "source_url": url, + "artist_slug": artist_slug, + "platform": platform, + "format": result.format, + "reason": result.reason, + "size": result.size, + "quarantined_at": datetime.now(UTC).isoformat(), + }, + indent=2, + ) + ) + except OSError as exc: + log.error("Failed to quarantine %s: %s. File left in place.", path, exc) + continue + + log.warning( + "Quarantined corrupt file: %s → %s (%s: %s)", + path, dest, result.format, result.reason, + ) + quarantined_relpaths.append(str(dest)) + failures.append( + { + "path": str(path), + "format": result.format, + "reason": result.reason, + "size": result.size, + } + ) + return quarantined_relpaths, failures + + def _compute_run_stats(self, return_code: int, stdout: str, stderr: str) -> dict: + stdout = stdout or "" + stderr = stderr or "" + + skipped_stdout = sum( + 1 for line in stdout.splitlines() if line.strip().startswith("#") + ) + skipped_stderr = sum( + 1 for line in stderr.splitlines() if "] skipping " in line.lower() + ) + per_item_failures = sum( + 1 for line in stderr.splitlines() + if "[download][error]" in line.lower() and "failed to download" in line.lower() + ) + warning_count = sum( + 1 for line in stderr.splitlines() if "][warning]" in line.lower() + ) + tier_gated_count = sum( + 1 for line in stderr.splitlines() + if "][warning]" in line.lower() and "not allowed to view post" in line.lower() + ) + + return { + "exit_code": return_code, + "downloaded_count": self._count_downloaded_files(stdout), + "skipped_count": skipped_stdout + skipped_stderr, + "per_item_failures": per_item_failures, + "warning_count": warning_count, + "tier_gated_count": tier_gated_count, + } + + @staticmethod + def _extract_errors_warnings(stderr: str) -> str: + if not stderr: + return "" + kept = [ + line for line in stderr.splitlines() + if "][error]" in line.lower() or "][warning]" in line.lower() + ] + return "\n".join(kept) + + @staticmethod + def _truncate_log(text: str, max_bytes: int = 500_000) -> str: + if not text: + return text + encoded = text.encode("utf-8") + if len(encoded) <= max_bytes: + return text + half = max_bytes // 2 + head = encoded[:half].decode("utf-8", errors="ignore") + tail = encoded[-half:].decode("utf-8", errors="ignore") + head_lines = head.count("\n") + tail_lines = tail.count("\n") + total_lines = text.count("\n") + elided = max(0, total_lines - head_lines - tail_lines) + marker = f"\n\n... [{elided} lines elided, {len(encoded) - max_bytes} bytes] ...\n\n" + return head + marker + tail + + async def download( + self, + url: str, + artist_slug: str, + platform: str, + source_config: SourceConfig | None = None, + cookies_path: str | None = None, + auth_token: str | None = None, + ) -> DownloadResult: + start_time = time.time() + started_at = datetime.now(UTC).isoformat() + + if source_config is None: + source_config = SourceConfig() + + config = self._build_config_for_source(platform, source_config, artist_slug) + + if cookies_path: + config["extractor"]["cookies"] = cookies_path + if auth_token and platform == "discord": + config["extractor"].setdefault("discord", {}) + config["extractor"]["discord"]["token"] = auth_token + if auth_token and platform == "pixiv": + config["extractor"].setdefault("pixiv", {}) + config["extractor"]["pixiv"]["refresh-token"] = auth_token + + with tempfile.NamedTemporaryFile( + mode="w", suffix=".json", delete=False, dir=str(self._config_dir), + ) as fh: + json.dump(config, fh, indent=2) + temp_config_path = fh.name + + try: + cmd = [ + sys.executable, "-m", "gallery_dl", + "--config", temp_config_path, "--verbose", url, + ] + log.info("Executing gallery-dl for %s/%s: %s", artist_slug, platform, url) + + loop = asyncio.get_running_loop() + proc = await loop.run_in_executor( + None, + lambda: subprocess.run( + cmd, capture_output=True, text=True, + timeout=source_config.timeout, + ), + ) + + duration = time.time() - start_time + completed_at = datetime.now(UTC).isoformat() + + quarantined_paths: list[str] = [] + quarantine_failures: list[dict] = [] + if self._validate_files: + quarantined_paths, quarantine_failures = self._validate_and_quarantine( + written_paths=self._written_paths(proc.stdout), + artist_slug=artist_slug, + platform=platform, + url=url, + ) + + written_paths = [ + str(p) for p in self._written_paths(proc.stdout) + if str(p) not in set(quarantined_paths) + ] + files_written = self._count_downloaded_files(proc.stdout) + files_quarantined = len(quarantined_paths) + files_kept = max(0, files_written - files_quarantined) + + if proc.returncode == 0: + if files_quarantined > 0: + return DownloadResult( + success=False, url=url, artist_slug=artist_slug, platform=platform, + files_downloaded=files_kept, + files_quarantined=files_quarantined, + quarantined_paths=quarantined_paths, + written_paths=written_paths, + stdout=proc.stdout, stderr=proc.stderr, + return_code=proc.returncode, + error_type=ErrorType.VALIDATION_FAILED, + error_message=_summarize_validation_failures(quarantine_failures), + duration_seconds=duration, + started_at=started_at, completed_at=completed_at, + ) + return DownloadResult( + success=True, url=url, artist_slug=artist_slug, platform=platform, + files_downloaded=files_kept, + written_paths=written_paths, + stdout=proc.stdout, stderr=proc.stderr, + return_code=proc.returncode, + duration_seconds=duration, + started_at=started_at, completed_at=completed_at, + ) + + error_type, error_message = self._categorize_error( + proc.returncode, proc.stdout, proc.stderr + ) + success = error_type in (ErrorType.NO_NEW_CONTENT, ErrorType.TIER_LIMITED) + + if files_quarantined > 0: + error_type = ErrorType.VALIDATION_FAILED + error_message = _summarize_validation_failures(quarantine_failures) + success = False + + return DownloadResult( + success=success, url=url, artist_slug=artist_slug, platform=platform, + files_downloaded=files_kept if success else 0, + files_quarantined=files_quarantined, + quarantined_paths=quarantined_paths, + written_paths=written_paths, + stdout=proc.stdout, stderr=proc.stderr, + return_code=proc.returncode, + error_type=error_type, error_message=error_message, + duration_seconds=duration, + started_at=started_at, completed_at=completed_at, + ) + + except subprocess.TimeoutExpired: + duration = time.time() - start_time + log.error("Download timeout for %s/%s after %.1fs", artist_slug, platform, duration) + return DownloadResult( + success=False, url=url, artist_slug=artist_slug, platform=platform, + error_type=ErrorType.TIMEOUT, + error_message=f"Download timed out after {source_config.timeout} seconds", + duration_seconds=duration, + started_at=started_at, + completed_at=datetime.now(UTC).isoformat(), + ) + except Exception as exc: + duration = time.time() - start_time + log.exception("Unexpected error downloading %s/%s: %s", artist_slug, platform, exc) + return DownloadResult( + success=False, url=url, artist_slug=artist_slug, platform=platform, + error_type=ErrorType.UNKNOWN_ERROR, error_message=str(exc), + duration_seconds=duration, + started_at=started_at, + completed_at=datetime.now(UTC).isoformat(), + ) + finally: + try: + Path(temp_config_path).unlink() + except Exception: + pass diff --git a/tests/test_gallery_dl_service.py b/tests/test_gallery_dl_service.py new file mode 100644 index 0000000..f172643 --- /dev/null +++ b/tests/test_gallery_dl_service.py @@ -0,0 +1,151 @@ +"""GalleryDLService tests — subprocess is mocked so no real gallery-dl runs.""" + +from types import SimpleNamespace + +import pytest + +from backend.app.services.gallery_dl import ( + ErrorType, + GalleryDLService, + SourceConfig, +) + + +@pytest.fixture +def gdl(tmp_path): + return GalleryDLService( + images_root=tmp_path / "images", + rate_limit=3.0, + validate_files=False, + ) + + +def _proc(stdout="", stderr="", returncode=0): + return SimpleNamespace(stdout=stdout, stderr=stderr, returncode=returncode) + + +@pytest.mark.asyncio +async def test_download_happy_path(gdl, monkeypatch): + out_path = gdl.images_root / "alice" / "patreon" / "post1" / "img.jpg" + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_bytes(b"\xff\xd8\xff\xe0" + b"\x00" * 32 + b"\xff\xd9") + + monkeypatch.setattr( + "backend.app.services.gallery_dl.subprocess.run", + lambda *a, **k: _proc(stdout=str(out_path) + "\n", returncode=0), + ) + result = await gdl.download( + url="https://patreon.com/alice", + artist_slug="alice", + platform="patreon", + source_config=SourceConfig(), + ) + assert result.success is True + assert result.files_downloaded == 1 + assert result.error_type is None + assert str(out_path) in result.written_paths + + +@pytest.mark.asyncio +async def test_download_no_new_content(gdl, monkeypatch): + monkeypatch.setattr( + "backend.app.services.gallery_dl.subprocess.run", + lambda *a, **k: _proc(stdout="# /images/alice/patreon/old.jpg\n", returncode=0), + ) + result = await gdl.download( + url="https://patreon.com/alice", artist_slug="alice", platform="patreon", + source_config=SourceConfig(), + ) + assert result.success is True + assert result.files_downloaded == 0 + + +@pytest.mark.asyncio +async def test_download_auth_error_categorization(gdl, monkeypatch): + monkeypatch.setattr( + "backend.app.services.gallery_dl.subprocess.run", + lambda *a, **k: _proc( + stdout="", stderr='[patreon][error] "GET /api/foo HTTP/1.1" 401 None\n', + returncode=1, + ), + ) + result = await gdl.download( + url="https://patreon.com/alice", artist_slug="alice", platform="patreon", + source_config=SourceConfig(), + ) + assert result.success is False + assert result.error_type == ErrorType.AUTH_ERROR + + +@pytest.mark.asyncio +async def test_download_timeout(gdl, monkeypatch): + import subprocess as sp + + def _raise(*a, **k): + raise sp.TimeoutExpired(cmd="gallery-dl", timeout=10) + + monkeypatch.setattr("backend.app.services.gallery_dl.subprocess.run", _raise) + result = await gdl.download( + url="https://patreon.com/alice", artist_slug="alice", platform="patreon", + source_config=SourceConfig(timeout=10), + ) + assert result.success is False + assert result.error_type == ErrorType.TIMEOUT + + +@pytest.mark.asyncio +async def test_validation_quarantines_truncated_files(tmp_path, monkeypatch): + gdl = GalleryDLService( + images_root=tmp_path / "images", rate_limit=3.0, validate_files=True, + ) + bad = gdl.images_root / "alice" / "patreon" / "post" / "bad.jpg" + bad.parent.mkdir(parents=True, exist_ok=True) + bad.write_bytes(b"\xff\xd8\xff" + b"\x00" * 100) # SOI but no EOI + + monkeypatch.setattr( + "backend.app.services.gallery_dl.subprocess.run", + lambda *a, **k: _proc(stdout=str(bad) + "\n", returncode=0), + ) + result = await gdl.download( + url="https://patreon.com/alice", artist_slug="alice", platform="patreon", + source_config=SourceConfig(), + ) + assert result.files_quarantined == 1 + assert len(result.quarantined_paths) == 1 + assert not bad.exists() + assert result.error_type == ErrorType.VALIDATION_FAILED + + +def test_compute_run_stats(gdl): + stats = gdl._compute_run_stats( + return_code=0, + stdout="/path/a.jpg\n/path/b.jpg\n# /path/c.jpg\n", + stderr="[patreon][warning] foo\n[patreon][download][error] failed to download /x\n", + ) + assert stats["downloaded_count"] == 2 + assert stats["skipped_count"] == 1 + assert stats["warning_count"] >= 1 + assert stats["per_item_failures"] == 1 + + +def test_extract_errors_warnings(gdl): + stderr = ( + "[patreon][debug] foo\n[patreon][error] bar\n" + "[urllib3][debug] baz\n[patreon][warning] qux\n" + ) + result = gdl._extract_errors_warnings(stderr) + assert "[error]" in result + assert "[warning]" in result + assert "[debug]" not in result + + +def test_truncate_log_caps_large_text(gdl): + huge = "line\n" * 200_000 + result = gdl._truncate_log(huge, max_bytes=10_000) + assert len(result.encode()) < 11_000 + assert "lines elided" in result + + +def test_truncate_log_passes_short_text(gdl): + text = "small\n" + assert gdl._truncate_log(text) == text