"""Gallery-dl subprocess wrapper. Ported from GallerySubscriber (~/Nextcloud/Projects/GallerySubscriber/ backend/app/services/gallery_dl.py). FC adaptations: - `artist_slug` replaces GS's `subscription_name` throughout - Validation uses backend/app/services/file_validator.py - Config dir under `/.gallery-dl/`, not a separate config_path - PLATFORM_DEFAULTS scoped to the GS-6 platforms registered in FC-3b """ from __future__ import annotations import asyncio import json import logging import subprocess import sys import tempfile import time from dataclasses import dataclass, field from datetime import UTC, datetime from enum import StrEnum from pathlib import Path from .file_validator import is_validatable, validate_file log = logging.getLogger(__name__) class ErrorType(StrEnum): AUTH_ERROR = "auth_error" RATE_LIMITED = "rate_limited" NOT_FOUND = "not_found" ACCESS_DENIED = "access_denied" NETWORK_ERROR = "network_error" NO_NEW_CONTENT = "no_new_content" TIER_LIMITED = "tier_limited" TIMEOUT = "timeout" HTTP_ERROR = "http_error" UNSUPPORTED_URL = "unsupported_url" VALIDATION_FAILED = "validation_failed" UNKNOWN_ERROR = "unknown_error" @dataclass class SourceConfig: content_types: list[str] = field(default_factory=lambda: ["all"]) sleep: float | None = None sleep_request: float | None = None directory_pattern: str | None = None filename_pattern: str | None = None skip_existing: bool = True save_metadata: bool = True timeout: int = 3600 @classmethod def from_dict(cls, data: dict) -> SourceConfig: return cls( content_types=data.get("content_types", ["all"]), sleep=data.get("sleep"), sleep_request=data.get("sleep_request"), directory_pattern=data.get("directory_pattern"), filename_pattern=data.get("filename_pattern"), skip_existing=data.get("skip_existing", True), save_metadata=data.get("save_metadata", True), timeout=data.get("timeout", 3600), ) @dataclass class DownloadResult: success: bool url: str artist_slug: str platform: str files_downloaded: int = 0 files_quarantined: int = 0 quarantined_paths: list[str] = field(default_factory=list) written_paths: list[str] = field(default_factory=list) stdout: str = "" stderr: str = "" return_code: int = 0 error_type: ErrorType | None = None error_message: str | None = None duration_seconds: float = 0.0 started_at: str | None = None completed_at: str | None = None def _summarize_validation_failures(failures: list[dict]) -> str: if not failures: return "Validation failed" reasons: dict[str, int] = {} for f in failures: key = f.get("reason") or "unknown" reasons[key] = reasons.get(key, 0) + 1 top_reason, top_count = max(reasons.items(), key=lambda kv: kv[1]) n = len(failures) if top_count == n: return f"{n} file{'s' if n != 1 else ''} quarantined: {top_reason}" return f"{n} files quarantined ({top_count}× {top_reason}, mixed)" class GalleryDLService: """Service for executing gallery-dl downloads.""" AUTH_ERROR_PATTERNS = [ "unauthorized", "login required", "please login", "must be logged in", "authentication required", "authenticationerror", "refresh-token", "session expired", "invalid cookie", "cookies are expired", "cookies have expired", "not logged in", ] RATE_LIMIT_PATTERNS = [ '" 429 ', "rate limit", "too many requests", "ratelimit", "throttl", ] NOT_FOUND_PATTERNS = [ '" 404 ', "error 404", "status: 404", "status code: 404", "404 not found", "page not found", "user not found", "creator not found", "artist not found", "content no longer available", "has been deleted", "account deleted", "profile does not exist", ] GENERIC_NOT_FOUND_PATTERNS = ["does not exist", "no longer available"] NETWORK_ERROR_PATTERNS = [ "timed out", "read timed out", "connect timed out", "connection timed out", "connection refused", "connection reset", "network unreachable", "name resolution failed", "name or service not known", "nodename nor servname provided", "ssl: certificate_verify_failed", "ssl: wrong_version_number", "certificate verify failed", "[errno", ] ACCESS_DENIED_PATTERNS = [ '" 403 ', "error 403", "forbidden", "access denied", "permission denied", "tier required", "pledge required", ] # Per-platform defaults. Lifted from GS — same six platforms FC supports. PLATFORM_DEFAULTS = { "patreon": { "content_types": ["images", "image_large", "attachments", "postfile", "content"], "directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"], "filename": "{num:>02}_{filename}.{extension}", "videos": True, "embeds": True, "cursor": True, }, "subscribestar": { "content_types": ["all"], "directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"], "filename": "{num:>02}_{filename}.{extension}", }, "hentaifoundry": { "content_types": ["all"], "directory": [], "filename": "{category}_{index:>03}_{title[:50]}.{extension}", "include": "all", }, "discord": { "content_types": ["all"], "directory": ["{channel[name]}"], "filename": "{date:%Y%m%d}_{id}_{filename}.{extension}", "embeds": "all", "stickers": True, "reactions": False, "threads": True, }, "pixiv": { "content_types": ["all"], "directory": ["{category}"], "filename": "{id}_{title[:50]}_{num:>02}.{extension}", "ugoira": True, }, "deviantart": { "content_types": ["all"], "directory": [], "filename": "{index:>03}_{title[:50]}.{extension}", "flat": True, "original": True, "mature": True, "metadata": True, }, } def __init__( self, images_root: Path, rate_limit: float = 3.0, validate_files: bool = True, ): self.images_root = Path(images_root) self._rate_limit = rate_limit self._validate_files = validate_files self._config_dir = self.images_root / ".gallery-dl" self._config_dir.mkdir(parents=True, exist_ok=True, mode=0o700) (self._config_dir / "temp").mkdir(parents=True, exist_ok=True, mode=0o700) def _get_default_config(self) -> dict: config = { "extractor": { "base-directory": str(self.images_root), "archive": str(self._config_dir / "archive.sqlite3"), "skip": True, "sleep": self._rate_limit, "sleep-request": max(0.5, self._rate_limit / 4), "retries": 3, "timeout": 30.0, "verify": True, "postprocessors": [ { "name": "metadata", "mode": "json", "directory": ".", "filename": "{filename}.json", } ], }, "downloader": { "part": True, "part-directory": str(self._config_dir / "temp"), "retries": 3, "timeout": 120.0, }, "output": {"progress": True}, } for platform_name, defaults in self.PLATFORM_DEFAULTS.items(): config["extractor"][platform_name] = { key: value for key, value in defaults.items() if key != "content_types" } return config def _build_config_for_source( self, platform: str, source_config: SourceConfig, artist_slug: str, ) -> dict: config = json.loads(json.dumps(self._get_default_config())) # deep copy destination = str(self.images_root / artist_slug / platform) config["extractor"]["base-directory"] = destination if source_config.sleep is not None: config["extractor"]["sleep"] = source_config.sleep if source_config.sleep_request is not None: config["extractor"]["sleep-request"] = source_config.sleep_request config["extractor"]["skip"] = source_config.skip_existing if source_config.save_metadata: config["extractor"]["postprocessors"] = [ { "name": "metadata", "mode": "json", "directory": ".", "filename": "{filename}.json", } ] else: config["extractor"].pop("postprocessors", None) platform_section = config["extractor"].setdefault(platform, {}) if platform == "patreon": if "all" in source_config.content_types: platform_section["files"] = [ "images", "image_large", "attachments", "postfile", "content", ] else: platform_section["files"] = source_config.content_types elif platform == "hentaifoundry": if "pictures" in source_config.content_types or "all" in source_config.content_types: platform_section["include"] = "all" if source_config.directory_pattern: platform_section["directory"] = source_config.directory_pattern if source_config.filename_pattern: platform_section["filename"] = source_config.filename_pattern platform_section["metadata"] = source_config.save_metadata return config def _categorize_error( self, return_code: int, stdout: str, stderr: str, ) -> tuple[ErrorType, str]: combined = f"{stdout} {stderr}".lower() skip_line_count = len([ line for line in stdout.split("\n") if line.strip().startswith("#") ]) skip_text_indicators = ["skipping", "already exists", "archive"] has_skip_text = any(ind in combined for ind in skip_text_indicators) actual_error_indicators = [ "][error]", "exception", "traceback", "download failed", "extraction failed", ] per_item_patterns = [ "not allowed to view", "unable to get post", "failed to extract campaign id", "failed to download", "cannot import yt-dlp", "cannot import youtube-dl", ] all_lines = combined.split("\n") error_lines = [ line for line in all_lines if any(ind in line for ind in actual_error_indicators) ] per_item_error_lines = [ line for line in error_lines if any(p in line for p in per_item_patterns) ] source_level_error_lines = [ line for line in error_lines if line not in set(per_item_error_lines) ] has_actual_error = bool(source_level_error_lines) if per_item_error_lines: per_item_set = set(per_item_error_lines) combined = "\n".join(line for line in all_lines if line not in per_item_set) if (skip_line_count > 0 or has_skip_text) and not has_actual_error: return ErrorType.NO_NEW_CONTENT, "No new content to download" if return_code == 0 and not stdout.strip(): return ErrorType.NO_NEW_CONTENT, "No new content to download" has_401_error = ( '" 401 ' in combined or "401 unauthorized" in combined or "error 401" in combined or "status: 401" in combined or "status code: 401" in combined ) if has_401_error or any(p in combined for p in self.AUTH_ERROR_PATTERNS): return ErrorType.AUTH_ERROR, "Authentication failed — cookies may be expired or invalid" if any(p in combined for p in self.RATE_LIMIT_PATTERNS): return ErrorType.RATE_LIMITED, "Rate limited by server — try increasing sleep time" has_specific_404 = any(p in combined for p in self.NOT_FOUND_PATTERNS) has_generic_with_error = any( p in combined and "][error]" in combined for p in self.GENERIC_NOT_FOUND_PATTERNS ) if has_specific_404 or has_generic_with_error: return ErrorType.NOT_FOUND, "URL not found — artist may have changed username or deleted content" if any(p in combined for p in self.NETWORK_ERROR_PATTERNS): return ErrorType.NETWORK_ERROR, "Network error — check internet connection" if any(p in combined for p in self.ACCESS_DENIED_PATTERNS): return ErrorType.ACCESS_DENIED, "Access denied — may need higher subscription tier" if "http error" in combined or "httperror" in combined: return ErrorType.HTTP_ERROR, "HTTP error occurred" if "no suitable" in combined or "unsupported" in combined or "no extractor" in combined: return ErrorType.UNSUPPORTED_URL, "URL format not supported by gallery-dl" if return_code in (1, 4) and (skip_line_count > 0 or has_skip_text) and not has_actual_error: return ErrorType.NO_NEW_CONTENT, "No new content to download" if return_code in (1, 4) and not has_actual_error: tier_gated_lines = [ line for line in combined.split("\n") if "][warning]" in line and "not allowed to view post" in line ] if tier_gated_lines: count = len(tier_gated_lines) return ( ErrorType.TIER_LIMITED, f"Subscription tier does not grant access to {count} post{'s' if count != 1 else ''}", ) return ErrorType.UNKNOWN_ERROR, f"Unknown error (return code: {return_code})" def _count_downloaded_files(self, stdout: str) -> int: if not stdout: return 0 return sum(1 for line in stdout.splitlines() if line.strip().startswith("/")) def _written_paths(self, stdout: str) -> list[Path]: if not stdout: return [] return [ Path(stripped) for line in stdout.splitlines() if (stripped := line.strip()).startswith("/") ] def _validate_and_quarantine( self, written_paths: list[Path], artist_slug: str, platform: str, url: str, ) -> tuple[list[str], list[dict]]: quarantined_relpaths: list[str] = [] failures: list[dict] = [] if not written_paths: return quarantined_relpaths, failures quarantine_root = ( self.images_root / "_quarantine" / artist_slug / platform ) for path in written_paths: if not is_validatable(path): continue try: result = validate_file(path) except Exception as exc: log.warning("Validator raised on %s: %s", path, exc) continue if result.ok: continue try: quarantine_root.mkdir(parents=True, exist_ok=True) dest = quarantine_root / path.name counter = 1 while dest.exists(): dest = quarantine_root / f"{path.stem}.{counter}{path.suffix}" counter += 1 path.rename(dest) sidecar = dest.with_suffix(dest.suffix + ".quarantine.json") sidecar.write_text( json.dumps( { "original_path": str(path), "source_url": url, "artist_slug": artist_slug, "platform": platform, "format": result.format, "reason": result.reason, "size": result.size, "quarantined_at": datetime.now(UTC).isoformat(), }, indent=2, ) ) except OSError as exc: log.error("Failed to quarantine %s: %s. File left in place.", path, exc) continue log.warning( "Quarantined corrupt file: %s → %s (%s: %s)", path, dest, result.format, result.reason, ) quarantined_relpaths.append(str(dest)) failures.append( { "path": str(path), "format": result.format, "reason": result.reason, "size": result.size, } ) return quarantined_relpaths, failures def _compute_run_stats(self, return_code: int, stdout: str, stderr: str) -> dict: stdout = stdout or "" stderr = stderr or "" skipped_stdout = sum( 1 for line in stdout.splitlines() if line.strip().startswith("#") ) skipped_stderr = sum( 1 for line in stderr.splitlines() if "] skipping " in line.lower() ) per_item_failures = sum( 1 for line in stderr.splitlines() if "[download][error]" in line.lower() and "failed to download" in line.lower() ) warning_count = sum( 1 for line in stderr.splitlines() if "][warning]" in line.lower() ) tier_gated_count = sum( 1 for line in stderr.splitlines() if "][warning]" in line.lower() and "not allowed to view post" in line.lower() ) return { "exit_code": return_code, "downloaded_count": self._count_downloaded_files(stdout), "skipped_count": skipped_stdout + skipped_stderr, "per_item_failures": per_item_failures, "warning_count": warning_count, "tier_gated_count": tier_gated_count, } @staticmethod def _extract_errors_warnings(stderr: str) -> str: if not stderr: return "" kept = [ line for line in stderr.splitlines() if "][error]" in line.lower() or "][warning]" in line.lower() ] return "\n".join(kept) @staticmethod def _truncate_log(text: str, max_bytes: int = 500_000) -> str: if not text: return text encoded = text.encode("utf-8") if len(encoded) <= max_bytes: return text half = max_bytes // 2 head = encoded[:half].decode("utf-8", errors="ignore") tail = encoded[-half:].decode("utf-8", errors="ignore") head_lines = head.count("\n") tail_lines = tail.count("\n") total_lines = text.count("\n") elided = max(0, total_lines - head_lines - tail_lines) marker = f"\n\n... [{elided} lines elided, {len(encoded) - max_bytes} bytes] ...\n\n" return head + marker + tail async def download( self, url: str, artist_slug: str, platform: str, source_config: SourceConfig | None = None, cookies_path: str | None = None, auth_token: str | None = None, ) -> DownloadResult: start_time = time.time() started_at = datetime.now(UTC).isoformat() if source_config is None: source_config = SourceConfig() config = self._build_config_for_source(platform, source_config, artist_slug) if cookies_path: config["extractor"]["cookies"] = cookies_path if auth_token and platform == "discord": config["extractor"].setdefault("discord", {}) config["extractor"]["discord"]["token"] = auth_token if auth_token and platform == "pixiv": config["extractor"].setdefault("pixiv", {}) config["extractor"]["pixiv"]["refresh-token"] = auth_token with tempfile.NamedTemporaryFile( mode="w", suffix=".json", delete=False, dir=str(self._config_dir), ) as fh: json.dump(config, fh, indent=2) temp_config_path = fh.name try: cmd = [ sys.executable, "-m", "gallery_dl", "--config", temp_config_path, "--verbose", url, ] log.info("Executing gallery-dl for %s/%s: %s", artist_slug, platform, url) loop = asyncio.get_running_loop() proc = await loop.run_in_executor( None, lambda: subprocess.run( cmd, capture_output=True, text=True, timeout=source_config.timeout, ), ) duration = time.time() - start_time completed_at = datetime.now(UTC).isoformat() quarantined_paths: list[str] = [] quarantine_failures: list[dict] = [] if self._validate_files: quarantined_paths, quarantine_failures = self._validate_and_quarantine( written_paths=self._written_paths(proc.stdout), artist_slug=artist_slug, platform=platform, url=url, ) written_paths = [ str(p) for p in self._written_paths(proc.stdout) if str(p) not in set(quarantined_paths) ] files_written = self._count_downloaded_files(proc.stdout) files_quarantined = len(quarantined_paths) files_kept = max(0, files_written - files_quarantined) if proc.returncode == 0: if files_quarantined > 0: return DownloadResult( success=False, url=url, artist_slug=artist_slug, platform=platform, files_downloaded=files_kept, files_quarantined=files_quarantined, quarantined_paths=quarantined_paths, written_paths=written_paths, stdout=proc.stdout, stderr=proc.stderr, return_code=proc.returncode, error_type=ErrorType.VALIDATION_FAILED, error_message=_summarize_validation_failures(quarantine_failures), duration_seconds=duration, started_at=started_at, completed_at=completed_at, ) return DownloadResult( success=True, url=url, artist_slug=artist_slug, platform=platform, files_downloaded=files_kept, written_paths=written_paths, stdout=proc.stdout, stderr=proc.stderr, return_code=proc.returncode, duration_seconds=duration, started_at=started_at, completed_at=completed_at, ) error_type, error_message = self._categorize_error( proc.returncode, proc.stdout, proc.stderr ) success = error_type in (ErrorType.NO_NEW_CONTENT, ErrorType.TIER_LIMITED) if files_quarantined > 0: error_type = ErrorType.VALIDATION_FAILED error_message = _summarize_validation_failures(quarantine_failures) success = False return DownloadResult( success=success, url=url, artist_slug=artist_slug, platform=platform, files_downloaded=files_kept if success else 0, files_quarantined=files_quarantined, quarantined_paths=quarantined_paths, written_paths=written_paths, stdout=proc.stdout, stderr=proc.stderr, return_code=proc.returncode, error_type=error_type, error_message=error_message, duration_seconds=duration, started_at=started_at, completed_at=completed_at, ) except subprocess.TimeoutExpired: duration = time.time() - start_time log.error("Download timeout for %s/%s after %.1fs", artist_slug, platform, duration) return DownloadResult( success=False, url=url, artist_slug=artist_slug, platform=platform, error_type=ErrorType.TIMEOUT, error_message=f"Download timed out after {source_config.timeout} seconds", duration_seconds=duration, started_at=started_at, completed_at=datetime.now(UTC).isoformat(), ) except Exception as exc: duration = time.time() - start_time log.exception("Unexpected error downloading %s/%s: %s", artist_slug, platform, exc) return DownloadResult( success=False, url=url, artist_slug=artist_slug, platform=platform, error_type=ErrorType.UNKNOWN_ERROR, error_message=str(exc), duration_seconds=duration, started_at=started_at, completed_at=datetime.now(UTC).isoformat(), ) finally: try: Path(temp_config_path).unlink() # noqa: ASYNC240 except Exception: pass