bvandeusen
2765c464bd
After gallery-dl returns, parse stdout for written file paths and run
the magic-byte validator on each. Files that fail are moved to
{download_path}/_quarantine/{subscription}/{platform}/ with a sidecar
JSON capturing the source URL, validation reason, and original path —
enough for an operator to redownload-from-source or delete.
Adds ErrorType.VALIDATION_FAILED. A successful gallery-dl run that
produced quarantined files is now a soft failure (success=False,
error_message names the dominant failure reason and count) so the
source's error_count ticks up and the dashboard surfaces it. Gated
by download.validate_files setting (default True).
files_quarantined and quarantined_paths are persisted into download
metadata for the UI/API to consume.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
1115 lines
46 KiB
Python
1115 lines
46 KiB
Python
"""Gallery-dl wrapper service.
|
||
|
||
This service provides a clean interface for executing gallery-dl downloads
|
||
with support for per-source configuration overrides.
|
||
"""
|
||
|
||
import json
|
||
import logging
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
import time
|
||
from dataclasses import dataclass, field
|
||
from enum import Enum
|
||
from pathlib import Path
|
||
from typing import Optional
|
||
|
||
from app.config import get_settings
|
||
from app.models.base import utcnow
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
class ErrorType(str, Enum):
|
||
"""Error types for categorizing download failures."""
|
||
AUTH_ERROR = "auth_error"
|
||
RATE_LIMITED = "rate_limited"
|
||
NOT_FOUND = "not_found"
|
||
ACCESS_DENIED = "access_denied"
|
||
NETWORK_ERROR = "network_error"
|
||
NO_NEW_CONTENT = "no_new_content"
|
||
TIER_LIMITED = "tier_limited"
|
||
TIMEOUT = "timeout"
|
||
HTTP_ERROR = "http_error"
|
||
UNSUPPORTED_URL = "unsupported_url"
|
||
VALIDATION_FAILED = "validation_failed"
|
||
UNKNOWN_ERROR = "unknown_error"
|
||
|
||
|
||
class ContentType(str, Enum):
|
||
"""Content types that can be downloaded from platforms."""
|
||
# Patreon content types
|
||
IMAGES = "images"
|
||
IMAGE_LARGE = "image_large"
|
||
ATTACHMENTS = "attachments"
|
||
POSTFILE = "postfile"
|
||
CONTENT = "content" # Embedded content in post body
|
||
|
||
# Generic types
|
||
ALL = "all"
|
||
|
||
|
||
@dataclass
|
||
class SourceConfig:
|
||
"""Per-source configuration that can override global settings.
|
||
|
||
These settings are stored in the source's metadata field and merged
|
||
with global defaults when executing downloads.
|
||
"""
|
||
# Content types to download (platform-specific)
|
||
# For Patreon: ["images", "image_large", "attachments", "postfile", "content"]
|
||
# For others: typically just ["all"] or specific types
|
||
content_types: list[str] = field(default_factory=lambda: ["all"])
|
||
|
||
# Rate limiting overrides
|
||
sleep: Optional[float] = None # Delay between downloads
|
||
sleep_request: Optional[float] = None # Delay between requests
|
||
|
||
# Directory/filename pattern overrides
|
||
directory_pattern: Optional[str] = None
|
||
filename_pattern: Optional[str] = None
|
||
|
||
# Other options
|
||
skip_existing: bool = True # Skip files already in archive
|
||
save_metadata: bool = True # Save JSON metadata alongside files
|
||
timeout: int = 3600 # Download timeout in seconds
|
||
|
||
@classmethod
|
||
def from_dict(cls, data: dict) -> "SourceConfig":
|
||
"""Create SourceConfig from a dictionary (e.g., from source.metadata)."""
|
||
return cls(
|
||
content_types=data.get("content_types", ["all"]),
|
||
sleep=data.get("sleep"),
|
||
sleep_request=data.get("sleep_request"),
|
||
directory_pattern=data.get("directory_pattern"),
|
||
filename_pattern=data.get("filename_pattern"),
|
||
skip_existing=data.get("skip_existing", True),
|
||
save_metadata=data.get("save_metadata", True),
|
||
timeout=data.get("timeout", 3600),
|
||
)
|
||
|
||
def to_dict(self) -> dict:
|
||
"""Convert to dictionary for storage."""
|
||
return {
|
||
"content_types": self.content_types,
|
||
"sleep": self.sleep,
|
||
"sleep_request": self.sleep_request,
|
||
"directory_pattern": self.directory_pattern,
|
||
"filename_pattern": self.filename_pattern,
|
||
"skip_existing": self.skip_existing,
|
||
"save_metadata": self.save_metadata,
|
||
"timeout": self.timeout,
|
||
}
|
||
|
||
|
||
def _summarize_validation_failures(failures: list[dict]) -> str:
|
||
"""Human-readable error_message for a run that quarantined files.
|
||
|
||
Picks the dominant failure reason (most common) and counts so the
|
||
dashboard's error column says something useful — "3 files truncated
|
||
(missing JPEG EOI marker)" rather than a generic VALIDATION_FAILED.
|
||
"""
|
||
if not failures:
|
||
return "Validation failed"
|
||
reasons: dict[str, int] = {}
|
||
for f in failures:
|
||
key = f.get("reason") or "unknown"
|
||
reasons[key] = reasons.get(key, 0) + 1
|
||
top_reason, top_count = max(reasons.items(), key=lambda kv: kv[1])
|
||
n = len(failures)
|
||
if top_count == n:
|
||
return f"{n} file{'s' if n != 1 else ''} quarantined: {top_reason}"
|
||
return f"{n} files quarantined ({top_count}× {top_reason}, mixed)"
|
||
|
||
|
||
@dataclass
|
||
class DownloadResult:
|
||
"""Result of a gallery-dl download execution."""
|
||
success: bool
|
||
url: str
|
||
subscription_name: str
|
||
platform: str
|
||
|
||
# Output details
|
||
files_downloaded: int = 0
|
||
files_quarantined: int = 0
|
||
quarantined_paths: list[str] = field(default_factory=list)
|
||
stdout: str = ""
|
||
stderr: str = ""
|
||
return_code: int = 0
|
||
|
||
# Error details (if failed)
|
||
error_type: Optional[ErrorType] = None
|
||
error_message: Optional[str] = None
|
||
|
||
# Timing
|
||
duration_seconds: float = 0.0
|
||
started_at: Optional[str] = None
|
||
completed_at: Optional[str] = None
|
||
|
||
|
||
class GalleryDLService:
|
||
"""Service for executing gallery-dl downloads.
|
||
|
||
Handles:
|
||
- Building gallery-dl configuration with per-source overrides
|
||
- Executing the subprocess with proper error handling
|
||
- Parsing output and categorizing errors
|
||
- Managing temporary config files
|
||
"""
|
||
|
||
# Error classification patterns — used by _categorize_error()
|
||
AUTH_ERROR_PATTERNS = [
|
||
"unauthorized",
|
||
"login required",
|
||
"please login",
|
||
"must be logged in",
|
||
"authentication required",
|
||
"authenticationerror",
|
||
"refresh-token",
|
||
"session expired",
|
||
"invalid cookie",
|
||
"cookies are expired",
|
||
"cookies have expired",
|
||
"not logged in",
|
||
]
|
||
RATE_LIMIT_PATTERNS = [
|
||
'" 429 ',
|
||
"rate limit",
|
||
"too many requests",
|
||
"ratelimit",
|
||
"throttl",
|
||
]
|
||
NOT_FOUND_PATTERNS = [
|
||
'" 404 ',
|
||
"error 404",
|
||
"status: 404",
|
||
"status code: 404",
|
||
"404 not found",
|
||
"page not found",
|
||
"user not found",
|
||
"creator not found",
|
||
"artist not found",
|
||
"content no longer available",
|
||
"has been deleted",
|
||
"account deleted",
|
||
"profile does not exist",
|
||
]
|
||
GENERIC_NOT_FOUND_PATTERNS = ["does not exist", "no longer available"]
|
||
NETWORK_ERROR_PATTERNS = [
|
||
"timed out",
|
||
"read timed out",
|
||
"connect timed out",
|
||
"connection timed out",
|
||
"connection refused",
|
||
"connection reset",
|
||
"network unreachable",
|
||
"name resolution failed",
|
||
"name or service not known",
|
||
"nodename nor servname provided",
|
||
"ssl: certificate_verify_failed",
|
||
"ssl: wrong_version_number",
|
||
"certificate verify failed",
|
||
"[errno",
|
||
]
|
||
ACCESS_DENIED_PATTERNS = [
|
||
'" 403 ',
|
||
"error 403",
|
||
"forbidden",
|
||
"access denied",
|
||
"permission denied",
|
||
"tier required",
|
||
"pledge required",
|
||
]
|
||
|
||
# Platform-specific default content types and options
|
||
# Directory patterns are relative - subscription_name/platform is prepended automatically
|
||
PLATFORM_DEFAULTS = {
|
||
"patreon": {
|
||
"content_types": ["images", "image_large", "attachments", "postfile", "content"],
|
||
"directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"],
|
||
"filename": "{num:>02}_{filename}.{extension}",
|
||
# Additional options
|
||
"videos": True, # Download video content
|
||
"embeds": True, # Download embedded URLs (YouTube, etc.)
|
||
# Deep crawling options
|
||
"cursor": True, # Enable cursor-based pagination for complete history
|
||
},
|
||
"subscribestar": {
|
||
"content_types": ["all"],
|
||
"directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"],
|
||
"filename": "{num:>02}_{filename}.{extension}",
|
||
# Automatic infinite scroll pagination is built-in
|
||
},
|
||
"hentaifoundry": {
|
||
"content_types": ["all"], # Download all content types
|
||
"directory": [], # Flat structure within platform folder
|
||
"filename": "{category}_{index:>03}_{title[:50]}.{extension}",
|
||
"include": "all", # Include pictures, scraps, stories - complete content
|
||
# Automatic pagination through all pages (25 items/page)
|
||
},
|
||
"discord": {
|
||
"content_types": ["all"],
|
||
"directory": ["{channel[name]}"],
|
||
"filename": "{date:%Y%m%d}_{id}_{filename}.{extension}",
|
||
# Additional options. gallery-dl's discord extractor treats `embeds`
|
||
# as an iterable of embed types (strings) or the string "all" — a
|
||
# bool causes "argument of type 'bool' is not iterable" at runtime.
|
||
"embeds": "all",
|
||
"stickers": True, # Download stickers
|
||
"reactions": False, # Skip reaction images
|
||
# Deep crawling options
|
||
"threads": True, # Include thread content for complete history
|
||
},
|
||
"pixiv": {
|
||
"content_types": ["all"],
|
||
"directory": ["{category}"],
|
||
"filename": "{id}_{title[:50]}_{num:>02}.{extension}",
|
||
# Content options
|
||
"ugoira": True, # Download ugoira (animated) as WebM
|
||
},
|
||
"deviantart": {
|
||
"content_types": ["all"],
|
||
"directory": [], # Flat structure within platform folder
|
||
"filename": "{index:>03}_{title[:50]}.{extension}",
|
||
# Content options
|
||
"flat": True, # Flatten folder structure
|
||
"original": True, # Download original quality
|
||
"mature": True, # Include mature content
|
||
"metadata": True, # Include metadata
|
||
},
|
||
}
|
||
|
||
def __init__(
|
||
self,
|
||
rate_limit: Optional[float] = None,
|
||
validate_files: bool = True,
|
||
):
|
||
"""Initialize the GalleryDL service.
|
||
|
||
Args:
|
||
rate_limit: Override for download rate limit (seconds between requests).
|
||
If None, uses environment default.
|
||
validate_files: When True, run magic-byte validation on each file
|
||
gallery-dl writes and quarantine truncated files. Caller
|
||
(tasks.downloads) typically reads this from the
|
||
`download.validate_files` DB setting.
|
||
"""
|
||
self.settings = get_settings()
|
||
self._rate_limit = rate_limit if rate_limit is not None else self.settings.download_rate_limit
|
||
self._validate_files = validate_files
|
||
self._base_config = self._load_base_config()
|
||
|
||
def _load_base_config(self) -> dict:
|
||
"""Load the base gallery-dl configuration file."""
|
||
config_path = Path(self.settings.config_path) / "gallery-dl.conf"
|
||
|
||
if config_path.exists():
|
||
try:
|
||
with open(config_path) as f:
|
||
return json.load(f)
|
||
except json.JSONDecodeError as e:
|
||
logger.warning(f"Invalid gallery-dl.conf, using defaults: {e}")
|
||
|
||
# Return sensible defaults if no config exists
|
||
return self._get_default_config()
|
||
|
||
def _get_default_config(self) -> dict:
|
||
"""Get default gallery-dl configuration.
|
||
|
||
Platform-specific defaults from PLATFORM_DEFAULTS are folded into
|
||
extractor.<platform> so the returned dict is a complete, self-contained
|
||
gallery-dl config — editable via the Settings view and writable to
|
||
gallery-dl.conf for full user control. The internal `content_types`
|
||
meta-key is stripped (it's our abstraction, not a gallery-dl option —
|
||
it gets translated per-platform to `files`/`include` at run time).
|
||
"""
|
||
config = {
|
||
"extractor": {
|
||
"base-directory": self.settings.download_path,
|
||
"archive": str(Path(self.settings.config_path) / "archive.sqlite3"),
|
||
"skip": True,
|
||
"sleep": self._rate_limit,
|
||
# Faster request rate for pagination/checking (1/4 of rate_limit, min 0.5s)
|
||
"sleep-request": max(0.5, self._rate_limit / 4),
|
||
"retries": 3,
|
||
"timeout": 30.0,
|
||
"verify": True,
|
||
"postprocessors": [
|
||
{
|
||
"name": "metadata",
|
||
"mode": "json",
|
||
"directory": ".",
|
||
"filename": "{filename}.json",
|
||
}
|
||
],
|
||
},
|
||
"downloader": {
|
||
"part": True,
|
||
"part-directory": str(Path(self.settings.config_path) / "temp"),
|
||
"retries": 3,
|
||
"timeout": 120.0,
|
||
},
|
||
"output": {
|
||
"progress": True,
|
||
},
|
||
}
|
||
for platform_name, defaults in self.PLATFORM_DEFAULTS.items():
|
||
config["extractor"][platform_name] = {
|
||
key: value for key, value in defaults.items() if key != "content_types"
|
||
}
|
||
return config
|
||
|
||
def _build_config_for_source(
|
||
self,
|
||
platform: str,
|
||
source_config: SourceConfig,
|
||
destination: str,
|
||
) -> dict:
|
||
"""Build a merged configuration for a specific source.
|
||
|
||
Precedence: _get_default_config() (built-in) → gallery-dl.conf on disk
|
||
→ per-source overrides. The platform's extractor section is preserved
|
||
from the base config (deep copy) and source overrides are applied on
|
||
top — so user tweaks in gallery-dl.conf survive. If the user's conf
|
||
dropped the platform section entirely, re-seed from defaults so
|
||
platform behavior stays sticky.
|
||
"""
|
||
config = json.loads(json.dumps(self._base_config)) # Deep copy
|
||
|
||
if "extractor" not in config:
|
||
config["extractor"] = {}
|
||
|
||
# Re-seed platform defaults if the user's conf is missing the section.
|
||
# Prevents "I deleted extractor.discord from my conf and silently lost
|
||
# embeds/stickers/threads." from becoming a footgun.
|
||
if platform not in config["extractor"]:
|
||
seed = self._get_default_config()
|
||
config["extractor"][platform] = seed.get("extractor", {}).get(platform, {})
|
||
|
||
# extractor.* global overrides from source config
|
||
config["extractor"]["base-directory"] = destination
|
||
if source_config.sleep is not None:
|
||
config["extractor"]["sleep"] = source_config.sleep
|
||
if source_config.sleep_request is not None:
|
||
config["extractor"]["sleep-request"] = source_config.sleep_request
|
||
config["extractor"]["skip"] = source_config.skip_existing
|
||
|
||
if source_config.save_metadata:
|
||
config["extractor"]["postprocessors"] = [
|
||
{
|
||
"name": "metadata",
|
||
"mode": "json",
|
||
"directory": ".",
|
||
"filename": "{filename}.json",
|
||
}
|
||
]
|
||
else:
|
||
config["extractor"].pop("postprocessors", None)
|
||
|
||
# Source overrides on the platform-specific section. We preserve
|
||
# whatever is in the base config (platform defaults or user's conf)
|
||
# and only set keys that the source explicitly controls.
|
||
platform_section = config["extractor"][platform]
|
||
|
||
# Content types translate to platform-specific gallery-dl keys.
|
||
if platform == "patreon":
|
||
if "all" in source_config.content_types:
|
||
platform_section["files"] = ["images", "image_large", "attachments", "postfile", "content"]
|
||
else:
|
||
platform_section["files"] = source_config.content_types
|
||
elif platform == "hentaifoundry":
|
||
if "pictures" in source_config.content_types or "all" in source_config.content_types:
|
||
platform_section["include"] = "all"
|
||
|
||
# Directory/filename: source override wins; otherwise inherit whatever
|
||
# the base config already has (platform default or user-set value).
|
||
if source_config.directory_pattern:
|
||
platform_section["directory"] = source_config.directory_pattern
|
||
if source_config.filename_pattern:
|
||
platform_section["filename"] = source_config.filename_pattern
|
||
|
||
platform_section["metadata"] = source_config.save_metadata
|
||
|
||
return config
|
||
|
||
def _categorize_error(self, return_code: int, stdout: str, stderr: str) -> tuple[ErrorType, str]:
|
||
"""Categorize the error based on output and return code.
|
||
|
||
Priority order:
|
||
1. Check for skip messages (# prefixed lines or text patterns) → NO_NEW_CONTENT
|
||
2. Check for actual errors (auth, rate limit, 404, network, etc.)
|
||
3. Fall back to UNKNOWN_ERROR
|
||
|
||
This ordering prevents false positives where debug output matches error patterns.
|
||
"""
|
||
combined = f"{stdout} {stderr}".lower()
|
||
|
||
# === PHASE 1: Detect skip/no-new-content scenarios ===
|
||
# Gallery-dl indicates skipped files two ways:
|
||
# 1. Lines starting with '#' in stdout (e.g., "# /data/downloads/Artist/file.png")
|
||
# 2. Text patterns like "skipping", "already exists"
|
||
|
||
# Count lines starting with '#' (skip messages)
|
||
skip_line_count = len([
|
||
line for line in stdout.split('\n')
|
||
if line.strip().startswith('#')
|
||
])
|
||
|
||
# Text-based skip indicators
|
||
skip_text_indicators = ["skipping", "already exists", "archive"]
|
||
has_skip_text = any(ind in combined for ind in skip_text_indicators)
|
||
|
||
# Actual error indicators that override skip detection
|
||
actual_error_indicators = [
|
||
"][error]", # gallery-dl error log level: [patreon][error]
|
||
"exception", # Python exceptions
|
||
"traceback", # Python tracebacks
|
||
"download failed",
|
||
"extraction failed",
|
||
]
|
||
|
||
# Non-fatal error lines gallery-dl logs but recovers from. These are
|
||
# per-item failures the run already moved past — they shouldn't cause
|
||
# source-level misclassification:
|
||
# - "not allowed to view" / "unable to get post": per-item paywall
|
||
# warnings on Patreon URLs with mixed tier access.
|
||
# - "failed to extract campaign id": vanity-URL HTML path fails,
|
||
# gallery-dl falls back to /cw/<vanity> and recovers.
|
||
# - "failed to download": per-item media failure (expired URL,
|
||
# yt-dlp HLS 403 on tier-gated video, deleted attachment).
|
||
# Downloader logs [download][error] and continues.
|
||
# - "cannot import yt-dlp"/"youtube-dl": video backend missing;
|
||
# surfaces via run_stats as a setup gap, not a source failure.
|
||
per_item_patterns = [
|
||
"not allowed to view",
|
||
"unable to get post",
|
||
"failed to extract campaign id",
|
||
"failed to download",
|
||
"cannot import yt-dlp",
|
||
"cannot import youtube-dl",
|
||
]
|
||
all_lines = combined.split('\n')
|
||
error_lines = [
|
||
line for line in all_lines
|
||
if any(ind in line for ind in actual_error_indicators)
|
||
]
|
||
per_item_error_lines = [
|
||
line for line in error_lines
|
||
if any(p in line for p in per_item_patterns)
|
||
]
|
||
source_level_error_lines = [
|
||
line for line in error_lines
|
||
if line not in set(per_item_error_lines)
|
||
]
|
||
|
||
# Only source-level errors block skip/tier-limited classification.
|
||
# Per-item failures are tracked separately in run_stats.
|
||
has_actual_error = bool(source_level_error_lines)
|
||
|
||
# Strip per-item error lines from the text used for downstream pattern
|
||
# matching. Otherwise noise like "HTTP Error 403: Forbidden" from a
|
||
# yt-dlp HLS failure latches onto ACCESS_DENIED and masks the real
|
||
# signal (tier gating, no new content, etc.).
|
||
if per_item_error_lines:
|
||
per_item_set = set(per_item_error_lines)
|
||
combined = '\n'.join(line for line in all_lines if line not in per_item_set)
|
||
|
||
# If we have skip evidence and no real errors, it's no_new_content
|
||
if (skip_line_count > 0 or has_skip_text) and not has_actual_error:
|
||
trigger = f"{skip_line_count} skip lines" if skip_line_count > 0 else "skip text indicator"
|
||
logger.debug(f"Error classified as no_new_content via: {trigger}")
|
||
return ErrorType.NO_NEW_CONTENT, "No new content to download"
|
||
|
||
# Empty output with low return codes often means nothing to download
|
||
if return_code == 0 and not stdout.strip():
|
||
logger.debug("Error classified as no_new_content via: empty stdout with return code 0")
|
||
return ErrorType.NO_NEW_CONTENT, "No new content to download"
|
||
|
||
# === PHASE 2: Check for specific error types ===
|
||
|
||
# Auth errors - look for error context, not just HTTP codes that appear in debug logs
|
||
# HTTP debug lines look like: '"GET /path HTTP/1.1" 401 None' - we need to detect actual errors
|
||
# Check for 401 specifically as an HTTP error response (not in URLs)
|
||
# Look for patterns like "401" followed by error context or at end of HTTP log line
|
||
has_401_error = (
|
||
'" 401 ' in combined or # HTTP response: "GET /path HTTP/1.1" 401 None
|
||
"401 unauthorized" in combined or
|
||
"error 401" in combined or
|
||
"status: 401" in combined or
|
||
"status code: 401" in combined
|
||
)
|
||
if has_401_error or any(pattern in combined for pattern in self.AUTH_ERROR_PATTERNS):
|
||
matched = "401 HTTP response" if has_401_error else next(p for p in self.AUTH_ERROR_PATTERNS if p in combined)
|
||
logger.debug(f"Error classified as auth_error via pattern: '{matched}'")
|
||
return ErrorType.AUTH_ERROR, "Authentication failed - cookies may be expired or invalid"
|
||
|
||
# Rate limiting - look for actual rate limit errors
|
||
if any(pattern in combined for pattern in self.RATE_LIMIT_PATTERNS):
|
||
matched = next(p for p in self.RATE_LIMIT_PATTERNS if p in combined)
|
||
logger.debug(f"Error classified as rate_limited via pattern: '{matched}'")
|
||
return ErrorType.RATE_LIMITED, "Rate limited by server - try increasing sleep time"
|
||
|
||
# 404 Not Found - check for HTTP 404 responses with error context
|
||
# Be specific to avoid matching "not found" in debug messages like "archive entry not found"
|
||
# Also require error-level context for generic patterns to avoid false positives
|
||
has_specific_404 = any(pattern in combined for pattern in self.NOT_FOUND_PATTERNS)
|
||
has_generic_with_error = any(
|
||
pattern in combined and "][error]" in combined
|
||
for pattern in self.GENERIC_NOT_FOUND_PATTERNS
|
||
)
|
||
if has_specific_404 or has_generic_with_error:
|
||
if has_specific_404:
|
||
matched = next(p for p in self.NOT_FOUND_PATTERNS if p in combined)
|
||
logger.debug(f"Error classified as not_found via specific 404 pattern: '{matched}'")
|
||
else:
|
||
matched = next(p for p in self.GENERIC_NOT_FOUND_PATTERNS if p in combined)
|
||
logger.debug(f"Error classified as not_found via generic pattern + error context: '{matched}'")
|
||
return ErrorType.NOT_FOUND, "URL not found - artist may have changed username or deleted content"
|
||
|
||
# Network errors - use specific patterns to avoid matching config values
|
||
# (e.g., "timeout": 30.0 in debug output should not trigger network_error)
|
||
if any(pattern in combined for pattern in self.NETWORK_ERROR_PATTERNS):
|
||
matched = next(p for p in self.NETWORK_ERROR_PATTERNS if p in combined)
|
||
logger.debug(f"Error classified as network_error via pattern: '{matched}'")
|
||
return ErrorType.NETWORK_ERROR, "Network error - check internet connection"
|
||
|
||
# 403 Forbidden / Access denied
|
||
if any(pattern in combined for pattern in self.ACCESS_DENIED_PATTERNS):
|
||
matched = next(p for p in self.ACCESS_DENIED_PATTERNS if p in combined)
|
||
logger.debug(f"Error classified as access_denied via pattern: '{matched}'")
|
||
return ErrorType.ACCESS_DENIED, "Access denied - may need higher subscription tier"
|
||
|
||
# Generic HTTP errors
|
||
if "http error" in combined or "httperror" in combined:
|
||
trigger = "http error" if "http error" in combined else "httperror"
|
||
logger.debug(f"Error classified as http_error via trigger: '{trigger}'")
|
||
return ErrorType.HTTP_ERROR, "HTTP error occurred"
|
||
|
||
# Unsupported URL
|
||
if "no suitable" in combined or "unsupported" in combined or "no extractor" in combined:
|
||
trigger = next(t for t in ("no suitable", "unsupported", "no extractor") if t in combined)
|
||
logger.debug(f"Error classified as unsupported_url via trigger: '{trigger}'")
|
||
return ErrorType.UNSUPPORTED_URL, "URL format not supported by gallery-dl"
|
||
|
||
# If we got here with a non-catastrophic return code, saw skip activity, and no
|
||
# specific error patterns matched, it's likely just no new content.
|
||
# gallery-dl exit codes: 1 = some extractions skipped, 4 = no URLs extracted
|
||
# (common with /c/user/posts Patreon URLs where all content is already archived)
|
||
if return_code in (1, 4) and (skip_line_count > 0 or has_skip_text) and not has_actual_error:
|
||
trigger = f"{skip_line_count} skip lines" if skip_line_count > 0 else "skip text indicator"
|
||
logger.debug(f"Error classified as no_new_content via: rc{return_code} + {trigger}")
|
||
return ErrorType.NO_NEW_CONTENT, "No new content to download"
|
||
|
||
# Subscription-tier gating: gallery-dl emits [patreon][warning] "Not allowed to
|
||
# view post N" for posts the current auth token can't access. A run that exits
|
||
# 1/4 with only these warnings and no downloads means the account's tier doesn't
|
||
# cover the requested posts — it's a legitimate state, not a failure.
|
||
if return_code in (1, 4) and not has_actual_error:
|
||
tier_gated_lines = [
|
||
line for line in combined.split('\n')
|
||
if "][warning]" in line and "not allowed to view post" in line
|
||
]
|
||
if tier_gated_lines:
|
||
count = len(tier_gated_lines)
|
||
logger.debug(f"Error classified as tier_limited via: rc{return_code} + {count} tier-gated warnings")
|
||
return (
|
||
ErrorType.TIER_LIMITED,
|
||
f"Subscription tier does not grant access to {count} post{'s' if count != 1 else ''}",
|
||
)
|
||
|
||
# Final fallback - genuine unknown error
|
||
return ErrorType.UNKNOWN_ERROR, f"Unknown error (return code: {return_code})"
|
||
|
||
def _count_downloaded_files(self, stdout: str) -> int:
|
||
"""Count the number of files downloaded from stdout.
|
||
|
||
Gallery-dl writes each downloaded file path to stdout as an absolute path
|
||
(starting with '/'). Log lines contain '[' prefixes; verbose HTTP lines
|
||
go to stderr. Counting only lines starting with '/' avoids over-counting
|
||
log and progress output.
|
||
"""
|
||
if not stdout:
|
||
return 0
|
||
|
||
return sum(1 for line in stdout.splitlines() if line.strip().startswith("/"))
|
||
|
||
def _written_paths(self, stdout: str) -> list[Path]:
|
||
"""Extract absolute file paths gallery-dl wrote during this run.
|
||
|
||
Mirrors `_count_downloaded_files` but returns the paths themselves so
|
||
post-download validation only inspects files we actually just wrote
|
||
(skips on '#' lines and log noise are excluded). Caller is responsible
|
||
for filtering further by extension if needed.
|
||
"""
|
||
if not stdout:
|
||
return []
|
||
return [
|
||
Path(stripped)
|
||
for line in stdout.splitlines()
|
||
if (stripped := line.strip()).startswith("/")
|
||
]
|
||
|
||
def _validate_and_quarantine(
|
||
self,
|
||
written_paths: list[Path],
|
||
subscription_name: str,
|
||
platform: str,
|
||
url: str,
|
||
) -> tuple[list[str], list[dict]]:
|
||
"""Validate freshly-written files; quarantine any that fail.
|
||
|
||
Returns (quarantined_relpaths, failure_details). Each failure detail
|
||
is a dict suitable for logging/persisting: {path, format, reason, size}.
|
||
Files that pass validation or aren't a validatable format are left in
|
||
place. Failures are moved to {download_path}/_quarantine/<subscription>/
|
||
<platform>/ along with a sidecar JSON capturing the source URL, the
|
||
validation reason, and the original on-disk path so an operator can
|
||
decide to redownload-from-source or delete.
|
||
"""
|
||
# Local import keeps the module import-cycle clean and lets tests
|
||
# patch validation behavior.
|
||
from app.services.file_validator import is_validatable, validate_file
|
||
|
||
quarantined_relpaths: list[str] = []
|
||
failures: list[dict] = []
|
||
|
||
if not written_paths:
|
||
return quarantined_relpaths, failures
|
||
|
||
quarantine_root = (
|
||
Path(self.settings.download_path)
|
||
/ "_quarantine"
|
||
/ subscription_name
|
||
/ platform
|
||
)
|
||
|
||
for path in written_paths:
|
||
if not is_validatable(path):
|
||
continue
|
||
try:
|
||
result = validate_file(path)
|
||
except Exception as e:
|
||
# Validator must never crash the download flow.
|
||
logger.warning(f"Validator raised on {path}: {e}")
|
||
continue
|
||
|
||
if result.ok:
|
||
continue
|
||
|
||
# Move the bad file out of the library so downstream consumers
|
||
# (thumbnailer, ML, sync) don't trip on it.
|
||
try:
|
||
quarantine_root.mkdir(parents=True, exist_ok=True)
|
||
dest = quarantine_root / path.name
|
||
# Avoid overwriting prior quarantine entries with the same
|
||
# filename — append a numeric suffix until we find a free slot.
|
||
counter = 1
|
||
while dest.exists():
|
||
dest = quarantine_root / f"{path.stem}.{counter}{path.suffix}"
|
||
counter += 1
|
||
path.rename(dest)
|
||
sidecar = dest.with_suffix(dest.suffix + ".quarantine.json")
|
||
sidecar.write_text(
|
||
json.dumps(
|
||
{
|
||
"original_path": str(path),
|
||
"source_url": url,
|
||
"subscription": subscription_name,
|
||
"platform": platform,
|
||
"format": result.format,
|
||
"reason": result.reason,
|
||
"size": result.size,
|
||
"quarantined_at": utcnow().isoformat(),
|
||
},
|
||
indent=2,
|
||
)
|
||
)
|
||
except OSError as e:
|
||
# If we can't move it, log loudly but keep going. A failed
|
||
# quarantine isn't worse than the pre-fix status quo.
|
||
logger.error(
|
||
f"Failed to quarantine {path}: {e}. File left in place."
|
||
)
|
||
continue
|
||
|
||
logger.warning(
|
||
f"Quarantined corrupt file: {path} → {dest} "
|
||
f"({result.format}: {result.reason})"
|
||
)
|
||
quarantined_relpaths.append(str(dest))
|
||
failures.append(
|
||
{
|
||
"path": str(path),
|
||
"format": result.format,
|
||
"reason": result.reason,
|
||
"size": result.size,
|
||
}
|
||
)
|
||
|
||
return quarantined_relpaths, failures
|
||
|
||
def _compute_run_stats(self, return_code: int, stdout: str, stderr: str) -> dict:
|
||
"""Summarize a gallery-dl run as structured counts for UI/analytics.
|
||
|
||
Returns a flat dict:
|
||
- exit_code: gallery-dl's process return code
|
||
- downloaded_count: files actually written (stdout lines starting with '/')
|
||
- skipped_count: archive-hit skips (stdout '#' lines + stderr 'skipping ' lines)
|
||
- per_item_failures: [download][error] lines (single-item failures the
|
||
downloader logs but recovers from)
|
||
- warning_count: [*][warning] lines
|
||
- tier_gated_count: 'not allowed to view post' warnings (Patreon
|
||
subscription-tier gating — informational, not a failure)
|
||
"""
|
||
stdout = stdout or ""
|
||
stderr = stderr or ""
|
||
|
||
skipped_stdout = sum(
|
||
1 for line in stdout.splitlines() if line.strip().startswith("#")
|
||
)
|
||
skipped_stderr = sum(
|
||
1 for line in stderr.splitlines() if "] skipping " in line.lower()
|
||
)
|
||
|
||
per_item_failures = sum(
|
||
1 for line in stderr.splitlines()
|
||
if "[download][error]" in line.lower() and "failed to download" in line.lower()
|
||
)
|
||
|
||
warning_count = sum(
|
||
1 for line in stderr.splitlines() if "][warning]" in line.lower()
|
||
)
|
||
|
||
tier_gated_count = sum(
|
||
1 for line in stderr.splitlines()
|
||
if "][warning]" in line.lower() and "not allowed to view post" in line.lower()
|
||
)
|
||
|
||
return {
|
||
"exit_code": return_code,
|
||
"downloaded_count": self._count_downloaded_files(stdout),
|
||
"skipped_count": skipped_stdout + skipped_stderr,
|
||
"per_item_failures": per_item_failures,
|
||
"warning_count": warning_count,
|
||
"tier_gated_count": tier_gated_count,
|
||
# quarantined_count is overlaid by the caller (tasks.downloads)
|
||
# when persisting metadata; surfaced via DownloadResult.files_quarantined
|
||
# because run_stats has no access to the live filesystem.
|
||
}
|
||
|
||
@staticmethod
|
||
def _extract_errors_warnings(stderr: str) -> str:
|
||
"""Filter stderr down to just [error] / [warning] lines.
|
||
|
||
Gallery-dl's verbose mode sends all logging to stderr including [debug]
|
||
and urllib3 connection-pool noise. This helper produces the short
|
||
summary a user actually wants when opening a failed download.
|
||
"""
|
||
if not stderr:
|
||
return ""
|
||
|
||
kept = [
|
||
line for line in stderr.splitlines()
|
||
if "][error]" in line.lower() or "][warning]" in line.lower()
|
||
]
|
||
return "\n".join(kept)
|
||
|
||
@staticmethod
|
||
def _truncate_log(text: str, max_bytes: int = 500_000) -> str:
|
||
"""Cap a log field at max_bytes by keeping the head and tail.
|
||
|
||
Verbose 10+ minute runs can produce multi-MB stderr blobs. For display
|
||
and storage, we keep the first and last halves and drop the middle with
|
||
a marker indicating how many lines were elided. Beginning is useful for
|
||
context (config, URL); end is useful for the actual failure.
|
||
"""
|
||
if not text:
|
||
return text
|
||
|
||
encoded = text.encode("utf-8")
|
||
if len(encoded) <= max_bytes:
|
||
return text
|
||
|
||
half = max_bytes // 2
|
||
head = encoded[:half].decode("utf-8", errors="ignore")
|
||
tail = encoded[-half:].decode("utf-8", errors="ignore")
|
||
|
||
head_lines = head.count("\n")
|
||
tail_lines = tail.count("\n")
|
||
total_lines = text.count("\n")
|
||
elided = max(0, total_lines - head_lines - tail_lines)
|
||
|
||
marker = f"\n\n... [{elided} lines elided, {len(encoded) - max_bytes} bytes] ...\n\n"
|
||
return head + marker + tail
|
||
|
||
async def download(
|
||
self,
|
||
url: str,
|
||
subscription_name: str,
|
||
platform: str,
|
||
source_config: Optional[SourceConfig] = None,
|
||
cookies_path: Optional[str] = None,
|
||
auth_token: Optional[str] = None,
|
||
) -> DownloadResult:
|
||
"""Execute a gallery-dl download.
|
||
|
||
Args:
|
||
url: The URL to download from
|
||
subscription_name: Name of the subscription/artist (used for destination folder)
|
||
platform: Platform name (patreon, subscribestar, etc.)
|
||
source_config: Optional per-source configuration overrides
|
||
cookies_path: Optional path to cookies file
|
||
auth_token: Optional auth token (for Discord and other token-based platforms)
|
||
|
||
Returns:
|
||
DownloadResult with success status and details
|
||
|
||
Downloads are saved to: {download_path}/{subscription_name}/{platform}/{post_folders}
|
||
"""
|
||
import asyncio
|
||
|
||
start_time = time.time()
|
||
started_at = utcnow().isoformat()
|
||
|
||
# Use default config if none provided
|
||
if source_config is None:
|
||
source_config = SourceConfig()
|
||
|
||
# Build destination path: subscription_name/platform
|
||
destination = str(Path(self.settings.download_path) / subscription_name / platform)
|
||
|
||
# Build merged configuration
|
||
config = self._build_config_for_source(platform, source_config, destination)
|
||
|
||
# Add cookies if provided
|
||
if cookies_path:
|
||
config["extractor"]["cookies"] = cookies_path
|
||
|
||
# Add auth token for Discord
|
||
if auth_token and platform == "discord":
|
||
if "discord" not in config["extractor"]:
|
||
config["extractor"]["discord"] = {}
|
||
config["extractor"]["discord"]["token"] = auth_token
|
||
|
||
# Add refresh token for Pixiv
|
||
if auth_token and platform == "pixiv":
|
||
if "pixiv" not in config["extractor"]:
|
||
config["extractor"]["pixiv"] = {}
|
||
config["extractor"]["pixiv"]["refresh-token"] = auth_token
|
||
|
||
# Write temporary config file
|
||
with tempfile.NamedTemporaryFile(
|
||
mode="w",
|
||
suffix=".json",
|
||
delete=False,
|
||
dir=self.settings.config_path,
|
||
) as f:
|
||
json.dump(config, f, indent=2)
|
||
temp_config_path = f.name
|
||
|
||
try:
|
||
# Build command
|
||
cmd = [
|
||
sys.executable,
|
||
"-m",
|
||
"gallery_dl",
|
||
"--config",
|
||
temp_config_path,
|
||
"--verbose",
|
||
url,
|
||
]
|
||
|
||
logger.info(f"Executing gallery-dl for {subscription_name}/{platform}: {url}")
|
||
logger.debug(f"Command: {' '.join(cmd)}")
|
||
|
||
# Execute subprocess
|
||
# Run in thread pool to not block async event loop
|
||
loop = asyncio.get_running_loop()
|
||
proc = await loop.run_in_executor(
|
||
None,
|
||
lambda: subprocess.run(
|
||
cmd,
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=source_config.timeout,
|
||
),
|
||
)
|
||
|
||
duration = time.time() - start_time
|
||
completed_at = utcnow().isoformat()
|
||
|
||
# Log output
|
||
if proc.stdout:
|
||
logger.debug(f"STDOUT for {subscription_name}/{platform}:\n{proc.stdout}")
|
||
if proc.stderr:
|
||
logger.debug(f"STDERR for {subscription_name}/{platform}:\n{proc.stderr}")
|
||
|
||
# Run validation on every fresh write before classifying. Files
|
||
# gallery-dl wrote and we then quarantined no longer count as
|
||
# "downloaded" — counting them would understate how many files
|
||
# need re-downloading and overstate the run's success.
|
||
quarantined_paths: list[str] = []
|
||
quarantine_failures: list[dict] = []
|
||
if self._validate_files:
|
||
quarantined_paths, quarantine_failures = self._validate_and_quarantine(
|
||
written_paths=self._written_paths(proc.stdout),
|
||
subscription_name=subscription_name,
|
||
platform=platform,
|
||
url=url,
|
||
)
|
||
|
||
files_written = self._count_downloaded_files(proc.stdout)
|
||
files_quarantined = len(quarantined_paths)
|
||
files_kept = max(0, files_written - files_quarantined)
|
||
|
||
# Determine success
|
||
if proc.returncode == 0:
|
||
# Exit 0 with quarantines: the run technically succeeded but
|
||
# delivered partially-corrupt content. Surface as a soft
|
||
# failure so the source's error_count ticks up and the user
|
||
# sees it on the dashboard, but keep the per-item details
|
||
# (quarantined_paths) for the maintenance UI.
|
||
if files_quarantined > 0:
|
||
return DownloadResult(
|
||
success=False,
|
||
url=url,
|
||
subscription_name=subscription_name,
|
||
platform=platform,
|
||
files_downloaded=files_kept,
|
||
files_quarantined=files_quarantined,
|
||
quarantined_paths=quarantined_paths,
|
||
stdout=proc.stdout,
|
||
stderr=proc.stderr,
|
||
return_code=proc.returncode,
|
||
error_type=ErrorType.VALIDATION_FAILED,
|
||
error_message=_summarize_validation_failures(quarantine_failures),
|
||
duration_seconds=duration,
|
||
started_at=started_at,
|
||
completed_at=completed_at,
|
||
)
|
||
return DownloadResult(
|
||
success=True,
|
||
url=url,
|
||
subscription_name=subscription_name,
|
||
platform=platform,
|
||
files_downloaded=files_kept,
|
||
stdout=proc.stdout,
|
||
stderr=proc.stderr,
|
||
return_code=proc.returncode,
|
||
duration_seconds=duration,
|
||
started_at=started_at,
|
||
completed_at=completed_at,
|
||
)
|
||
|
||
# Handle errors
|
||
error_type, error_message = self._categorize_error(
|
||
proc.returncode, proc.stdout, proc.stderr
|
||
)
|
||
|
||
# NO_NEW_CONTENT and TIER_LIMITED are non-failure states — the run completed
|
||
# as well as it could given external constraints (archive up-to-date, or
|
||
# current subscription tier can't see the requested posts).
|
||
success = error_type in (ErrorType.NO_NEW_CONTENT, ErrorType.TIER_LIMITED)
|
||
|
||
# If a non-zero exit also produced quarantined files, prefer the
|
||
# validation-failed signal — it's actionable (the file needs to
|
||
# be redownloaded) and tells the operator the partial-write
|
||
# problem is happening, not just the categorized exit-code error.
|
||
if files_quarantined > 0:
|
||
error_type = ErrorType.VALIDATION_FAILED
|
||
error_message = _summarize_validation_failures(quarantine_failures)
|
||
success = False
|
||
|
||
return DownloadResult(
|
||
success=success,
|
||
url=url,
|
||
subscription_name=subscription_name,
|
||
platform=platform,
|
||
files_downloaded=files_kept if success else 0,
|
||
files_quarantined=files_quarantined,
|
||
quarantined_paths=quarantined_paths,
|
||
stdout=proc.stdout,
|
||
stderr=proc.stderr,
|
||
return_code=proc.returncode,
|
||
error_type=error_type,
|
||
error_message=error_message,
|
||
duration_seconds=duration,
|
||
started_at=started_at,
|
||
completed_at=completed_at,
|
||
)
|
||
|
||
except subprocess.TimeoutExpired:
|
||
duration = time.time() - start_time
|
||
logger.error(f"Download timeout for {subscription_name}/{platform} after {duration:.1f}s")
|
||
|
||
return DownloadResult(
|
||
success=False,
|
||
url=url,
|
||
subscription_name=subscription_name,
|
||
platform=platform,
|
||
error_type=ErrorType.TIMEOUT,
|
||
error_message=f"Download timed out after {source_config.timeout} seconds",
|
||
duration_seconds=duration,
|
||
started_at=started_at,
|
||
completed_at=utcnow().isoformat(),
|
||
)
|
||
|
||
except Exception as e:
|
||
duration = time.time() - start_time
|
||
logger.exception(f"Unexpected error downloading {subscription_name}/{platform}: {e}")
|
||
|
||
return DownloadResult(
|
||
success=False,
|
||
url=url,
|
||
subscription_name=subscription_name,
|
||
platform=platform,
|
||
error_type=ErrorType.UNKNOWN_ERROR,
|
||
error_message=str(e),
|
||
duration_seconds=duration,
|
||
started_at=started_at,
|
||
completed_at=utcnow().isoformat(),
|
||
)
|
||
|
||
finally:
|
||
# Clean up temporary config file
|
||
try:
|
||
Path(temp_config_path).unlink()
|
||
except Exception:
|
||
pass
|
||
|
||
def get_platform_content_types(self, platform: str) -> list[str]:
|
||
"""Get available content types for a platform.
|
||
|
||
Useful for UI to show what options are available.
|
||
"""
|
||
if platform == "patreon":
|
||
return ["images", "image_large", "attachments", "postfile", "content"]
|
||
elif platform == "hentaifoundry":
|
||
return ["all", "pictures", "scraps", "stories"]
|
||
elif platform == "subscribestar":
|
||
return ["all"] # No granular control
|
||
elif platform == "discord":
|
||
return ["all"] # No granular control
|
||
elif platform == "pixiv":
|
||
return ["all", "illusts", "manga", "bookmarks"]
|
||
elif platform == "deviantart":
|
||
return ["all", "gallery", "scraps", "favorites"]
|
||
else:
|
||
return ["all"]
|
||
|
||
def get_default_config_for_platform(self, platform: str) -> dict:
|
||
"""Get the default configuration for a platform.
|
||
|
||
Useful for showing defaults in UI.
|
||
"""
|
||
defaults = self.PLATFORM_DEFAULTS.get(platform, {})
|
||
return {
|
||
"content_types": defaults.get("content_types", ["all"]),
|
||
"directory_pattern": defaults.get("directory"),
|
||
"filename_pattern": defaults.get("filename"),
|
||
"sleep": self._rate_limit,
|
||
"sleep_request": max(0.5, self._rate_limit / 4),
|
||
}
|