"""Gallery-dl wrapper service. This service provides a clean interface for executing gallery-dl downloads with support for per-source configuration overrides. """ import json import logging import subprocess import sys import tempfile import time from dataclasses import dataclass, field from enum import Enum from pathlib import Path from typing import Optional from app.config import get_settings from app.models.base import utcnow logger = logging.getLogger(__name__) class ErrorType(str, Enum): """Error types for categorizing download failures.""" AUTH_ERROR = "auth_error" RATE_LIMITED = "rate_limited" NOT_FOUND = "not_found" ACCESS_DENIED = "access_denied" NETWORK_ERROR = "network_error" NO_NEW_CONTENT = "no_new_content" TIMEOUT = "timeout" HTTP_ERROR = "http_error" UNSUPPORTED_URL = "unsupported_url" UNKNOWN_ERROR = "unknown_error" class ContentType(str, Enum): """Content types that can be downloaded from platforms.""" # Patreon content types IMAGES = "images" IMAGE_LARGE = "image_large" ATTACHMENTS = "attachments" POSTFILE = "postfile" CONTENT = "content" # Embedded content in post body # Generic types ALL = "all" @dataclass class SourceConfig: """Per-source configuration that can override global settings. These settings are stored in the source's metadata field and merged with global defaults when executing downloads. """ # Content types to download (platform-specific) # For Patreon: ["images", "image_large", "attachments", "postfile", "content"] # For others: typically just ["all"] or specific types content_types: list[str] = field(default_factory=lambda: ["all"]) # Rate limiting overrides sleep: Optional[float] = None # Delay between downloads sleep_request: Optional[float] = None # Delay between requests # Directory/filename pattern overrides directory_pattern: Optional[str] = None filename_pattern: Optional[str] = None # Other options skip_existing: bool = True # Skip files already in archive save_metadata: bool = True # Save JSON metadata alongside files timeout: int = 3600 # Download timeout in seconds @classmethod def from_dict(cls, data: dict) -> "SourceConfig": """Create SourceConfig from a dictionary (e.g., from source.metadata).""" return cls( content_types=data.get("content_types", ["all"]), sleep=data.get("sleep"), sleep_request=data.get("sleep_request"), directory_pattern=data.get("directory_pattern"), filename_pattern=data.get("filename_pattern"), skip_existing=data.get("skip_existing", True), save_metadata=data.get("save_metadata", True), timeout=data.get("timeout", 3600), ) def to_dict(self) -> dict: """Convert to dictionary for storage.""" return { "content_types": self.content_types, "sleep": self.sleep, "sleep_request": self.sleep_request, "directory_pattern": self.directory_pattern, "filename_pattern": self.filename_pattern, "skip_existing": self.skip_existing, "save_metadata": self.save_metadata, "timeout": self.timeout, } @dataclass class DownloadResult: """Result of a gallery-dl download execution.""" success: bool url: str subscription_name: str platform: str # Output details files_downloaded: int = 0 stdout: str = "" stderr: str = "" return_code: int = 0 # Error details (if failed) error_type: Optional[ErrorType] = None error_message: Optional[str] = None # Timing duration_seconds: float = 0.0 started_at: Optional[str] = None completed_at: Optional[str] = None class GalleryDLService: """Service for executing gallery-dl downloads. Handles: - Building gallery-dl configuration with per-source overrides - Executing the subprocess with proper error handling - Parsing output and categorizing errors - Managing temporary config files """ # Error classification patterns — used by _categorize_error() AUTH_ERROR_PATTERNS = [ "unauthorized", "login required", "please login", "must be logged in", "authentication required", "authenticationerror", "refresh-token", "session expired", "invalid cookie", "cookies are expired", "cookies have expired", "not logged in", ] RATE_LIMIT_PATTERNS = [ '" 429 ', "rate limit", "too many requests", "ratelimit", "throttl", ] NOT_FOUND_PATTERNS = [ '" 404 ', "error 404", "status: 404", "status code: 404", "404 not found", "page not found", "user not found", "creator not found", "artist not found", "content no longer available", "has been deleted", "account deleted", "profile does not exist", ] GENERIC_NOT_FOUND_PATTERNS = ["does not exist", "no longer available"] NETWORK_ERROR_PATTERNS = [ "timed out", "read timed out", "connect timed out", "connection timed out", "connection refused", "connection reset", "network unreachable", "name resolution failed", "name or service not known", "nodename nor servname provided", "ssl: certificate_verify_failed", "ssl: wrong_version_number", "certificate verify failed", "[errno", ] ACCESS_DENIED_PATTERNS = [ '" 403 ', "error 403", "forbidden", "access denied", "permission denied", "tier required", "pledge required", ] # Platform-specific default content types and options # Directory patterns are relative - subscription_name/platform is prepended automatically PLATFORM_DEFAULTS = { "patreon": { "content_types": ["images", "image_large", "attachments", "postfile", "content"], "directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"], "filename": "{num:>02}_{filename}.{extension}", # Additional options "videos": True, # Download video content "embeds": True, # Download embedded URLs (YouTube, etc.) # Deep crawling options "cursor": True, # Enable cursor-based pagination for complete history }, "subscribestar": { "content_types": ["all"], "directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"], "filename": "{num:>02}_{filename}.{extension}", # Automatic infinite scroll pagination is built-in }, "hentaifoundry": { "content_types": ["all"], # Download all content types "directory": [], # Flat structure within platform folder "filename": "{category}_{index:>03}_{title[:50]}.{extension}", "include": "all", # Include pictures, scraps, stories - complete content # Automatic pagination through all pages (25 items/page) }, "discord": { "content_types": ["all"], "directory": ["{channel[name]}"], "filename": "{date:%Y%m%d}_{id}_{filename}.{extension}", # Additional options "embeds": True, # Download embedded URLs "stickers": True, # Download stickers "reactions": False, # Skip reaction images # Deep crawling options "threads": True, # Include thread content for complete history }, "pixiv": { "content_types": ["all"], "directory": ["{category}"], "filename": "{id}_{title[:50]}_{num:>02}.{extension}", # Content options "ugoira": True, # Download ugoira (animated) as WebM }, "deviantart": { "content_types": ["all"], "directory": [], # Flat structure within platform folder "filename": "{index:>03}_{title[:50]}.{extension}", # Content options "flat": True, # Flatten folder structure "original": True, # Download original quality "mature": True, # Include mature content "metadata": True, # Include metadata }, } def __init__(self, rate_limit: Optional[float] = None): """Initialize the GalleryDL service. Args: rate_limit: Override for download rate limit (seconds between requests). If None, uses environment default. """ self.settings = get_settings() self._rate_limit = rate_limit if rate_limit is not None else self.settings.download_rate_limit self._base_config = self._load_base_config() def _load_base_config(self) -> dict: """Load the base gallery-dl configuration file.""" config_path = Path(self.settings.config_path) / "gallery-dl.conf" if config_path.exists(): try: with open(config_path) as f: return json.load(f) except json.JSONDecodeError as e: logger.warning(f"Invalid gallery-dl.conf, using defaults: {e}") # Return sensible defaults if no config exists return self._get_default_config() def _get_default_config(self) -> dict: """Get default gallery-dl configuration.""" return { "extractor": { "base-directory": self.settings.download_path, "archive": str(Path(self.settings.config_path) / "archive.sqlite3"), "skip": True, "sleep": self._rate_limit, # Faster request rate for pagination/checking (1/4 of rate_limit, min 0.5s) "sleep-request": max(0.5, self._rate_limit / 4), "retries": 3, "timeout": 30.0, "verify": True, "postprocessors": [ { "name": "metadata", "mode": "json", "directory": ".", "filename": "{filename}.json", } ], }, "downloader": { "part": True, "part-directory": str(Path(self.settings.config_path) / "temp"), "retries": 3, "timeout": 120.0, }, "output": { "progress": True, }, } def _build_config_for_source( self, platform: str, source_config: SourceConfig, destination: str, ) -> dict: """Build a merged configuration for a specific source. Merges: base config → platform defaults → source overrides """ config = json.loads(json.dumps(self._base_config)) # Deep copy # Get platform defaults platform_defaults = self.PLATFORM_DEFAULTS.get(platform, {}) # Ensure extractor section exists if "extractor" not in config: config["extractor"] = {} # Apply base directory config["extractor"]["base-directory"] = destination # Apply rate limiting overrides if source_config.sleep is not None: config["extractor"]["sleep"] = source_config.sleep if source_config.sleep_request is not None: config["extractor"]["sleep-request"] = source_config.sleep_request # Apply skip setting config["extractor"]["skip"] = source_config.skip_existing # Configure metadata postprocessor if source_config.save_metadata: config["extractor"]["postprocessors"] = [ { "name": "metadata", "mode": "json", "directory": ".", "filename": "{filename}.json", } ] else: config["extractor"].pop("postprocessors", None) # Build platform-specific section platform_config = {} # Apply all platform defaults first (videos, embeds, stickers, etc.) for key, value in platform_defaults.items(): if key not in ["content_types", "directory", "filename"]: platform_config[key] = value # Content types (platform-specific handling) if platform == "patreon": # Patreon uses "files" array for content types if "all" in source_config.content_types: platform_config["files"] = ["images", "image_large", "attachments", "postfile", "content"] else: platform_config["files"] = source_config.content_types elif platform == "hentaifoundry": # HentaiFoundry uses "include" for content filtering if "pictures" in source_config.content_types or "all" in source_config.content_types: platform_config["include"] = "all" # Directory pattern if source_config.directory_pattern: platform_config["directory"] = source_config.directory_pattern elif "directory" in platform_defaults: platform_config["directory"] = platform_defaults["directory"] # Filename pattern if source_config.filename_pattern: platform_config["filename"] = source_config.filename_pattern elif "filename" in platform_defaults: platform_config["filename"] = platform_defaults["filename"] # Always save metadata at platform level too platform_config["metadata"] = source_config.save_metadata # Add platform config to extractor config["extractor"][platform] = platform_config return config def _categorize_error(self, return_code: int, stdout: str, stderr: str) -> tuple[ErrorType, str]: """Categorize the error based on output and return code. Priority order: 1. Check for skip messages (# prefixed lines or text patterns) → NO_NEW_CONTENT 2. Check for actual errors (auth, rate limit, 404, network, etc.) 3. Fall back to UNKNOWN_ERROR This ordering prevents false positives where debug output matches error patterns. """ combined = f"{stdout} {stderr}".lower() # === PHASE 1: Detect skip/no-new-content scenarios === # Gallery-dl indicates skipped files two ways: # 1. Lines starting with '#' in stdout (e.g., "# /data/downloads/Artist/file.png") # 2. Text patterns like "skipping", "already exists" # Count lines starting with '#' (skip messages) skip_line_count = len([ line for line in stdout.split('\n') if line.strip().startswith('#') ]) # Text-based skip indicators skip_text_indicators = ["skipping", "already exists", "archive"] has_skip_text = any(ind in combined for ind in skip_text_indicators) # Actual error indicators that override skip detection actual_error_indicators = [ "][error]", # gallery-dl error log level: [patreon][error] "exception", # Python exceptions "traceback", # Python tracebacks "download failed", "extraction failed", ] has_actual_error = any(err in combined for err in actual_error_indicators) # Non-fatal error lines gallery-dl logs but recovers from. These shouldn't # block skip detection when the overall download proceeded successfully: # - "not allowed to view" / "unable to get post": per-item paywall warnings # on Patreon /c/user/posts URLs where some posts are paywalled. # - "failed to extract campaign id": logged when the vanity-URL HTML path # fails, but gallery-dl falls back to the /cw/ redirect and # recovers the campaign_id on its own. # - "failed to download": per-item download failure (e.g., a single # Patreon attachment whose media URL expired / was deleted). The # downloader logs [download][error] and moves to the next item. if has_actual_error and (skip_line_count > 0 or has_skip_text): per_item_patterns = [ "not allowed to view", "unable to get post", "failed to extract campaign id", "failed to download", ] error_lines = [ line for line in combined.split('\n') if any(ind in line for ind in actual_error_indicators) ] if error_lines and all( any(p in line for p in per_item_patterns) for line in error_lines ): has_actual_error = False # If we have skip evidence and no real errors, it's no_new_content if (skip_line_count > 0 or has_skip_text) and not has_actual_error: trigger = f"{skip_line_count} skip lines" if skip_line_count > 0 else "skip text indicator" logger.debug(f"Error classified as no_new_content via: {trigger}") return ErrorType.NO_NEW_CONTENT, "No new content to download" # Empty output with low return codes often means nothing to download if return_code == 0 and not stdout.strip(): logger.debug("Error classified as no_new_content via: empty stdout with return code 0") return ErrorType.NO_NEW_CONTENT, "No new content to download" # === PHASE 2: Check for specific error types === # Auth errors - look for error context, not just HTTP codes that appear in debug logs # HTTP debug lines look like: '"GET /path HTTP/1.1" 401 None' - we need to detect actual errors # Check for 401 specifically as an HTTP error response (not in URLs) # Look for patterns like "401" followed by error context or at end of HTTP log line has_401_error = ( '" 401 ' in combined or # HTTP response: "GET /path HTTP/1.1" 401 None "401 unauthorized" in combined or "error 401" in combined or "status: 401" in combined or "status code: 401" in combined ) if has_401_error or any(pattern in combined for pattern in self.AUTH_ERROR_PATTERNS): matched = "401 HTTP response" if has_401_error else next(p for p in self.AUTH_ERROR_PATTERNS if p in combined) logger.debug(f"Error classified as auth_error via pattern: '{matched}'") return ErrorType.AUTH_ERROR, "Authentication failed - cookies may be expired or invalid" # Rate limiting - look for actual rate limit errors if any(pattern in combined for pattern in self.RATE_LIMIT_PATTERNS): matched = next(p for p in self.RATE_LIMIT_PATTERNS if p in combined) logger.debug(f"Error classified as rate_limited via pattern: '{matched}'") return ErrorType.RATE_LIMITED, "Rate limited by server - try increasing sleep time" # 404 Not Found - check for HTTP 404 responses with error context # Be specific to avoid matching "not found" in debug messages like "archive entry not found" # Also require error-level context for generic patterns to avoid false positives has_specific_404 = any(pattern in combined for pattern in self.NOT_FOUND_PATTERNS) has_generic_with_error = any( pattern in combined and "][error]" in combined for pattern in self.GENERIC_NOT_FOUND_PATTERNS ) if has_specific_404 or has_generic_with_error: if has_specific_404: matched = next(p for p in self.NOT_FOUND_PATTERNS if p in combined) logger.debug(f"Error classified as not_found via specific 404 pattern: '{matched}'") else: matched = next(p for p in self.GENERIC_NOT_FOUND_PATTERNS if p in combined) logger.debug(f"Error classified as not_found via generic pattern + error context: '{matched}'") return ErrorType.NOT_FOUND, "URL not found - artist may have changed username or deleted content" # Network errors - use specific patterns to avoid matching config values # (e.g., "timeout": 30.0 in debug output should not trigger network_error) if any(pattern in combined for pattern in self.NETWORK_ERROR_PATTERNS): matched = next(p for p in self.NETWORK_ERROR_PATTERNS if p in combined) logger.debug(f"Error classified as network_error via pattern: '{matched}'") return ErrorType.NETWORK_ERROR, "Network error - check internet connection" # 403 Forbidden / Access denied if any(pattern in combined for pattern in self.ACCESS_DENIED_PATTERNS): matched = next(p for p in self.ACCESS_DENIED_PATTERNS if p in combined) logger.debug(f"Error classified as access_denied via pattern: '{matched}'") return ErrorType.ACCESS_DENIED, "Access denied - may need higher subscription tier" # Generic HTTP errors if "http error" in combined or "httperror" in combined: trigger = "http error" if "http error" in combined else "httperror" logger.debug(f"Error classified as http_error via trigger: '{trigger}'") return ErrorType.HTTP_ERROR, "HTTP error occurred" # Unsupported URL if "no suitable" in combined or "unsupported" in combined or "no extractor" in combined: trigger = next(t for t in ("no suitable", "unsupported", "no extractor") if t in combined) logger.debug(f"Error classified as unsupported_url via trigger: '{trigger}'") return ErrorType.UNSUPPORTED_URL, "URL format not supported by gallery-dl" # If we got here with a non-catastrophic return code, saw skip activity, and no # specific error patterns matched, it's likely just no new content. # gallery-dl exit codes: 1 = some extractions skipped, 4 = no URLs extracted # (common with /c/user/posts Patreon URLs where all content is already archived) if return_code in (1, 4) and (skip_line_count > 0 or has_skip_text) and not has_actual_error: trigger = f"{skip_line_count} skip lines" if skip_line_count > 0 else "skip text indicator" logger.debug(f"Error classified as no_new_content via: rc{return_code} + {trigger}") return ErrorType.NO_NEW_CONTENT, "No new content to download" # Final fallback - genuine unknown error return ErrorType.UNKNOWN_ERROR, f"Unknown error (return code: {return_code})" def _count_downloaded_files(self, stdout: str) -> int: """Count the number of files downloaded from stdout. Gallery-dl writes each downloaded file path to stdout as an absolute path (starting with '/'). Log lines contain '[' prefixes; verbose HTTP lines go to stderr. Counting only lines starting with '/' avoids over-counting log and progress output. """ if not stdout: return 0 return sum(1 for line in stdout.splitlines() if line.strip().startswith("/")) def _compute_run_stats(self, return_code: int, stdout: str, stderr: str) -> dict: """Summarize a gallery-dl run as structured counts for UI/analytics. Returns a flat dict: - exit_code: gallery-dl's process return code - downloaded_count: files actually written (stdout lines starting with '/') - skipped_count: archive-hit skips (stdout '#' lines + stderr 'skipping ' lines) - per_item_failures: [download][error] lines (single-item failures the downloader logs but recovers from) - warning_count: [*][warning] lines """ stdout = stdout or "" stderr = stderr or "" skipped_stdout = sum( 1 for line in stdout.splitlines() if line.strip().startswith("#") ) skipped_stderr = sum( 1 for line in stderr.splitlines() if "] skipping " in line.lower() ) per_item_failures = sum( 1 for line in stderr.splitlines() if "[download][error]" in line.lower() and "failed to download" in line.lower() ) warning_count = sum( 1 for line in stderr.splitlines() if "][warning]" in line.lower() ) return { "exit_code": return_code, "downloaded_count": self._count_downloaded_files(stdout), "skipped_count": skipped_stdout + skipped_stderr, "per_item_failures": per_item_failures, "warning_count": warning_count, } @staticmethod def _extract_errors_warnings(stderr: str) -> str: """Filter stderr down to just [error] / [warning] lines. Gallery-dl's verbose mode sends all logging to stderr including [debug] and urllib3 connection-pool noise. This helper produces the short summary a user actually wants when opening a failed download. """ if not stderr: return "" kept = [ line for line in stderr.splitlines() if "][error]" in line.lower() or "][warning]" in line.lower() ] return "\n".join(kept) @staticmethod def _truncate_log(text: str, max_bytes: int = 500_000) -> str: """Cap a log field at max_bytes by keeping the head and tail. Verbose 10+ minute runs can produce multi-MB stderr blobs. For display and storage, we keep the first and last halves and drop the middle with a marker indicating how many lines were elided. Beginning is useful for context (config, URL); end is useful for the actual failure. """ if not text: return text encoded = text.encode("utf-8") if len(encoded) <= max_bytes: return text half = max_bytes // 2 head = encoded[:half].decode("utf-8", errors="ignore") tail = encoded[-half:].decode("utf-8", errors="ignore") head_lines = head.count("\n") tail_lines = tail.count("\n") total_lines = text.count("\n") elided = max(0, total_lines - head_lines - tail_lines) marker = f"\n\n... [{elided} lines elided, {len(encoded) - max_bytes} bytes] ...\n\n" return head + marker + tail async def download( self, url: str, subscription_name: str, platform: str, source_config: Optional[SourceConfig] = None, cookies_path: Optional[str] = None, auth_token: Optional[str] = None, ) -> DownloadResult: """Execute a gallery-dl download. Args: url: The URL to download from subscription_name: Name of the subscription/artist (used for destination folder) platform: Platform name (patreon, subscribestar, etc.) source_config: Optional per-source configuration overrides cookies_path: Optional path to cookies file auth_token: Optional auth token (for Discord and other token-based platforms) Returns: DownloadResult with success status and details Downloads are saved to: {download_path}/{subscription_name}/{platform}/{post_folders} """ import asyncio start_time = time.time() started_at = utcnow().isoformat() # Use default config if none provided if source_config is None: source_config = SourceConfig() # Build destination path: subscription_name/platform destination = str(Path(self.settings.download_path) / subscription_name / platform) # Build merged configuration config = self._build_config_for_source(platform, source_config, destination) # Add cookies if provided if cookies_path: config["extractor"]["cookies"] = cookies_path # Add auth token for Discord if auth_token and platform == "discord": if "discord" not in config["extractor"]: config["extractor"]["discord"] = {} config["extractor"]["discord"]["token"] = auth_token # Add refresh token for Pixiv if auth_token and platform == "pixiv": if "pixiv" not in config["extractor"]: config["extractor"]["pixiv"] = {} config["extractor"]["pixiv"]["refresh-token"] = auth_token # Write temporary config file with tempfile.NamedTemporaryFile( mode="w", suffix=".json", delete=False, dir=self.settings.config_path, ) as f: json.dump(config, f, indent=2) temp_config_path = f.name try: # Build command cmd = [ sys.executable, "-m", "gallery_dl", "--config", temp_config_path, "--verbose", url, ] logger.info(f"Executing gallery-dl for {subscription_name}/{platform}: {url}") logger.debug(f"Command: {' '.join(cmd)}") # Execute subprocess # Run in thread pool to not block async event loop loop = asyncio.get_running_loop() proc = await loop.run_in_executor( None, lambda: subprocess.run( cmd, capture_output=True, text=True, timeout=source_config.timeout, ), ) duration = time.time() - start_time completed_at = utcnow().isoformat() # Log output if proc.stdout: logger.debug(f"STDOUT for {subscription_name}/{platform}:\n{proc.stdout}") if proc.stderr: logger.debug(f"STDERR for {subscription_name}/{platform}:\n{proc.stderr}") # Determine success if proc.returncode == 0: return DownloadResult( success=True, url=url, subscription_name=subscription_name, platform=platform, files_downloaded=self._count_downloaded_files(proc.stdout), stdout=proc.stdout, stderr=proc.stderr, return_code=proc.returncode, duration_seconds=duration, started_at=started_at, completed_at=completed_at, ) # Handle errors error_type, error_message = self._categorize_error( proc.returncode, proc.stdout, proc.stderr ) # NO_NEW_CONTENT is considered successful success = error_type == ErrorType.NO_NEW_CONTENT return DownloadResult( success=success, url=url, subscription_name=subscription_name, platform=platform, files_downloaded=self._count_downloaded_files(proc.stdout) if success else 0, stdout=proc.stdout, stderr=proc.stderr, return_code=proc.returncode, error_type=error_type, error_message=error_message, duration_seconds=duration, started_at=started_at, completed_at=completed_at, ) except subprocess.TimeoutExpired: duration = time.time() - start_time logger.error(f"Download timeout for {subscription_name}/{platform} after {duration:.1f}s") return DownloadResult( success=False, url=url, subscription_name=subscription_name, platform=platform, error_type=ErrorType.TIMEOUT, error_message=f"Download timed out after {source_config.timeout} seconds", duration_seconds=duration, started_at=started_at, completed_at=utcnow().isoformat(), ) except Exception as e: duration = time.time() - start_time logger.exception(f"Unexpected error downloading {subscription_name}/{platform}: {e}") return DownloadResult( success=False, url=url, subscription_name=subscription_name, platform=platform, error_type=ErrorType.UNKNOWN_ERROR, error_message=str(e), duration_seconds=duration, started_at=started_at, completed_at=utcnow().isoformat(), ) finally: # Clean up temporary config file try: Path(temp_config_path).unlink() except Exception: pass def get_platform_content_types(self, platform: str) -> list[str]: """Get available content types for a platform. Useful for UI to show what options are available. """ if platform == "patreon": return ["images", "image_large", "attachments", "postfile", "content"] elif platform == "hentaifoundry": return ["all", "pictures", "scraps", "stories"] elif platform == "subscribestar": return ["all"] # No granular control elif platform == "discord": return ["all"] # No granular control elif platform == "pixiv": return ["all", "illusts", "manga", "bookmarks"] elif platform == "deviantart": return ["all", "gallery", "scraps", "favorites"] else: return ["all"] def get_default_config_for_platform(self, platform: str) -> dict: """Get the default configuration for a platform. Useful for showing defaults in UI. """ defaults = self.PLATFORM_DEFAULTS.get(platform, {}) return { "content_types": defaults.get("content_types", ["all"]), "directory_pattern": defaults.get("directory"), "filename_pattern": defaults.get("filename"), "sleep": self._rate_limit, "sleep_request": max(0.5, self._rate_limit / 4), }