diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py index b1bc31a..ef5e328 100644 --- a/backend/app/services/gallery_dl.py +++ b/backend/app/services/gallery_dl.py @@ -135,6 +135,70 @@ class GalleryDLService: - Managing temporary config files """ + # Error classification patterns — used by _categorize_error() + AUTH_ERROR_PATTERNS = [ + "unauthorized", + "login required", + "please login", + "must be logged in", + "authentication required", + "authenticationerror", + "refresh-token", + "session expired", + "invalid cookie", + "cookies are expired", + "cookies have expired", + "not logged in", + ] + RATE_LIMIT_PATTERNS = [ + '" 429 ', + "rate limit", + "too many requests", + "ratelimit", + "throttl", + ] + NOT_FOUND_PATTERNS = [ + '" 404 ', + "error 404", + "status: 404", + "status code: 404", + "404 not found", + "page not found", + "user not found", + "creator not found", + "artist not found", + "content no longer available", + "has been deleted", + "account.*deleted", + "profile.*not.*exist", + ] + GENERIC_NOT_FOUND_PATTERNS = ["does not exist", "no longer available"] + NETWORK_ERROR_PATTERNS = [ + "timed out", + "read timed out", + "connect timed out", + "connection timed out", + "connection refused", + "connection reset", + "network unreachable", + "name resolution failed", + "name or service not known", + "nodename nor servname provided", + "ssl: certificate_verify_failed", + "ssl: wrong_version_number", + "certificate verify failed", + "[errno", + ] + ACCESS_DENIED_PATTERNS = [ + '" 403 ', + "error 403", + "forbidden", + "access denied", + "permission denied", + "tier required", + "pledge required", + ] + # Platform-specific default content types and options # Directory patterns are relative - subscription_name/platform is prepended automatically PLATFORM_DEFAULTS = { @@ -388,30 +452,19 @@ class GalleryDLService: # If we have skip evidence and no real errors, it's no_new_content if (skip_line_count > 0 or has_skip_text) and not has_actual_error: + trigger = f"{skip_line_count} skip lines" if skip_line_count > 0 else "skip text indicator" + logger.debug(f"Error classified as no_new_content via: {trigger}") return ErrorType.NO_NEW_CONTENT, "No new content to download" # Empty output with low return codes often means nothing to download - if return_code in (0, 1) and not stdout.strip(): + if return_code == 0 and not stdout.strip(): + logger.debug("Error classified as no_new_content via: empty stdout with return code 0") return ErrorType.NO_NEW_CONTENT, "No new content to download" # === PHASE 2: Check for specific error types === # Auth errors - look for error context, not just HTTP codes that appear in debug logs # HTTP debug lines look like: '"GET /path HTTP/1.1" 401 None' - we need to detect actual errors - auth_error_patterns = [ - "unauthorized", - "login required", - "please login", - "must be logged in", - "authentication required", - "authenticationerror", - "refresh-token", # Pixiv OAuth - "session expired", - "invalid cookie", - "cookies are expired", - "cookies have expired", - "not logged in", - ] # Check for 401 specifically as an HTTP error response (not in URLs) # Look for patterns like "401" followed by error context or at end of HTTP log line has_401_error = ( @@ -421,87 +474,57 @@ class GalleryDLService: "status: 401" in combined or "status code: 401" in combined ) - if has_401_error or any(pattern in combined for pattern in auth_error_patterns): + if has_401_error or any(pattern in combined for pattern in self.AUTH_ERROR_PATTERNS): + matched = "401 HTTP response" if has_401_error else next(p for p in self.AUTH_ERROR_PATTERNS if p in combined) + logger.debug(f"Error classified as auth_error via pattern: '{matched}'") return ErrorType.AUTH_ERROR, "Authentication failed - cookies may be expired or invalid" # Rate limiting - look for actual rate limit errors - rate_limit_patterns = [ - '" 429 ', # HTTP response - "rate limit", - "too many requests", - "ratelimit", - "throttl", - ] - if any(pattern in combined for pattern in rate_limit_patterns): + if any(pattern in combined for pattern in self.RATE_LIMIT_PATTERNS): + matched = next(p for p in self.RATE_LIMIT_PATTERNS if p in combined) + logger.debug(f"Error classified as rate_limited via pattern: '{matched}'") return ErrorType.RATE_LIMITED, "Rate limited by server - try increasing sleep time" # 404 Not Found - check for HTTP 404 responses with error context # Be specific to avoid matching "not found" in debug messages like "archive entry not found" - not_found_patterns = [ - '" 404 ', # HTTP response log line - "error 404", # Explicit error 404 - "status: 404", # Status code indicator - "status code: 404", # Status code indicator - "404 not found", # Standard 404 message - "page not found", # Common 404 page text - "user not found", # User doesn't exist - "creator not found", - "artist not found", - "content no longer available", - "has been deleted", - "account.*deleted", - "profile.*not.*exist", - ] # Also require error-level context for generic patterns to avoid false positives - generic_not_found = ["does not exist", "no longer available"] - has_specific_404 = any(pattern in combined for pattern in not_found_patterns) + has_specific_404 = any(pattern in combined for pattern in self.NOT_FOUND_PATTERNS) has_generic_with_error = any( pattern in combined and "][error]" in combined - for pattern in generic_not_found + for pattern in self.GENERIC_NOT_FOUND_PATTERNS ) if has_specific_404 or has_generic_with_error: + if has_specific_404: + matched = next(p for p in self.NOT_FOUND_PATTERNS if p in combined) + logger.debug(f"Error classified as not_found via specific 404 pattern: '{matched}'") + else: + matched = next(p for p in self.GENERIC_NOT_FOUND_PATTERNS if p in combined) + logger.debug(f"Error classified as not_found via generic pattern + error context: '{matched}'") return ErrorType.NOT_FOUND, "URL not found - artist may have changed username or deleted content" # Network errors - use specific patterns to avoid matching config values # (e.g., "timeout": 30.0 in debug output should not trigger network_error) - network_error_patterns = [ - "timed out", # Actual timeout occurred - "read timed out", # urllib3/requests timeout - "connect timed out", # Connection timeout - "connection timed out", - "connection refused", - "connection reset", - "network unreachable", - "name resolution failed", - "name or service not known", - "nodename nor servname provided", - "ssl: certificate_verify_failed", - "ssl: wrong_version_number", - "certificate verify failed", - "[errno", # Socket errors like [Errno 111] - ] - if any(pattern in combined for pattern in network_error_patterns): + if any(pattern in combined for pattern in self.NETWORK_ERROR_PATTERNS): + matched = next(p for p in self.NETWORK_ERROR_PATTERNS if p in combined) + logger.debug(f"Error classified as network_error via pattern: '{matched}'") return ErrorType.NETWORK_ERROR, "Network error - check internet connection" # 403 Forbidden / Access denied - access_denied_patterns = [ - '" 403 ', # HTTP response - "error 403", - "forbidden", - "access denied", - "permission denied", - "tier required", - "pledge required", - ] - if any(pattern in combined for pattern in access_denied_patterns): + if any(pattern in combined for pattern in self.ACCESS_DENIED_PATTERNS): + matched = next(p for p in self.ACCESS_DENIED_PATTERNS if p in combined) + logger.debug(f"Error classified as access_denied via pattern: '{matched}'") return ErrorType.ACCESS_DENIED, "Access denied - may need higher subscription tier" # Generic HTTP errors if "http error" in combined or "httperror" in combined: + trigger = "http error" if "http error" in combined else "httperror" + logger.debug(f"Error classified as http_error via trigger: '{trigger}'") return ErrorType.HTTP_ERROR, "HTTP error occurred" # Unsupported URL if "no suitable" in combined or "unsupported" in combined or "no extractor" in combined: + trigger = next(t for t in ("no suitable", "unsupported", "no extractor") if t in combined) + logger.debug(f"Error classified as unsupported_url via trigger: '{trigger}'") return ErrorType.UNSUPPORTED_URL, "URL format not supported by gallery-dl" # If we got here with a non-catastrophic return code, saw skip activity, and no @@ -509,6 +532,8 @@ class GalleryDLService: # gallery-dl exit codes: 1 = some extractions skipped, 4 = no URLs extracted # (common with /c/user/posts Patreon URLs where all content is already archived) if return_code in (1, 4) and (skip_line_count > 0 or has_skip_text) and not has_actual_error: + trigger = f"{skip_line_count} skip lines" if skip_line_count > 0 else "skip text indicator" + logger.debug(f"Error classified as no_new_content via: rc{return_code} + {trigger}") return ErrorType.NO_NEW_CONTENT, "No new content to download" # Final fallback - genuine unknown error