bvandeusen/GallerySubscriber
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

refactor: extract error patterns to constants, add debug logging, fix rc1 no_new_content

Browse Source 
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Bryan Van Deusen
2026-03-18 23:37:04 -04:00
parent 6432210a36
commit dc146f4a16
1 changed files with 94 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/services/gallery_dl.py
+94 -69
View File
@@ -135,6 +135,70 @@ class GalleryDLService:
- Managing temporary config files - Managing temporary config files
""" """
# Error classification patterns — used by _categorize_error()
AUTH_ERROR_PATTERNS = [
"unauthorized",
"login required",
"please login",
"must be logged in",
"authentication required",
"authenticationerror",
"refresh-token",
"session expired",
"invalid cookie",
"cookies are expired",
"cookies have expired",
"not logged in",
]
RATE_LIMIT_PATTERNS = [
'" 429 ',
"rate limit",
"too many requests",
"ratelimit",
"throttl",
]
NOT_FOUND_PATTERNS = [
'" 404 ',
"error 404",
"status: 404",
"status code: 404",
"404 not found",
"page not found",
"user not found",
"creator not found",
"artist not found",
"content no longer available",
"has been deleted",
"account.*deleted",
"profile.*not.*exist",
]
GENERIC_NOT_FOUND_PATTERNS = ["does not exist", "no longer available"]
NETWORK_ERROR_PATTERNS = [
"timed out",
"read timed out",
"connect timed out",
"connection timed out",
"connection refused",
"connection reset",
"network unreachable",
"name resolution failed",
"name or service not known",
"nodename nor servname provided",
"ssl: certificate_verify_failed",
"ssl: wrong_version_number",
"certificate verify failed",
"[errno",
]
ACCESS_DENIED_PATTERNS = [
'" 403 ',
"error 403",
"forbidden",
"access denied",
"permission denied",
"tier required",
"pledge required",
]
# Platform-specific default content types and options # Platform-specific default content types and options
# Directory patterns are relative - subscription_name/platform is prepended automatically # Directory patterns are relative - subscription_name/platform is prepended automatically
PLATFORM_DEFAULTS = { PLATFORM_DEFAULTS = {
@@ -388,30 +452,19 @@ class GalleryDLService:
# If we have skip evidence and no real errors, it's no_new_content # If we have skip evidence and no real errors, it's no_new_content
if (skip_line_count > 0 or has_skip_text) and not has_actual_error: if (skip_line_count > 0 or has_skip_text) and not has_actual_error:
trigger = f"{skip_line_count} skip lines" if skip_line_count > 0 else "skip text indicator"
logger.debug(f"Error classified as no_new_content via: {trigger}")
return ErrorType.NO_NEW_CONTENT, "No new content to download" return ErrorType.NO_NEW_CONTENT, "No new content to download"
# Empty output with low return codes often means nothing to download # Empty output with low return codes often means nothing to download
if return_code in (0, 1) and not stdout.strip(): if return_code == 0 and not stdout.strip():
logger.debug("Error classified as no_new_content via: empty stdout with return code 0")
return ErrorType.NO_NEW_CONTENT, "No new content to download" return ErrorType.NO_NEW_CONTENT, "No new content to download"
# === PHASE 2: Check for specific error types === # === PHASE 2: Check for specific error types ===
# Auth errors - look for error context, not just HTTP codes that appear in debug logs # Auth errors - look for error context, not just HTTP codes that appear in debug logs
# HTTP debug lines look like: '"GET /path HTTP/1.1" 401 None' - we need to detect actual errors # HTTP debug lines look like: '"GET /path HTTP/1.1" 401 None' - we need to detect actual errors
auth_error_patterns = [
"unauthorized",
"login required",
"please login",
"must be logged in",
"authentication required",
"authenticationerror",
"refresh-token", # Pixiv OAuth
"session expired",
"invalid cookie",
"cookies are expired",
"cookies have expired",
"not logged in",
]
# Check for 401 specifically as an HTTP error response (not in URLs) # Check for 401 specifically as an HTTP error response (not in URLs)
# Look for patterns like "401" followed by error context or at end of HTTP log line # Look for patterns like "401" followed by error context or at end of HTTP log line
has_401_error = ( has_401_error = (
@@ -421,87 +474,57 @@ class GalleryDLService:
"status: 401" in combined or "status: 401" in combined or
"status code: 401" in combined "status code: 401" in combined
) )
if has_401_error or any(pattern in combined for pattern in auth_error_patterns): if has_401_error or any(pattern in combined for pattern in self.AUTH_ERROR_PATTERNS):
matched = "401 HTTP response" if has_401_error else next(p for p in self.AUTH_ERROR_PATTERNS if p in combined)
logger.debug(f"Error classified as auth_error via pattern: '{matched}'")
return ErrorType.AUTH_ERROR, "Authentication failed - cookies may be expired or invalid" return ErrorType.AUTH_ERROR, "Authentication failed - cookies may be expired or invalid"
# Rate limiting - look for actual rate limit errors # Rate limiting - look for actual rate limit errors
rate_limit_patterns = [ if any(pattern in combined for pattern in self.RATE_LIMIT_PATTERNS):
'" 429 ', # HTTP response matched = next(p for p in self.RATE_LIMIT_PATTERNS if p in combined)
"rate limit", logger.debug(f"Error classified as rate_limited via pattern: '{matched}'")
"too many requests",
"ratelimit",
"throttl",
]
if any(pattern in combined for pattern in rate_limit_patterns):
return ErrorType.RATE_LIMITED, "Rate limited by server - try increasing sleep time" return ErrorType.RATE_LIMITED, "Rate limited by server - try increasing sleep time"
# 404 Not Found - check for HTTP 404 responses with error context # 404 Not Found - check for HTTP 404 responses with error context
# Be specific to avoid matching "not found" in debug messages like "archive entry not found" # Be specific to avoid matching "not found" in debug messages like "archive entry not found"
not_found_patterns = [
'" 404 ', # HTTP response log line
"error 404", # Explicit error 404
"status: 404", # Status code indicator
"status code: 404", # Status code indicator
"404 not found", # Standard 404 message
"page not found", # Common 404 page text
"user not found", # User doesn't exist
"creator not found",
"artist not found",
"content no longer available",
"has been deleted",
"account.*deleted",
"profile.*not.*exist",
]
# Also require error-level context for generic patterns to avoid false positives # Also require error-level context for generic patterns to avoid false positives
generic_not_found = ["does not exist", "no longer available"] has_specific_404 = any(pattern in combined for pattern in self.NOT_FOUND_PATTERNS)
has_specific_404 = any(pattern in combined for pattern in not_found_patterns)
has_generic_with_error = any( has_generic_with_error = any(
pattern in combined and "][error]" in combined pattern in combined and "][error]" in combined
for pattern in generic_not_found for pattern in self.GENERIC_NOT_FOUND_PATTERNS
) )
if has_specific_404 or has_generic_with_error: if has_specific_404 or has_generic_with_error:
if has_specific_404:
matched = next(p for p in self.NOT_FOUND_PATTERNS if p in combined)
logger.debug(f"Error classified as not_found via specific 404 pattern: '{matched}'")
else:
matched = next(p for p in self.GENERIC_NOT_FOUND_PATTERNS if p in combined)
logger.debug(f"Error classified as not_found via generic pattern + error context: '{matched}'")
return ErrorType.NOT_FOUND, "URL not found - artist may have changed username or deleted content" return ErrorType.NOT_FOUND, "URL not found - artist may have changed username or deleted content"
# Network errors - use specific patterns to avoid matching config values # Network errors - use specific patterns to avoid matching config values
# (e.g., "timeout": 30.0 in debug output should not trigger network_error) # (e.g., "timeout": 30.0 in debug output should not trigger network_error)
network_error_patterns = [ if any(pattern in combined for pattern in self.NETWORK_ERROR_PATTERNS):
"timed out", # Actual timeout occurred matched = next(p for p in self.NETWORK_ERROR_PATTERNS if p in combined)
"read timed out", # urllib3/requests timeout logger.debug(f"Error classified as network_error via pattern: '{matched}'")
"connect timed out", # Connection timeout
"connection timed out",
"connection refused",
"connection reset",
"network unreachable",
"name resolution failed",
"name or service not known",
"nodename nor servname provided",
"ssl: certificate_verify_failed",
"ssl: wrong_version_number",
"certificate verify failed",
"[errno", # Socket errors like [Errno 111]
]
if any(pattern in combined for pattern in network_error_patterns):
return ErrorType.NETWORK_ERROR, "Network error - check internet connection" return ErrorType.NETWORK_ERROR, "Network error - check internet connection"
# 403 Forbidden / Access denied # 403 Forbidden / Access denied
access_denied_patterns = [ if any(pattern in combined for pattern in self.ACCESS_DENIED_PATTERNS):
'" 403 ', # HTTP response matched = next(p for p in self.ACCESS_DENIED_PATTERNS if p in combined)
"error 403", logger.debug(f"Error classified as access_denied via pattern: '{matched}'")
"forbidden",
"access denied",
"permission denied",
"tier required",
"pledge required",
]
if any(pattern in combined for pattern in access_denied_patterns):
return ErrorType.ACCESS_DENIED, "Access denied - may need higher subscription tier" return ErrorType.ACCESS_DENIED, "Access denied - may need higher subscription tier"
# Generic HTTP errors # Generic HTTP errors
if "http error" in combined or "httperror" in combined: if "http error" in combined or "httperror" in combined:
trigger = "http error" if "http error" in combined else "httperror"
logger.debug(f"Error classified as http_error via trigger: '{trigger}'")
return ErrorType.HTTP_ERROR, "HTTP error occurred" return ErrorType.HTTP_ERROR, "HTTP error occurred"
# Unsupported URL # Unsupported URL
if "no suitable" in combined or "unsupported" in combined or "no extractor" in combined: if "no suitable" in combined or "unsupported" in combined or "no extractor" in combined:
trigger = next(t for t in ("no suitable", "unsupported", "no extractor") if t in combined)
logger.debug(f"Error classified as unsupported_url via trigger: '{trigger}'")
return ErrorType.UNSUPPORTED_URL, "URL format not supported by gallery-dl" return ErrorType.UNSUPPORTED_URL, "URL format not supported by gallery-dl"
# If we got here with a non-catastrophic return code, saw skip activity, and no # If we got here with a non-catastrophic return code, saw skip activity, and no
@@ -509,6 +532,8 @@ class GalleryDLService:
# gallery-dl exit codes: 1 = some extractions skipped, 4 = no URLs extracted # gallery-dl exit codes: 1 = some extractions skipped, 4 = no URLs extracted
# (common with /c/user/posts Patreon URLs where all content is already archived) # (common with /c/user/posts Patreon URLs where all content is already archived)
if return_code in (1, 4) and (skip_line_count > 0 or has_skip_text) and not has_actual_error: if return_code in (1, 4) and (skip_line_count > 0 or has_skip_text) and not has_actual_error:
trigger = f"{skip_line_count} skip lines" if skip_line_count > 0 else "skip text indicator"
logger.debug(f"Error classified as no_new_content via: rc{return_code} + {trigger}")
return ErrorType.NO_NEW_CONTENT, "No new content to download" return ErrorType.NO_NEW_CONTENT, "No new content to download"
# Final fallback - genuine unknown error # Final fallback - genuine unknown error