ongoing polish in queue handling

backend/app/services/gallery_dl.py

@@ -173,8 +173,15 @@ class GalleryDLService:
         },
     }
 
-    def __init__(self):
+    def __init__(self, rate_limit: Optional[float] = None):
+        """Initialize the GalleryDL service.
+
+        Args:
+            rate_limit: Override for download rate limit (seconds between requests).
+                       If None, uses environment default.
+        """
         self.settings = get_settings()
+        self._rate_limit = rate_limit if rate_limit is not None else self.settings.download_rate_limit
         self._base_config = self._load_base_config()
 
     def _load_base_config(self) -> dict:
@@ -198,8 +205,8 @@ class GalleryDLService:
                 "base-directory": self.settings.download_path,
                 "archive": str(Path(self.settings.config_path) / "archive.sqlite3"),
                 "skip": True,
-                "sleep": self.settings.download_rate_limit,
-                "sleep-request": max(1.0, self.settings.download_rate_limit / 2),
+                "sleep": self._rate_limit,
+                "sleep-request": max(1.0, self._rate_limit / 2),
                 "retries": 3,
                 "timeout": 30.0,
                 "verify": True,
@@ -311,19 +318,23 @@ class GalleryDLService:
         """Categorize the error based on output and return code."""
         combined = f"{stdout} {stderr}".lower()
 
-        # Check for "no new content" first - skipped files with return code 1 is normal
-        if return_code == 1:
-            # Check if output indicates content was processed (skipped = already downloaded)
-            if "skipping" in combined or "skip" in combined:
-                return ErrorType.NO_NEW_CONTENT, "No new content to download"
-            # Empty output with return code 1 often means nothing to download
-            if not stdout.strip() or "no results" in combined:
+        # Check for "no new content" FIRST for any non-zero return code
+        # Gallery-dl returns 1 when all files are skipped or no new content found
+        # This check must run before other error checks to avoid false positives
+        skip_indicators = ["skipping", "skip", "already exists", "archive"]
+        if any(indicator in combined for indicator in skip_indicators):
+            # If we see skip messages and no actual error messages, it's just "no new content"
+            actual_errors = ["error", "failed", "exception", "traceback"]
+            if not any(err in combined for err in actual_errors):
                 return ErrorType.NO_NEW_CONTENT, "No new content to download"
 
-        # Auth errors - be more specific to avoid false positives from debug URLs
-        # Look for actual error messages, not just keywords that appear in URLs
-        auth_patterns = [
-            "401",
+        # Empty output with low return codes often means nothing to download
+        if return_code in (0, 1) and not stdout.strip():
+            return ErrorType.NO_NEW_CONTENT, "No new content to download"
+
+        # Auth errors - look for error context, not just HTTP codes that appear in debug logs
+        # HTTP debug lines look like: '"GET /path HTTP/1.1" 401 None' - we need to detect actual errors
+        auth_error_patterns = [
             "unauthorized",
             "login required",
             "please login",
@@ -332,28 +343,84 @@ class GalleryDLService:
             "session expired",
             "invalid cookie",
             "cookies are expired",
+            "cookies have expired",
+            "not logged in",
         ]
-        if any(pattern in combined for pattern in auth_patterns):
+        # Check for 401 specifically as an HTTP error response (not in URLs)
+        # Look for patterns like "401" followed by error context or at end of HTTP log line
+        has_401_error = (
+            '" 401 ' in combined or  # HTTP response: "GET /path HTTP/1.1" 401 None
+            "401 unauthorized" in combined or
+            "error 401" in combined or
+            "status: 401" in combined or
+            "status code: 401" in combined
+        )
+        if has_401_error or any(pattern in combined for pattern in auth_error_patterns):
             return ErrorType.AUTH_ERROR, "Authentication failed - cookies may be expired or invalid"
 
-        if "429" in combined or "rate limit" in combined or "too many requests" in combined:
+        # Rate limiting - look for actual rate limit errors
+        rate_limit_patterns = [
+            '" 429 ',  # HTTP response
+            "rate limit",
+            "too many requests",
+            "ratelimit",
+            "throttl",
+        ]
+        if any(pattern in combined for pattern in rate_limit_patterns):
             return ErrorType.RATE_LIMITED, "Rate limited by server - try increasing sleep time"
 
-        if "404" in combined or "not found" in combined:
+        # 404 Not Found - check for HTTP 404 responses, not just "404" anywhere
+        not_found_patterns = [
+            '" 404 ',  # HTTP response
+            "error 404",
+            "status: 404",
+            "not found",
+            "does not exist",
+            "no longer available",
+        ]
+        if any(pattern in combined for pattern in not_found_patterns):
             return ErrorType.NOT_FOUND, "URL not found - artist may have changed username or deleted content"
 
-        if "timeout" in combined or "connection" in combined or "network" in combined:
+        # Network errors
+        network_error_patterns = [
+            "timeout",
+            "timed out",
+            "connection refused",
+            "connection reset",
+            "network unreachable",
+            "name resolution",
+            "dns",
+            "ssl error",
+            "certificate verify",
+        ]
+        if any(pattern in combined for pattern in network_error_patterns):
             return ErrorType.NETWORK_ERROR, "Network error - check internet connection"
 
-        if "403" in combined or "forbidden" in combined or "access denied" in combined:
+        # 403 Forbidden / Access denied
+        access_denied_patterns = [
+            '" 403 ',  # HTTP response
+            "error 403",
+            "forbidden",
+            "access denied",
+            "permission denied",
+            "tier required",
+            "pledge required",
+        ]
+        if any(pattern in combined for pattern in access_denied_patterns):
             return ErrorType.ACCESS_DENIED, "Access denied - may need higher subscription tier"
 
-        if "http error" in combined:
+        # Generic HTTP errors
+        if "http error" in combined or "httperror" in combined:
             return ErrorType.HTTP_ERROR, "HTTP error occurred"
 
-        if "no suitable" in combined or "unsupported" in combined:
+        # Unsupported URL
+        if "no suitable" in combined or "unsupported" in combined or "no extractor" in combined:
             return ErrorType.UNSUPPORTED_URL, "URL format not supported by gallery-dl"
 
+        # If we got here with return code 1 and saw some activity, it might just be no new content
+        if return_code == 1:
+            return ErrorType.NO_NEW_CONTENT, "No new content to download"
+
         return ErrorType.UNKNOWN_ERROR, f"Unknown error (return code: {return_code})"
 
     def _count_downloaded_files(self, stdout: str) -> int:
@@ -575,6 +642,6 @@ class GalleryDLService:
             "content_types": defaults.get("content_types", ["all"]),
             "directory_pattern": defaults.get("directory"),
             "filename_pattern": defaults.get("filename"),
-            "sleep": self.settings.download_rate_limit,
-            "sleep_request": max(1.0, self.settings.download_rate_limit / 2),
+            "sleep": self._rate_limit,
+            "sleep_request": max(1.0, self._rate_limit / 2),
         }