feat(fc3c): GalleryDLService — subprocess wrap + error classification + validate/quarantine
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,660 @@
|
||||
"""Gallery-dl subprocess wrapper.
|
||||
|
||||
Ported from GallerySubscriber (~/Nextcloud/Projects/GallerySubscriber/
|
||||
backend/app/services/gallery_dl.py). FC adaptations:
|
||||
- `artist_slug` replaces GS's `subscription_name` throughout
|
||||
- Validation uses backend/app/services/file_validator.py
|
||||
- Config dir under `<images_root>/.gallery-dl/`, not a separate config_path
|
||||
- PLATFORM_DEFAULTS scoped to the GS-6 platforms registered in FC-3b
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
|
||||
from .file_validator import is_validatable, validate_file
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ErrorType(str, Enum):
|
||||
AUTH_ERROR = "auth_error"
|
||||
RATE_LIMITED = "rate_limited"
|
||||
NOT_FOUND = "not_found"
|
||||
ACCESS_DENIED = "access_denied"
|
||||
NETWORK_ERROR = "network_error"
|
||||
NO_NEW_CONTENT = "no_new_content"
|
||||
TIER_LIMITED = "tier_limited"
|
||||
TIMEOUT = "timeout"
|
||||
HTTP_ERROR = "http_error"
|
||||
UNSUPPORTED_URL = "unsupported_url"
|
||||
VALIDATION_FAILED = "validation_failed"
|
||||
UNKNOWN_ERROR = "unknown_error"
|
||||
|
||||
|
||||
@dataclass
|
||||
class SourceConfig:
|
||||
content_types: list[str] = field(default_factory=lambda: ["all"])
|
||||
sleep: float | None = None
|
||||
sleep_request: float | None = None
|
||||
directory_pattern: str | None = None
|
||||
filename_pattern: str | None = None
|
||||
skip_existing: bool = True
|
||||
save_metadata: bool = True
|
||||
timeout: int = 3600
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict) -> SourceConfig:
|
||||
return cls(
|
||||
content_types=data.get("content_types", ["all"]),
|
||||
sleep=data.get("sleep"),
|
||||
sleep_request=data.get("sleep_request"),
|
||||
directory_pattern=data.get("directory_pattern"),
|
||||
filename_pattern=data.get("filename_pattern"),
|
||||
skip_existing=data.get("skip_existing", True),
|
||||
save_metadata=data.get("save_metadata", True),
|
||||
timeout=data.get("timeout", 3600),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DownloadResult:
|
||||
success: bool
|
||||
url: str
|
||||
artist_slug: str
|
||||
platform: str
|
||||
files_downloaded: int = 0
|
||||
files_quarantined: int = 0
|
||||
quarantined_paths: list[str] = field(default_factory=list)
|
||||
written_paths: list[str] = field(default_factory=list)
|
||||
stdout: str = ""
|
||||
stderr: str = ""
|
||||
return_code: int = 0
|
||||
error_type: ErrorType | None = None
|
||||
error_message: str | None = None
|
||||
duration_seconds: float = 0.0
|
||||
started_at: str | None = None
|
||||
completed_at: str | None = None
|
||||
|
||||
|
||||
def _summarize_validation_failures(failures: list[dict]) -> str:
|
||||
if not failures:
|
||||
return "Validation failed"
|
||||
reasons: dict[str, int] = {}
|
||||
for f in failures:
|
||||
key = f.get("reason") or "unknown"
|
||||
reasons[key] = reasons.get(key, 0) + 1
|
||||
top_reason, top_count = max(reasons.items(), key=lambda kv: kv[1])
|
||||
n = len(failures)
|
||||
if top_count == n:
|
||||
return f"{n} file{'s' if n != 1 else ''} quarantined: {top_reason}"
|
||||
return f"{n} files quarantined ({top_count}× {top_reason}, mixed)"
|
||||
|
||||
|
||||
class GalleryDLService:
|
||||
"""Service for executing gallery-dl downloads."""
|
||||
|
||||
AUTH_ERROR_PATTERNS = [
|
||||
"unauthorized", "login required", "please login", "must be logged in",
|
||||
"authentication required", "authenticationerror", "refresh-token",
|
||||
"session expired", "invalid cookie", "cookies are expired",
|
||||
"cookies have expired", "not logged in",
|
||||
]
|
||||
RATE_LIMIT_PATTERNS = [
|
||||
'" 429 ', "rate limit", "too many requests", "ratelimit", "throttl",
|
||||
]
|
||||
NOT_FOUND_PATTERNS = [
|
||||
'" 404 ', "error 404", "status: 404", "status code: 404",
|
||||
"404 not found", "page not found", "user not found",
|
||||
"creator not found", "artist not found",
|
||||
"content no longer available", "has been deleted",
|
||||
"account deleted", "profile does not exist",
|
||||
]
|
||||
GENERIC_NOT_FOUND_PATTERNS = ["does not exist", "no longer available"]
|
||||
NETWORK_ERROR_PATTERNS = [
|
||||
"timed out", "read timed out", "connect timed out",
|
||||
"connection timed out", "connection refused", "connection reset",
|
||||
"network unreachable", "name resolution failed",
|
||||
"name or service not known", "nodename nor servname provided",
|
||||
"ssl: certificate_verify_failed", "ssl: wrong_version_number",
|
||||
"certificate verify failed", "[errno",
|
||||
]
|
||||
ACCESS_DENIED_PATTERNS = [
|
||||
'" 403 ', "error 403", "forbidden", "access denied",
|
||||
"permission denied", "tier required", "pledge required",
|
||||
]
|
||||
|
||||
# Per-platform defaults. Lifted from GS — same six platforms FC supports.
|
||||
PLATFORM_DEFAULTS = {
|
||||
"patreon": {
|
||||
"content_types": ["images", "image_large", "attachments", "postfile", "content"],
|
||||
"directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"],
|
||||
"filename": "{num:>02}_{filename}.{extension}",
|
||||
"videos": True,
|
||||
"embeds": True,
|
||||
"cursor": True,
|
||||
},
|
||||
"subscribestar": {
|
||||
"content_types": ["all"],
|
||||
"directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"],
|
||||
"filename": "{num:>02}_{filename}.{extension}",
|
||||
},
|
||||
"hentaifoundry": {
|
||||
"content_types": ["all"],
|
||||
"directory": [],
|
||||
"filename": "{category}_{index:>03}_{title[:50]}.{extension}",
|
||||
"include": "all",
|
||||
},
|
||||
"discord": {
|
||||
"content_types": ["all"],
|
||||
"directory": ["{channel[name]}"],
|
||||
"filename": "{date:%Y%m%d}_{id}_{filename}.{extension}",
|
||||
"embeds": "all",
|
||||
"stickers": True,
|
||||
"reactions": False,
|
||||
"threads": True,
|
||||
},
|
||||
"pixiv": {
|
||||
"content_types": ["all"],
|
||||
"directory": ["{category}"],
|
||||
"filename": "{id}_{title[:50]}_{num:>02}.{extension}",
|
||||
"ugoira": True,
|
||||
},
|
||||
"deviantart": {
|
||||
"content_types": ["all"],
|
||||
"directory": [],
|
||||
"filename": "{index:>03}_{title[:50]}.{extension}",
|
||||
"flat": True,
|
||||
"original": True,
|
||||
"mature": True,
|
||||
"metadata": True,
|
||||
},
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
images_root: Path,
|
||||
rate_limit: float = 3.0,
|
||||
validate_files: bool = True,
|
||||
):
|
||||
self.images_root = Path(images_root)
|
||||
self._rate_limit = rate_limit
|
||||
self._validate_files = validate_files
|
||||
self._config_dir = self.images_root / ".gallery-dl"
|
||||
self._config_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
(self._config_dir / "temp").mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
|
||||
def _get_default_config(self) -> dict:
|
||||
config = {
|
||||
"extractor": {
|
||||
"base-directory": str(self.images_root),
|
||||
"archive": str(self._config_dir / "archive.sqlite3"),
|
||||
"skip": True,
|
||||
"sleep": self._rate_limit,
|
||||
"sleep-request": max(0.5, self._rate_limit / 4),
|
||||
"retries": 3,
|
||||
"timeout": 30.0,
|
||||
"verify": True,
|
||||
"postprocessors": [
|
||||
{
|
||||
"name": "metadata",
|
||||
"mode": "json",
|
||||
"directory": ".",
|
||||
"filename": "{filename}.json",
|
||||
}
|
||||
],
|
||||
},
|
||||
"downloader": {
|
||||
"part": True,
|
||||
"part-directory": str(self._config_dir / "temp"),
|
||||
"retries": 3,
|
||||
"timeout": 120.0,
|
||||
},
|
||||
"output": {"progress": True},
|
||||
}
|
||||
for platform_name, defaults in self.PLATFORM_DEFAULTS.items():
|
||||
config["extractor"][platform_name] = {
|
||||
key: value for key, value in defaults.items() if key != "content_types"
|
||||
}
|
||||
return config
|
||||
|
||||
def _build_config_for_source(
|
||||
self,
|
||||
platform: str,
|
||||
source_config: SourceConfig,
|
||||
artist_slug: str,
|
||||
) -> dict:
|
||||
config = json.loads(json.dumps(self._get_default_config())) # deep copy
|
||||
|
||||
destination = str(self.images_root / artist_slug / platform)
|
||||
config["extractor"]["base-directory"] = destination
|
||||
|
||||
if source_config.sleep is not None:
|
||||
config["extractor"]["sleep"] = source_config.sleep
|
||||
if source_config.sleep_request is not None:
|
||||
config["extractor"]["sleep-request"] = source_config.sleep_request
|
||||
config["extractor"]["skip"] = source_config.skip_existing
|
||||
|
||||
if source_config.save_metadata:
|
||||
config["extractor"]["postprocessors"] = [
|
||||
{
|
||||
"name": "metadata",
|
||||
"mode": "json",
|
||||
"directory": ".",
|
||||
"filename": "{filename}.json",
|
||||
}
|
||||
]
|
||||
else:
|
||||
config["extractor"].pop("postprocessors", None)
|
||||
|
||||
platform_section = config["extractor"].setdefault(platform, {})
|
||||
|
||||
if platform == "patreon":
|
||||
if "all" in source_config.content_types:
|
||||
platform_section["files"] = [
|
||||
"images", "image_large", "attachments", "postfile", "content",
|
||||
]
|
||||
else:
|
||||
platform_section["files"] = source_config.content_types
|
||||
elif platform == "hentaifoundry":
|
||||
if "pictures" in source_config.content_types or "all" in source_config.content_types:
|
||||
platform_section["include"] = "all"
|
||||
|
||||
if source_config.directory_pattern:
|
||||
platform_section["directory"] = source_config.directory_pattern
|
||||
if source_config.filename_pattern:
|
||||
platform_section["filename"] = source_config.filename_pattern
|
||||
|
||||
platform_section["metadata"] = source_config.save_metadata
|
||||
|
||||
return config
|
||||
|
||||
def _categorize_error(
|
||||
self, return_code: int, stdout: str, stderr: str,
|
||||
) -> tuple[ErrorType, str]:
|
||||
combined = f"{stdout} {stderr}".lower()
|
||||
|
||||
skip_line_count = len([
|
||||
line for line in stdout.split("\n") if line.strip().startswith("#")
|
||||
])
|
||||
skip_text_indicators = ["skipping", "already exists", "archive"]
|
||||
has_skip_text = any(ind in combined for ind in skip_text_indicators)
|
||||
|
||||
actual_error_indicators = [
|
||||
"][error]", "exception", "traceback",
|
||||
"download failed", "extraction failed",
|
||||
]
|
||||
per_item_patterns = [
|
||||
"not allowed to view", "unable to get post",
|
||||
"failed to extract campaign id", "failed to download",
|
||||
"cannot import yt-dlp", "cannot import youtube-dl",
|
||||
]
|
||||
all_lines = combined.split("\n")
|
||||
error_lines = [
|
||||
line for line in all_lines
|
||||
if any(ind in line for ind in actual_error_indicators)
|
||||
]
|
||||
per_item_error_lines = [
|
||||
line for line in error_lines
|
||||
if any(p in line for p in per_item_patterns)
|
||||
]
|
||||
source_level_error_lines = [
|
||||
line for line in error_lines
|
||||
if line not in set(per_item_error_lines)
|
||||
]
|
||||
has_actual_error = bool(source_level_error_lines)
|
||||
|
||||
if per_item_error_lines:
|
||||
per_item_set = set(per_item_error_lines)
|
||||
combined = "\n".join(line for line in all_lines if line not in per_item_set)
|
||||
|
||||
if (skip_line_count > 0 or has_skip_text) and not has_actual_error:
|
||||
return ErrorType.NO_NEW_CONTENT, "No new content to download"
|
||||
|
||||
if return_code == 0 and not stdout.strip():
|
||||
return ErrorType.NO_NEW_CONTENT, "No new content to download"
|
||||
|
||||
has_401_error = (
|
||||
'" 401 ' in combined
|
||||
or "401 unauthorized" in combined
|
||||
or "error 401" in combined
|
||||
or "status: 401" in combined
|
||||
or "status code: 401" in combined
|
||||
)
|
||||
if has_401_error or any(p in combined for p in self.AUTH_ERROR_PATTERNS):
|
||||
return ErrorType.AUTH_ERROR, "Authentication failed — cookies may be expired or invalid"
|
||||
|
||||
if any(p in combined for p in self.RATE_LIMIT_PATTERNS):
|
||||
return ErrorType.RATE_LIMITED, "Rate limited by server — try increasing sleep time"
|
||||
|
||||
has_specific_404 = any(p in combined for p in self.NOT_FOUND_PATTERNS)
|
||||
has_generic_with_error = any(
|
||||
p in combined and "][error]" in combined
|
||||
for p in self.GENERIC_NOT_FOUND_PATTERNS
|
||||
)
|
||||
if has_specific_404 or has_generic_with_error:
|
||||
return ErrorType.NOT_FOUND, "URL not found — artist may have changed username or deleted content"
|
||||
|
||||
if any(p in combined for p in self.NETWORK_ERROR_PATTERNS):
|
||||
return ErrorType.NETWORK_ERROR, "Network error — check internet connection"
|
||||
|
||||
if any(p in combined for p in self.ACCESS_DENIED_PATTERNS):
|
||||
return ErrorType.ACCESS_DENIED, "Access denied — may need higher subscription tier"
|
||||
|
||||
if "http error" in combined or "httperror" in combined:
|
||||
return ErrorType.HTTP_ERROR, "HTTP error occurred"
|
||||
|
||||
if "no suitable" in combined or "unsupported" in combined or "no extractor" in combined:
|
||||
return ErrorType.UNSUPPORTED_URL, "URL format not supported by gallery-dl"
|
||||
|
||||
if return_code in (1, 4) and (skip_line_count > 0 or has_skip_text) and not has_actual_error:
|
||||
return ErrorType.NO_NEW_CONTENT, "No new content to download"
|
||||
|
||||
if return_code in (1, 4) and not has_actual_error:
|
||||
tier_gated_lines = [
|
||||
line for line in combined.split("\n")
|
||||
if "][warning]" in line and "not allowed to view post" in line
|
||||
]
|
||||
if tier_gated_lines:
|
||||
count = len(tier_gated_lines)
|
||||
return (
|
||||
ErrorType.TIER_LIMITED,
|
||||
f"Subscription tier does not grant access to {count} post{'s' if count != 1 else ''}",
|
||||
)
|
||||
|
||||
return ErrorType.UNKNOWN_ERROR, f"Unknown error (return code: {return_code})"
|
||||
|
||||
def _count_downloaded_files(self, stdout: str) -> int:
|
||||
if not stdout:
|
||||
return 0
|
||||
return sum(1 for line in stdout.splitlines() if line.strip().startswith("/"))
|
||||
|
||||
def _written_paths(self, stdout: str) -> list[Path]:
|
||||
if not stdout:
|
||||
return []
|
||||
return [
|
||||
Path(stripped)
|
||||
for line in stdout.splitlines()
|
||||
if (stripped := line.strip()).startswith("/")
|
||||
]
|
||||
|
||||
def _validate_and_quarantine(
|
||||
self,
|
||||
written_paths: list[Path],
|
||||
artist_slug: str,
|
||||
platform: str,
|
||||
url: str,
|
||||
) -> tuple[list[str], list[dict]]:
|
||||
quarantined_relpaths: list[str] = []
|
||||
failures: list[dict] = []
|
||||
if not written_paths:
|
||||
return quarantined_relpaths, failures
|
||||
|
||||
quarantine_root = (
|
||||
self.images_root / "_quarantine" / artist_slug / platform
|
||||
)
|
||||
|
||||
for path in written_paths:
|
||||
if not is_validatable(path):
|
||||
continue
|
||||
try:
|
||||
result = validate_file(path)
|
||||
except Exception as exc:
|
||||
log.warning("Validator raised on %s: %s", path, exc)
|
||||
continue
|
||||
if result.ok:
|
||||
continue
|
||||
try:
|
||||
quarantine_root.mkdir(parents=True, exist_ok=True)
|
||||
dest = quarantine_root / path.name
|
||||
counter = 1
|
||||
while dest.exists():
|
||||
dest = quarantine_root / f"{path.stem}.{counter}{path.suffix}"
|
||||
counter += 1
|
||||
path.rename(dest)
|
||||
sidecar = dest.with_suffix(dest.suffix + ".quarantine.json")
|
||||
sidecar.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"original_path": str(path),
|
||||
"source_url": url,
|
||||
"artist_slug": artist_slug,
|
||||
"platform": platform,
|
||||
"format": result.format,
|
||||
"reason": result.reason,
|
||||
"size": result.size,
|
||||
"quarantined_at": datetime.now(UTC).isoformat(),
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
except OSError as exc:
|
||||
log.error("Failed to quarantine %s: %s. File left in place.", path, exc)
|
||||
continue
|
||||
|
||||
log.warning(
|
||||
"Quarantined corrupt file: %s → %s (%s: %s)",
|
||||
path, dest, result.format, result.reason,
|
||||
)
|
||||
quarantined_relpaths.append(str(dest))
|
||||
failures.append(
|
||||
{
|
||||
"path": str(path),
|
||||
"format": result.format,
|
||||
"reason": result.reason,
|
||||
"size": result.size,
|
||||
}
|
||||
)
|
||||
return quarantined_relpaths, failures
|
||||
|
||||
def _compute_run_stats(self, return_code: int, stdout: str, stderr: str) -> dict:
|
||||
stdout = stdout or ""
|
||||
stderr = stderr or ""
|
||||
|
||||
skipped_stdout = sum(
|
||||
1 for line in stdout.splitlines() if line.strip().startswith("#")
|
||||
)
|
||||
skipped_stderr = sum(
|
||||
1 for line in stderr.splitlines() if "] skipping " in line.lower()
|
||||
)
|
||||
per_item_failures = sum(
|
||||
1 for line in stderr.splitlines()
|
||||
if "[download][error]" in line.lower() and "failed to download" in line.lower()
|
||||
)
|
||||
warning_count = sum(
|
||||
1 for line in stderr.splitlines() if "][warning]" in line.lower()
|
||||
)
|
||||
tier_gated_count = sum(
|
||||
1 for line in stderr.splitlines()
|
||||
if "][warning]" in line.lower() and "not allowed to view post" in line.lower()
|
||||
)
|
||||
|
||||
return {
|
||||
"exit_code": return_code,
|
||||
"downloaded_count": self._count_downloaded_files(stdout),
|
||||
"skipped_count": skipped_stdout + skipped_stderr,
|
||||
"per_item_failures": per_item_failures,
|
||||
"warning_count": warning_count,
|
||||
"tier_gated_count": tier_gated_count,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _extract_errors_warnings(stderr: str) -> str:
|
||||
if not stderr:
|
||||
return ""
|
||||
kept = [
|
||||
line for line in stderr.splitlines()
|
||||
if "][error]" in line.lower() or "][warning]" in line.lower()
|
||||
]
|
||||
return "\n".join(kept)
|
||||
|
||||
@staticmethod
|
||||
def _truncate_log(text: str, max_bytes: int = 500_000) -> str:
|
||||
if not text:
|
||||
return text
|
||||
encoded = text.encode("utf-8")
|
||||
if len(encoded) <= max_bytes:
|
||||
return text
|
||||
half = max_bytes // 2
|
||||
head = encoded[:half].decode("utf-8", errors="ignore")
|
||||
tail = encoded[-half:].decode("utf-8", errors="ignore")
|
||||
head_lines = head.count("\n")
|
||||
tail_lines = tail.count("\n")
|
||||
total_lines = text.count("\n")
|
||||
elided = max(0, total_lines - head_lines - tail_lines)
|
||||
marker = f"\n\n... [{elided} lines elided, {len(encoded) - max_bytes} bytes] ...\n\n"
|
||||
return head + marker + tail
|
||||
|
||||
async def download(
|
||||
self,
|
||||
url: str,
|
||||
artist_slug: str,
|
||||
platform: str,
|
||||
source_config: SourceConfig | None = None,
|
||||
cookies_path: str | None = None,
|
||||
auth_token: str | None = None,
|
||||
) -> DownloadResult:
|
||||
start_time = time.time()
|
||||
started_at = datetime.now(UTC).isoformat()
|
||||
|
||||
if source_config is None:
|
||||
source_config = SourceConfig()
|
||||
|
||||
config = self._build_config_for_source(platform, source_config, artist_slug)
|
||||
|
||||
if cookies_path:
|
||||
config["extractor"]["cookies"] = cookies_path
|
||||
if auth_token and platform == "discord":
|
||||
config["extractor"].setdefault("discord", {})
|
||||
config["extractor"]["discord"]["token"] = auth_token
|
||||
if auth_token and platform == "pixiv":
|
||||
config["extractor"].setdefault("pixiv", {})
|
||||
config["extractor"]["pixiv"]["refresh-token"] = auth_token
|
||||
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w", suffix=".json", delete=False, dir=str(self._config_dir),
|
||||
) as fh:
|
||||
json.dump(config, fh, indent=2)
|
||||
temp_config_path = fh.name
|
||||
|
||||
try:
|
||||
cmd = [
|
||||
sys.executable, "-m", "gallery_dl",
|
||||
"--config", temp_config_path, "--verbose", url,
|
||||
]
|
||||
log.info("Executing gallery-dl for %s/%s: %s", artist_slug, platform, url)
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
proc = await loop.run_in_executor(
|
||||
None,
|
||||
lambda: subprocess.run(
|
||||
cmd, capture_output=True, text=True,
|
||||
timeout=source_config.timeout,
|
||||
),
|
||||
)
|
||||
|
||||
duration = time.time() - start_time
|
||||
completed_at = datetime.now(UTC).isoformat()
|
||||
|
||||
quarantined_paths: list[str] = []
|
||||
quarantine_failures: list[dict] = []
|
||||
if self._validate_files:
|
||||
quarantined_paths, quarantine_failures = self._validate_and_quarantine(
|
||||
written_paths=self._written_paths(proc.stdout),
|
||||
artist_slug=artist_slug,
|
||||
platform=platform,
|
||||
url=url,
|
||||
)
|
||||
|
||||
written_paths = [
|
||||
str(p) for p in self._written_paths(proc.stdout)
|
||||
if str(p) not in set(quarantined_paths)
|
||||
]
|
||||
files_written = self._count_downloaded_files(proc.stdout)
|
||||
files_quarantined = len(quarantined_paths)
|
||||
files_kept = max(0, files_written - files_quarantined)
|
||||
|
||||
if proc.returncode == 0:
|
||||
if files_quarantined > 0:
|
||||
return DownloadResult(
|
||||
success=False, url=url, artist_slug=artist_slug, platform=platform,
|
||||
files_downloaded=files_kept,
|
||||
files_quarantined=files_quarantined,
|
||||
quarantined_paths=quarantined_paths,
|
||||
written_paths=written_paths,
|
||||
stdout=proc.stdout, stderr=proc.stderr,
|
||||
return_code=proc.returncode,
|
||||
error_type=ErrorType.VALIDATION_FAILED,
|
||||
error_message=_summarize_validation_failures(quarantine_failures),
|
||||
duration_seconds=duration,
|
||||
started_at=started_at, completed_at=completed_at,
|
||||
)
|
||||
return DownloadResult(
|
||||
success=True, url=url, artist_slug=artist_slug, platform=platform,
|
||||
files_downloaded=files_kept,
|
||||
written_paths=written_paths,
|
||||
stdout=proc.stdout, stderr=proc.stderr,
|
||||
return_code=proc.returncode,
|
||||
duration_seconds=duration,
|
||||
started_at=started_at, completed_at=completed_at,
|
||||
)
|
||||
|
||||
error_type, error_message = self._categorize_error(
|
||||
proc.returncode, proc.stdout, proc.stderr
|
||||
)
|
||||
success = error_type in (ErrorType.NO_NEW_CONTENT, ErrorType.TIER_LIMITED)
|
||||
|
||||
if files_quarantined > 0:
|
||||
error_type = ErrorType.VALIDATION_FAILED
|
||||
error_message = _summarize_validation_failures(quarantine_failures)
|
||||
success = False
|
||||
|
||||
return DownloadResult(
|
||||
success=success, url=url, artist_slug=artist_slug, platform=platform,
|
||||
files_downloaded=files_kept if success else 0,
|
||||
files_quarantined=files_quarantined,
|
||||
quarantined_paths=quarantined_paths,
|
||||
written_paths=written_paths,
|
||||
stdout=proc.stdout, stderr=proc.stderr,
|
||||
return_code=proc.returncode,
|
||||
error_type=error_type, error_message=error_message,
|
||||
duration_seconds=duration,
|
||||
started_at=started_at, completed_at=completed_at,
|
||||
)
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
duration = time.time() - start_time
|
||||
log.error("Download timeout for %s/%s after %.1fs", artist_slug, platform, duration)
|
||||
return DownloadResult(
|
||||
success=False, url=url, artist_slug=artist_slug, platform=platform,
|
||||
error_type=ErrorType.TIMEOUT,
|
||||
error_message=f"Download timed out after {source_config.timeout} seconds",
|
||||
duration_seconds=duration,
|
||||
started_at=started_at,
|
||||
completed_at=datetime.now(UTC).isoformat(),
|
||||
)
|
||||
except Exception as exc:
|
||||
duration = time.time() - start_time
|
||||
log.exception("Unexpected error downloading %s/%s: %s", artist_slug, platform, exc)
|
||||
return DownloadResult(
|
||||
success=False, url=url, artist_slug=artist_slug, platform=platform,
|
||||
error_type=ErrorType.UNKNOWN_ERROR, error_message=str(exc),
|
||||
duration_seconds=duration,
|
||||
started_at=started_at,
|
||||
completed_at=datetime.now(UTC).isoformat(),
|
||||
)
|
||||
finally:
|
||||
try:
|
||||
Path(temp_config_path).unlink()
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,151 @@
|
||||
"""GalleryDLService tests — subprocess is mocked so no real gallery-dl runs."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from backend.app.services.gallery_dl import (
|
||||
ErrorType,
|
||||
GalleryDLService,
|
||||
SourceConfig,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def gdl(tmp_path):
|
||||
return GalleryDLService(
|
||||
images_root=tmp_path / "images",
|
||||
rate_limit=3.0,
|
||||
validate_files=False,
|
||||
)
|
||||
|
||||
|
||||
def _proc(stdout="", stderr="", returncode=0):
|
||||
return SimpleNamespace(stdout=stdout, stderr=stderr, returncode=returncode)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_happy_path(gdl, monkeypatch):
|
||||
out_path = gdl.images_root / "alice" / "patreon" / "post1" / "img.jpg"
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_path.write_bytes(b"\xff\xd8\xff\xe0" + b"\x00" * 32 + b"\xff\xd9")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"backend.app.services.gallery_dl.subprocess.run",
|
||||
lambda *a, **k: _proc(stdout=str(out_path) + "\n", returncode=0),
|
||||
)
|
||||
result = await gdl.download(
|
||||
url="https://patreon.com/alice",
|
||||
artist_slug="alice",
|
||||
platform="patreon",
|
||||
source_config=SourceConfig(),
|
||||
)
|
||||
assert result.success is True
|
||||
assert result.files_downloaded == 1
|
||||
assert result.error_type is None
|
||||
assert str(out_path) in result.written_paths
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_no_new_content(gdl, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"backend.app.services.gallery_dl.subprocess.run",
|
||||
lambda *a, **k: _proc(stdout="# /images/alice/patreon/old.jpg\n", returncode=0),
|
||||
)
|
||||
result = await gdl.download(
|
||||
url="https://patreon.com/alice", artist_slug="alice", platform="patreon",
|
||||
source_config=SourceConfig(),
|
||||
)
|
||||
assert result.success is True
|
||||
assert result.files_downloaded == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_auth_error_categorization(gdl, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"backend.app.services.gallery_dl.subprocess.run",
|
||||
lambda *a, **k: _proc(
|
||||
stdout="", stderr='[patreon][error] "GET /api/foo HTTP/1.1" 401 None\n',
|
||||
returncode=1,
|
||||
),
|
||||
)
|
||||
result = await gdl.download(
|
||||
url="https://patreon.com/alice", artist_slug="alice", platform="patreon",
|
||||
source_config=SourceConfig(),
|
||||
)
|
||||
assert result.success is False
|
||||
assert result.error_type == ErrorType.AUTH_ERROR
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_timeout(gdl, monkeypatch):
|
||||
import subprocess as sp
|
||||
|
||||
def _raise(*a, **k):
|
||||
raise sp.TimeoutExpired(cmd="gallery-dl", timeout=10)
|
||||
|
||||
monkeypatch.setattr("backend.app.services.gallery_dl.subprocess.run", _raise)
|
||||
result = await gdl.download(
|
||||
url="https://patreon.com/alice", artist_slug="alice", platform="patreon",
|
||||
source_config=SourceConfig(timeout=10),
|
||||
)
|
||||
assert result.success is False
|
||||
assert result.error_type == ErrorType.TIMEOUT
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validation_quarantines_truncated_files(tmp_path, monkeypatch):
|
||||
gdl = GalleryDLService(
|
||||
images_root=tmp_path / "images", rate_limit=3.0, validate_files=True,
|
||||
)
|
||||
bad = gdl.images_root / "alice" / "patreon" / "post" / "bad.jpg"
|
||||
bad.parent.mkdir(parents=True, exist_ok=True)
|
||||
bad.write_bytes(b"\xff\xd8\xff" + b"\x00" * 100) # SOI but no EOI
|
||||
|
||||
monkeypatch.setattr(
|
||||
"backend.app.services.gallery_dl.subprocess.run",
|
||||
lambda *a, **k: _proc(stdout=str(bad) + "\n", returncode=0),
|
||||
)
|
||||
result = await gdl.download(
|
||||
url="https://patreon.com/alice", artist_slug="alice", platform="patreon",
|
||||
source_config=SourceConfig(),
|
||||
)
|
||||
assert result.files_quarantined == 1
|
||||
assert len(result.quarantined_paths) == 1
|
||||
assert not bad.exists()
|
||||
assert result.error_type == ErrorType.VALIDATION_FAILED
|
||||
|
||||
|
||||
def test_compute_run_stats(gdl):
|
||||
stats = gdl._compute_run_stats(
|
||||
return_code=0,
|
||||
stdout="/path/a.jpg\n/path/b.jpg\n# /path/c.jpg\n",
|
||||
stderr="[patreon][warning] foo\n[patreon][download][error] failed to download /x\n",
|
||||
)
|
||||
assert stats["downloaded_count"] == 2
|
||||
assert stats["skipped_count"] == 1
|
||||
assert stats["warning_count"] >= 1
|
||||
assert stats["per_item_failures"] == 1
|
||||
|
||||
|
||||
def test_extract_errors_warnings(gdl):
|
||||
stderr = (
|
||||
"[patreon][debug] foo\n[patreon][error] bar\n"
|
||||
"[urllib3][debug] baz\n[patreon][warning] qux\n"
|
||||
)
|
||||
result = gdl._extract_errors_warnings(stderr)
|
||||
assert "[error]" in result
|
||||
assert "[warning]" in result
|
||||
assert "[debug]" not in result
|
||||
|
||||
|
||||
def test_truncate_log_caps_large_text(gdl):
|
||||
huge = "line\n" * 200_000
|
||||
result = gdl._truncate_log(huge, max_bytes=10_000)
|
||||
assert len(result.encode()) < 11_000
|
||||
assert "lines elided" in result
|
||||
|
||||
|
||||
def test_truncate_log_passes_short_text(gdl):
|
||||
text = "small\n"
|
||||
assert gdl._truncate_log(text) == text
|
||||
Reference in New Issue
Block a user