FabledCurator/backend/app/services/gallery_dl.py

"""Gallery-dl subprocess wrapper.

Ported from GallerySubscriber (~/Nextcloud/Projects/GallerySubscriber/
backend/app/services/gallery_dl.py). FC adaptations:
  - `artist_slug` replaces GS's `subscription_name` throughout
  - Validation uses backend/app/services/file_validator.py
  - Config dir under `<images_root>/.gallery-dl/`, not a separate config_path
  - PLATFORM_DEFAULTS scoped to the GS-6 platforms registered in FC-3b
"""

from __future__ import annotations

import asyncio
import json
import logging
import subprocess
import sys
import tempfile
import time
from dataclasses import dataclass, field
from datetime import UTC, datetime
from enum import StrEnum
from pathlib import Path

from .file_validator import is_validatable, validate_file

log = logging.getLogger(__name__)


class ErrorType(StrEnum):
    AUTH_ERROR = "auth_error"
    RATE_LIMITED = "rate_limited"
    NOT_FOUND = "not_found"
    ACCESS_DENIED = "access_denied"
    NETWORK_ERROR = "network_error"
    NO_NEW_CONTENT = "no_new_content"
    TIER_LIMITED = "tier_limited"
    TIMEOUT = "timeout"
    HTTP_ERROR = "http_error"
    UNSUPPORTED_URL = "unsupported_url"
    VALIDATION_FAILED = "validation_failed"
    UNKNOWN_ERROR = "unknown_error"


# Pinned to download_source's Celery soft_time_limit (900s, see
# tasks/download.py:32). Anything larger and Celery kills the task before
# subprocess.run can raise TimeoutExpired — leaving the DownloadEvent
# stranded for the recovery sweep instead of capturing a clean timeout
# error. Per-source bumps live in source.config_overrides for legitimately
# long syncs. Operator-confirmed 2026-05-30 (~40-min hang investigation).
_DEFAULT_GDL_TIMEOUT_SECONDS = 900


@dataclass
class SourceConfig:
    content_types: list[str] = field(default_factory=lambda: ["all"])
    sleep: float | None = None
    sleep_request: float | None = None
    directory_pattern: str | None = None
    filename_pattern: str | None = None
    skip_existing: bool = True
    save_metadata: bool = True
    timeout: int = _DEFAULT_GDL_TIMEOUT_SECONDS

    @classmethod
    def from_dict(cls, data: dict) -> SourceConfig:
        return cls(
            content_types=data.get("content_types", ["all"]),
            sleep=data.get("sleep"),
            sleep_request=data.get("sleep_request"),
            directory_pattern=data.get("directory_pattern"),
            filename_pattern=data.get("filename_pattern"),
            skip_existing=data.get("skip_existing", True),
            save_metadata=data.get("save_metadata", True),
            timeout=data.get("timeout", _DEFAULT_GDL_TIMEOUT_SECONDS),
        )


@dataclass
class DownloadResult:
    success: bool
    url: str
    artist_slug: str
    platform: str
    files_downloaded: int = 0
    files_quarantined: int = 0
    quarantined_paths: list[str] = field(default_factory=list)
    written_paths: list[str] = field(default_factory=list)
    stdout: str = ""
    stderr: str = ""
    return_code: int = 0
    error_type: ErrorType | None = None
    error_message: str | None = None
    duration_seconds: float = 0.0
    started_at: str | None = None
    completed_at: str | None = None


def _summarize_validation_failures(failures: list[dict]) -> str:
    if not failures:
        return "Validation failed"
    reasons: dict[str, int] = {}
    for f in failures:
        key = f.get("reason") or "unknown"
        reasons[key] = reasons.get(key, 0) + 1
    top_reason, top_count = max(reasons.items(), key=lambda kv: kv[1])
    n = len(failures)
    if top_count == n:
        return f"{n} file{'s' if n != 1 else ''} quarantined: {top_reason}"
    return f"{n} files quarantined ({top_count}× {top_reason}, mixed)"


class GalleryDLService:
    """Service for executing gallery-dl downloads."""

    AUTH_ERROR_PATTERNS = [
        "unauthorized", "login required", "please login", "must be logged in",
        "authentication required", "authenticationerror", "refresh-token",
        "session expired", "invalid cookie", "cookies are expired",
        "cookies have expired", "not logged in",
    ]
    RATE_LIMIT_PATTERNS = [
        '" 429 ', "rate limit", "too many requests", "ratelimit", "throttl",
    ]
    NOT_FOUND_PATTERNS = [
        '" 404 ', "error 404", "status: 404", "status code: 404",
        "404 not found", "page not found", "user not found",
        "creator not found", "artist not found",
        "content no longer available", "has been deleted",
        "account deleted", "profile does not exist",
    ]
    GENERIC_NOT_FOUND_PATTERNS = ["does not exist", "no longer available"]
    NETWORK_ERROR_PATTERNS = [
        "timed out", "read timed out", "connect timed out",
        "connection timed out", "connection refused", "connection reset",
        "network unreachable", "name resolution failed",
        "name or service not known", "nodename nor servname provided",
        "ssl: certificate_verify_failed", "ssl: wrong_version_number",
        "certificate verify failed", "[errno",
    ]
    ACCESS_DENIED_PATTERNS = [
        '" 403 ', "error 403", "forbidden", "access denied",
        "permission denied", "tier required", "pledge required",
    ]

    # Per-platform defaults. Lifted from GS — same six platforms FC supports.
    PLATFORM_DEFAULTS = {
        "patreon": {
            "content_types": ["images", "image_large", "attachments", "postfile", "content"],
            "directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"],
            "filename": "{num:>02}_{filename}.{extension}",
            "videos": True,
            "embeds": True,
            "cursor": True,
        },
        "subscribestar": {
            "content_types": ["all"],
            "directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"],
            "filename": "{num:>02}_{filename}.{extension}",
        },
        "hentaifoundry": {
            "content_types": ["all"],
            "directory": [],
            "filename": "{category}_{index:>03}_{title[:50]}.{extension}",
            "include": "all",
        },
        "discord": {
            "content_types": ["all"],
            "directory": ["{channel[name]}"],
            "filename": "{date:%Y%m%d}_{id}_{filename}.{extension}",
            "embeds": "all",
            "stickers": True,
            "reactions": False,
            "threads": True,
        },
        "pixiv": {
            "content_types": ["all"],
            "directory": ["{category}"],
            "filename": "{id}_{title[:50]}_{num:>02}.{extension}",
            "ugoira": True,
        },
        "deviantart": {
            "content_types": ["all"],
            "directory": [],
            "filename": "{index:>03}_{title[:50]}.{extension}",
            "flat": True,
            "original": True,
            "mature": True,
            "metadata": True,
        },
    }

    def __init__(
        self,
        images_root: Path,
        rate_limit: float = 3.0,
        validate_files: bool = True,
    ):
        self.images_root = Path(images_root)
        self._rate_limit = rate_limit
        self._validate_files = validate_files
        self._config_dir = self.images_root / ".gallery-dl"
        self._config_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
        (self._config_dir / "temp").mkdir(parents=True, exist_ok=True, mode=0o700)

    def _get_default_config(self) -> dict:
        config = {
            "extractor": {
                "base-directory": str(self.images_root),
                "archive": str(self._config_dir / "archive.sqlite3"),
                "skip": True,
                "sleep": self._rate_limit,
                "sleep-request": max(0.5, self._rate_limit / 4),
                "retries": 3,
                "timeout": 30.0,
                "verify": True,
                "postprocessors": [
                    {
                        "name": "metadata",
                        "mode": "json",
                        "directory": ".",
                        "filename": "{filename}.json",
                    }
                ],
            },
            "downloader": {
                "part": True,
                "part-directory": str(self._config_dir / "temp"),
                "retries": 3,
                "timeout": 120.0,
            },
            "output": {"progress": True},
        }
        for platform_name, defaults in self.PLATFORM_DEFAULTS.items():
            config["extractor"][platform_name] = {
                key: value for key, value in defaults.items() if key != "content_types"
            }
        return config

    def _build_config_for_source(
        self,
        platform: str,
        source_config: SourceConfig,
        artist_slug: str,
    ) -> dict:
        config = json.loads(json.dumps(self._get_default_config()))  # deep copy

        destination = str(self.images_root / artist_slug / platform)
        config["extractor"]["base-directory"] = destination

        if source_config.sleep is not None:
            config["extractor"]["sleep"] = source_config.sleep
        if source_config.sleep_request is not None:
            config["extractor"]["sleep-request"] = source_config.sleep_request
        config["extractor"]["skip"] = source_config.skip_existing

        if source_config.save_metadata:
            config["extractor"]["postprocessors"] = [
                {
                    "name": "metadata",
                    "mode": "json",
                    "directory": ".",
                    "filename": "{filename}.json",
                }
            ]
        else:
            config["extractor"].pop("postprocessors", None)

        platform_section = config["extractor"].setdefault(platform, {})

        if platform == "patreon":
            if "all" in source_config.content_types:
                platform_section["files"] = [
                    "images", "image_large", "attachments", "postfile", "content",
                ]
            else:
                platform_section["files"] = source_config.content_types
        elif platform == "hentaifoundry":
            if "pictures" in source_config.content_types or "all" in source_config.content_types:
                platform_section["include"] = "all"

        if source_config.directory_pattern:
            platform_section["directory"] = source_config.directory_pattern
        if source_config.filename_pattern:
            platform_section["filename"] = source_config.filename_pattern

        platform_section["metadata"] = source_config.save_metadata

        return config

    def _categorize_error(
        self, return_code: int, stdout: str, stderr: str,
    ) -> tuple[ErrorType, str]:
        combined = f"{stdout} {stderr}".lower()

        skip_line_count = len([
            line for line in stdout.split("\n") if line.strip().startswith("#")
        ])
        skip_text_indicators = ["skipping", "already exists", "archive"]
        has_skip_text = any(ind in combined for ind in skip_text_indicators)

        actual_error_indicators = [
            "][error]", "exception", "traceback",
            "download failed", "extraction failed",
        ]
        per_item_patterns = [
            "not allowed to view", "unable to get post",
            "failed to extract campaign id", "failed to download",
            "cannot import yt-dlp", "cannot import youtube-dl",
        ]
        all_lines = combined.split("\n")
        error_lines = [
            line for line in all_lines
            if any(ind in line for ind in actual_error_indicators)
        ]
        per_item_error_lines = [
            line for line in error_lines
            if any(p in line for p in per_item_patterns)
        ]
        source_level_error_lines = [
            line for line in error_lines
            if line not in set(per_item_error_lines)
        ]
        has_actual_error = bool(source_level_error_lines)

        if per_item_error_lines:
            per_item_set = set(per_item_error_lines)
            combined = "\n".join(line for line in all_lines if line not in per_item_set)

        if (skip_line_count > 0 or has_skip_text) and not has_actual_error:
            return ErrorType.NO_NEW_CONTENT, "No new content to download"

        if return_code == 0 and not stdout.strip():
            return ErrorType.NO_NEW_CONTENT, "No new content to download"

        has_401_error = (
            '" 401 ' in combined
            or "401 unauthorized" in combined
            or "error 401" in combined
            or "status: 401" in combined
            or "status code: 401" in combined
        )
        if has_401_error or any(p in combined for p in self.AUTH_ERROR_PATTERNS):
            return ErrorType.AUTH_ERROR, "Authentication failed — cookies may be expired or invalid"

        if any(p in combined for p in self.RATE_LIMIT_PATTERNS):
            return ErrorType.RATE_LIMITED, "Rate limited by server — try increasing sleep time"

        has_specific_404 = any(p in combined for p in self.NOT_FOUND_PATTERNS)
        has_generic_with_error = any(
            p in combined and "][error]" in combined
            for p in self.GENERIC_NOT_FOUND_PATTERNS
        )
        if has_specific_404 or has_generic_with_error:
            return ErrorType.NOT_FOUND, "URL not found — artist may have changed username or deleted content"

        if any(p in combined for p in self.NETWORK_ERROR_PATTERNS):
            return ErrorType.NETWORK_ERROR, "Network error — check internet connection"

        if any(p in combined for p in self.ACCESS_DENIED_PATTERNS):
            return ErrorType.ACCESS_DENIED, "Access denied — may need higher subscription tier"

        if "http error" in combined or "httperror" in combined:
            return ErrorType.HTTP_ERROR, "HTTP error occurred"

        if "no suitable" in combined or "unsupported" in combined or "no extractor" in combined:
            return ErrorType.UNSUPPORTED_URL, "URL format not supported by gallery-dl"

        if return_code in (1, 4) and (skip_line_count > 0 or has_skip_text) and not has_actual_error:
            return ErrorType.NO_NEW_CONTENT, "No new content to download"

        if return_code in (1, 4) and not has_actual_error:
            tier_gated_lines = [
                line for line in combined.split("\n")
                if "][warning]" in line and "not allowed to view post" in line
            ]
            if tier_gated_lines:
                count = len(tier_gated_lines)
                return (
                    ErrorType.TIER_LIMITED,
                    f"Subscription tier does not grant access to {count} post{'s' if count != 1 else ''}",
                )

        return ErrorType.UNKNOWN_ERROR, f"Unknown error (return code: {return_code})"

    def _count_downloaded_files(self, stdout: str) -> int:
        if not stdout:
            return 0
        return sum(1 for line in stdout.splitlines() if line.strip().startswith("/"))

    def _written_paths(self, stdout: str) -> list[Path]:
        if not stdout:
            return []
        return [
            Path(stripped)
            for line in stdout.splitlines()
            if (stripped := line.strip()).startswith("/")
        ]

    def _validate_and_quarantine(
        self,
        written_paths: list[Path],
        artist_slug: str,
        platform: str,
        url: str,
    ) -> tuple[list[str], list[dict]]:
        quarantined_relpaths: list[str] = []
        failures: list[dict] = []
        if not written_paths:
            return quarantined_relpaths, failures

        quarantine_root = (
            self.images_root / "_quarantine" / artist_slug / platform
        )

        for path in written_paths:
            if not is_validatable(path):
                continue
            try:
                result = validate_file(path)
            except Exception as exc:
                log.warning("Validator raised on %s: %s", path, exc)
                continue
            if result.ok:
                continue
            try:
                quarantine_root.mkdir(parents=True, exist_ok=True)
                dest = quarantine_root / path.name
                counter = 1
                while dest.exists():
                    dest = quarantine_root / f"{path.stem}.{counter}{path.suffix}"
                    counter += 1
                path.rename(dest)
                sidecar = dest.with_suffix(dest.suffix + ".quarantine.json")
                sidecar.write_text(
                    json.dumps(
                        {
                            "original_path": str(path),
                            "source_url": url,
                            "artist_slug": artist_slug,
                            "platform": platform,
                            "format": result.format,
                            "reason": result.reason,
                            "size": result.size,
                            "quarantined_at": datetime.now(UTC).isoformat(),
                        },
                        indent=2,
                    )
                )
            except OSError as exc:
                log.error("Failed to quarantine %s: %s. File left in place.", path, exc)
                continue

            log.warning(
                "Quarantined corrupt file: %s → %s (%s: %s)",
                path, dest, result.format, result.reason,
            )
            quarantined_relpaths.append(str(dest))
            failures.append(
                {
                    "path": str(path),
                    "format": result.format,
                    "reason": result.reason,
                    "size": result.size,
                }
            )
        return quarantined_relpaths, failures

    def _compute_run_stats(self, return_code: int, stdout: str, stderr: str) -> dict:
        stdout = stdout or ""
        stderr = stderr or ""

        skipped_stdout = sum(
            1 for line in stdout.splitlines() if line.strip().startswith("#")
        )
        skipped_stderr = sum(
            1 for line in stderr.splitlines() if "] skipping " in line.lower()
        )
        per_item_failures = sum(
            1 for line in stderr.splitlines()
            if "[download][error]" in line.lower() and "failed to download" in line.lower()
        )
        warning_count = sum(
            1 for line in stderr.splitlines() if "][warning]" in line.lower()
        )
        tier_gated_count = sum(
            1 for line in stderr.splitlines()
            if "][warning]" in line.lower() and "not allowed to view post" in line.lower()
        )

        return {
            "exit_code": return_code,
            "downloaded_count": self._count_downloaded_files(stdout),
            "skipped_count": skipped_stdout + skipped_stderr,
            "per_item_failures": per_item_failures,
            "warning_count": warning_count,
            "tier_gated_count": tier_gated_count,
        }

    @staticmethod
    def _extract_errors_warnings(stderr: str) -> str:
        if not stderr:
            return ""
        kept = [
            line for line in stderr.splitlines()
            if "][error]" in line.lower() or "][warning]" in line.lower()
        ]
        return "\n".join(kept)

    @staticmethod
    def _truncate_log(text: str, max_bytes: int = 500_000) -> str:
        if not text:
            return text
        encoded = text.encode("utf-8")
        if len(encoded) <= max_bytes:
            return text
        half = max_bytes // 2
        head = encoded[:half].decode("utf-8", errors="ignore")
        tail = encoded[-half:].decode("utf-8", errors="ignore")
        head_lines = head.count("\n")
        tail_lines = tail.count("\n")
        total_lines = text.count("\n")
        elided = max(0, total_lines - head_lines - tail_lines)
        marker = f"\n\n... [{elided} lines elided, {len(encoded) - max_bytes} bytes] ...\n\n"
        return head + marker + tail

    async def download(
        self,
        url: str,
        artist_slug: str,
        platform: str,
        source_config: SourceConfig | None = None,
        cookies_path: str | None = None,
        auth_token: str | None = None,
    ) -> DownloadResult:
        start_time = time.time()
        started_at = datetime.now(UTC).isoformat()

        if source_config is None:
            source_config = SourceConfig()

        config = self._build_config_for_source(platform, source_config, artist_slug)

        if cookies_path:
            config["extractor"]["cookies"] = cookies_path
        if auth_token and platform == "discord":
            config["extractor"].setdefault("discord", {})
            config["extractor"]["discord"]["token"] = auth_token
        if auth_token and platform == "pixiv":
            config["extractor"].setdefault("pixiv", {})
            config["extractor"]["pixiv"]["refresh-token"] = auth_token

        with tempfile.NamedTemporaryFile(
            mode="w", suffix=".json", delete=False, dir=str(self._config_dir),
        ) as fh:
            json.dump(config, fh, indent=2)
            temp_config_path = fh.name

        try:
            cmd = [
                sys.executable, "-m", "gallery_dl",
                "--config", temp_config_path, "--verbose", url,
            ]
            log.info("Executing gallery-dl for %s/%s: %s", artist_slug, platform, url)

            loop = asyncio.get_running_loop()
            proc = await loop.run_in_executor(
                None,
                lambda: subprocess.run(
                    cmd, capture_output=True, text=True,
                    timeout=source_config.timeout,
                ),
            )

            duration = time.time() - start_time
            completed_at = datetime.now(UTC).isoformat()

            quarantined_paths: list[str] = []
            quarantine_failures: list[dict] = []
            if self._validate_files:
                quarantined_paths, quarantine_failures = self._validate_and_quarantine(
                    written_paths=self._written_paths(proc.stdout),
                    artist_slug=artist_slug,
                    platform=platform,
                    url=url,
                )

            written_paths = [
                str(p) for p in self._written_paths(proc.stdout)
                if str(p) not in set(quarantined_paths)
            ]
            files_written = self._count_downloaded_files(proc.stdout)
            files_quarantined = len(quarantined_paths)
            files_kept = max(0, files_written - files_quarantined)

            if proc.returncode == 0:
                if files_quarantined > 0:
                    return DownloadResult(
                        success=False, url=url, artist_slug=artist_slug, platform=platform,
                        files_downloaded=files_kept,
                        files_quarantined=files_quarantined,
                        quarantined_paths=quarantined_paths,
                        written_paths=written_paths,
                        stdout=proc.stdout, stderr=proc.stderr,
                        return_code=proc.returncode,
                        error_type=ErrorType.VALIDATION_FAILED,
                        error_message=_summarize_validation_failures(quarantine_failures),
                        duration_seconds=duration,
                        started_at=started_at, completed_at=completed_at,
                    )
                return DownloadResult(
                    success=True, url=url, artist_slug=artist_slug, platform=platform,
                    files_downloaded=files_kept,
                    written_paths=written_paths,
                    stdout=proc.stdout, stderr=proc.stderr,
                    return_code=proc.returncode,
                    duration_seconds=duration,
                    started_at=started_at, completed_at=completed_at,
                )

            error_type, error_message = self._categorize_error(
                proc.returncode, proc.stdout, proc.stderr
            )
            success = error_type in (ErrorType.NO_NEW_CONTENT, ErrorType.TIER_LIMITED)

            if files_quarantined > 0:
                error_type = ErrorType.VALIDATION_FAILED
                error_message = _summarize_validation_failures(quarantine_failures)
                success = False

            return DownloadResult(
                success=success, url=url, artist_slug=artist_slug, platform=platform,
                files_downloaded=files_kept if success else 0,
                files_quarantined=files_quarantined,
                quarantined_paths=quarantined_paths,
                written_paths=written_paths,
                stdout=proc.stdout, stderr=proc.stderr,
                return_code=proc.returncode,
                error_type=error_type, error_message=error_message,
                duration_seconds=duration,
                started_at=started_at, completed_at=completed_at,
            )

        except subprocess.TimeoutExpired:
            duration = time.time() - start_time
            log.error("Download timeout for %s/%s after %.1fs", artist_slug, platform, duration)
            return DownloadResult(
                success=False, url=url, artist_slug=artist_slug, platform=platform,
                error_type=ErrorType.TIMEOUT,
                error_message=f"Download timed out after {source_config.timeout} seconds",
                duration_seconds=duration,
                started_at=started_at,
                completed_at=datetime.now(UTC).isoformat(),
            )
        except Exception as exc:
            duration = time.time() - start_time
            log.exception("Unexpected error downloading %s/%s: %s", artist_slug, platform, exc)
            return DownloadResult(
                success=False, url=url, artist_slug=artist_slug, platform=platform,
                error_type=ErrorType.UNKNOWN_ERROR, error_message=str(exc),
                duration_seconds=duration,
                started_at=started_at,
                completed_at=datetime.now(UTC).isoformat(),
            )
        finally:
            try:
                Path(temp_config_path).unlink()  # noqa: ASYNC240
            except Exception:
                pass

    async def verify(
        self,
        url: str,
        artist_slug: str,
        platform: str,
        source_config: SourceConfig | None = None,
        cookies_path: str | None = None,
        auth_token: str | None = None,
        timeout: float = 45.0,  # noqa: ASYNC109 — subprocess.run timeout, not a coroutine deadline
    ) -> tuple[bool, str]:
        """Test that credentials authenticate against `url` WITHOUT
        downloading anything. Runs gallery-dl in --simulate mode limited
        to the first item; if auth is bad the extractor errors before it
        can list, which _categorize_error flags as AUTH_ERROR. Returns
        (ok, message). Used by the credential Verify button."""
        if source_config is None:
            source_config = SourceConfig()
        config = self._build_config_for_source(platform, source_config, artist_slug)
        if cookies_path:
            config["extractor"]["cookies"] = cookies_path
        if auth_token and platform == "discord":
            config["extractor"].setdefault("discord", {})["token"] = auth_token
        if auth_token and platform == "pixiv":
            config["extractor"].setdefault("pixiv", {})["refresh-token"] = auth_token

        with tempfile.NamedTemporaryFile(
            mode="w", suffix=".json", delete=False, dir=str(self._config_dir),
        ) as fh:
            json.dump(config, fh, indent=2)
            temp_config_path = fh.name
        try:
            cmd = [
                sys.executable, "-m", "gallery_dl",
                "--config", temp_config_path,
                "--simulate", "--range", "1-1", "--verbose", url,
            ]
            loop = asyncio.get_running_loop()
            proc = await loop.run_in_executor(
                None,
                lambda: subprocess.run(
                    cmd, capture_output=True, text=True, timeout=timeout,
                ),
            )
            etype, msg = self._categorize_error(proc.returncode, proc.stdout, proc.stderr)
            if proc.returncode == 0 or etype == ErrorType.NO_NEW_CONTENT:
                return True, "Credentials valid — the feed authenticated."
            if etype == ErrorType.AUTH_ERROR:
                return False, msg
            # Network / not-found / rate-limit / unknown: inconclusive,
            # not a definitive credential failure. Surface the reason.
            return False, f"Could not confirm ({etype.value}): {msg}"
        except subprocess.TimeoutExpired:
            return False, f"Verification timed out after {timeout:.0f}s"
        except Exception as exc:  # noqa: BLE001
            return False, f"Verification error: {exc}"
        finally:
            try:
                Path(temp_config_path).unlink()  # noqa: ASYNC240
            except Exception:
                pass