FabledCurator/backend/app/services/discord_downloader.py

"""Native Discord media downloader — the Discord counterpart to
subscribestar_downloader.

Writes files exactly where gallery-dl wrote them, so a cutover finds every
existing file on disk (`skipped_disk`) instead of fetching it again:

    <images>/<artist>/discord/<channel>/<YYYYMMDD>_<message_id>_<NN>_<name>.<ext>

That is what FC's gallery-dl config produced (directory `{channel}`, filename
`{date:%Y%m%d}_{message_id}_{num:>02}_{filename}.{extension}`, under the
per-source base directory `<images>/<artist>/<platform>`), retired from that
config once Discord moved here; tests/test_discord_naming.py pins the match
against a real gallery-dl sidecar. The name is cleaned the way gallery-dl cleans it
on Linux — `/` becomes `_` and control characters are removed, nothing else
(`path-restrict: auto`, `path-remove` defaults). It is NOT `sanitize_segment`,
whose Windows set would turn a `:` in a channel or file name into `_` and miss
the file gallery-dl wrote.

Post-first (rule 120): each file gets a minimal sidecar named like it minus the
extension (what `find_sidecar` pairs first), and the message itself gets one
record, `<YYYYMMDD>_<message_id>_post.json`, carrying gallery-dl's metadata keys
— `message_id` for the post id, `server_id`/`channel_id` for the permalink,
`message` for the body, `date` — so `parse_sidecar` reads it exactly as it read
the gallery-dl sidecars. Neither file carries an `id` or `post_id` key: both
outrank `message_id` in the post-id chain (`platforms.base`).

PURE: no DB; the seen-skip is an injected predicate.
"""

from __future__ import annotations

import json
import logging
import re
import time
from collections.abc import Callable
from datetime import UTC, datetime
from pathlib import Path

import requests

from .discord_client import firefox_user_agent, message_text
from .native_ingest_common import (
    BaseNativeDownloader,
    MediaOutcome,
    PostRecordOutcome,
    make_session,
)

log = logging.getLogger(__name__)

PLATFORM = "discord"
_CONTROL = re.compile("[\x00-\x1f\x7f]")
# gallery-dl falls back to the response's type for a URL with no extension;
# we never see the response before naming, and such URLs do not occur for
# Discord attachments or embed proxies in practice.
_NO_EXTENSION = "bin"


def gdl_clean(segment: str) -> str:
    """One path segment as gallery-dl writes it on Linux."""
    return _CONTROL.sub("", segment.replace("/", "_"))


def message_date(post: dict) -> datetime | None:
    raw = post.get("timestamp")
    if not isinstance(raw, str) or not raw:
        return None
    try:
        dt = datetime.fromisoformat(raw.replace("Z", "+00:00"))
    except ValueError:
        return None
    return (dt if dt.tzinfo else dt.replace(tzinfo=UTC)).astimezone(UTC)


def channel_dir(images_root: Path, artist_slug: str, post: dict) -> Path:
    """gallery-dl's `{channel}` directory; an empty name adds no segment."""
    base = Path(images_root) / artist_slug / PLATFORM
    channel = gdl_clean(((post.get("_meta") or {}).get("channel") or "").strip())
    return base / channel if channel else base


def media_stem(post: dict, media) -> str:
    """`<YYYYMMDD>_<message_id>_<NN>_<name>` — the file's name minus `.<ext>`."""
    when = message_date(post)
    day = f"{when:%Y%m%d}" if when else "None"
    return gdl_clean(f"{day}_{post.get('id')}_{media.num:>02}_{media.filename}")


class DiscordDownloader(BaseNativeDownloader):
    """Download a message's files to gallery-dl's layout. The CDN gets
    gallery-dl's browser profile and no token — gallery-dl sends the token only
    to the API, and the CDN URLs are pre-signed."""

    def __init__(
        self,
        images_root: Path,
        cookies_path: str | None = None,
        *,
        validate: bool = True,
        rate_limit: float = 0.0,
        session: requests.Session | None = None,
    ):
        super().__init__(
            images_root, None, platform=PLATFORM,
            validate=validate, rate_limit=rate_limit,
            session=session if session is not None else make_session(None, extra_headers={
                "User-Agent": firefox_user_agent(),
                "Accept-Language": "en-US,en;q=0.5",
                "Referer": "https://discord.com/",
            }),
        )

    def download_post(
        self,
        post: dict,
        media_items: list,
        artist_slug: str,
        *,
        is_seen: Callable[[object], bool] = lambda m: False,
        should_stop: Callable[[], bool] = lambda: False,
        recapture: bool = False,
    ) -> list[MediaOutcome]:
        """Every file of one message; per-file outcomes, one failure isolated."""
        folder = channel_dir(self.images_root, artist_slug, post)
        outcomes: list[MediaOutcome] = []
        for media in media_items:
            if should_stop():
                break
            try:
                outcomes.append(self._download_one(
                    post, media, folder, artist_slug, is_seen, recapture=recapture,
                ))
            except Exception as exc:  # resilient: isolate one item's failure
                log.warning(
                    "Discord media failed (message %s, file %d): %s",
                    post.get("id"), media.num, exc,
                )
                outcomes.append(
                    MediaOutcome(media=media, status="error", path=None, error=str(exc))
                )
        return outcomes

    def _download_one(
        self,
        post: dict,
        media,
        folder: Path,
        artist_slug: str,
        is_seen: Callable[[object], bool],
        *,
        recapture: bool = False,
    ) -> MediaOutcome:
        seen = is_seen(media)
        if seen and not recapture:
            return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
        stem = media_stem(post, media)
        path = folder / f"{stem}.{media.extension or _NO_EXTENSION}"
        if path.exists():  # tier-2: gallery-dl (or an earlier walk) wrote it
            return MediaOutcome(media=media, status="skipped_disk", path=path, error=None)
        if seen:  # recapture never re-fetches a seen file that is gone
            return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)

        folder.mkdir(parents=True, exist_ok=True)
        if self._rate_limit > 0:
            time.sleep(self._rate_limit)
        out = self._fetch_get(media.url, path)
        reason, quarantined = self._validate_path(out, artist_slug, media.url)
        if reason is not None:
            return MediaOutcome(media=media, status="quarantined", path=quarantined, error=reason)
        sidecar = {"category": PLATFORM, "message_id": str(post.get("id") or "")}
        sidecar["source_url"] = media.url
        (folder / f"{stem}.json").write_text(json.dumps(sidecar, indent=2))
        return MediaOutcome(media=media, status="downloaded", path=out, error=None)

    def write_post_record(
        self, post: dict, artist_slug: str, *, revisit: bool = False,
    ) -> PostRecordOutcome:
        """The message record — the one writer of a Discord post's body, date
        and permalink ids. `revisit` re-reads a message already captured (an
        edit); an empty re-read writes nothing, so it never blanks a body."""
        mid = str(post.get("id") or "")
        body = message_text(post)
        if not mid or (revisit and not body.strip()):
            return PostRecordOutcome(path=None, post_type=None, title=None, body_chars=0)
        meta = post.get("_meta") or {}
        author = post.get("author") or {}
        record = {
            "category": PLATFORM,
            "message_id": mid,
            "server": meta.get("server"),
            "server_id": meta.get("server_id"),
            "channel": meta.get("channel"),
            "channel_id": meta.get("channel_id") or post.get("channel_id"),
            "parent": meta.get("parent"),
            "is_thread": meta.get("is_thread"),
            "author": author.get("username"),
            "author_id": author.get("id"),
            "message": body,
            "date": post.get("timestamp"),
        }
        folder = channel_dir(self.images_root, artist_slug, post)
        folder.mkdir(parents=True, exist_ok=True)
        when = message_date(post)
        day = f"{when:%Y%m%d}" if when else "None"
        path = folder / f"{day}_{mid}_post.json"
        path.write_text(json.dumps(
            {k: v for k, v in record.items() if v is not None}, indent=2,
        ))
        return PostRecordOutcome(
            path=path, post_type=None, title=None, body_chars=len(body),
        )