"""Native Discord media downloader — the Discord counterpart to subscribestar_downloader. Writes files exactly where gallery-dl wrote them, so a cutover finds every existing file on disk (`skipped_disk`) instead of fetching it again: //discord//___. That is what FC's gallery-dl config produced (directory `{channel}`, filename `{date:%Y%m%d}_{message_id}_{num:>02}_{filename}.{extension}`, under the per-source base directory `//`), retired from that config once Discord moved here; tests/test_discord_naming.py pins the match against a real gallery-dl sidecar. The name is cleaned the way gallery-dl cleans it on Linux — `/` becomes `_` and control characters are removed, nothing else (`path-restrict: auto`, `path-remove` defaults). It is NOT `sanitize_segment`, whose Windows set would turn a `:` in a channel or file name into `_` and miss the file gallery-dl wrote. Post-first (rule 120): each file gets a minimal sidecar named like it minus the extension (what `find_sidecar` pairs first), and the message itself gets one record, `__post.json`, carrying gallery-dl's metadata keys — `message_id` for the post id, `server_id`/`channel_id` for the permalink, `message` for the body, `date` — so `parse_sidecar` reads it exactly as it read the gallery-dl sidecars. Neither file carries an `id` or `post_id` key: both outrank `message_id` in the post-id chain (`platforms.base`). PURE: no DB; the seen-skip is an injected predicate. """ from __future__ import annotations import json import logging import re import time from collections.abc import Callable from datetime import UTC, datetime from pathlib import Path import requests from .discord_client import firefox_user_agent, message_text from .native_ingest_common import ( BaseNativeDownloader, MediaOutcome, PostRecordOutcome, make_session, ) log = logging.getLogger(__name__) PLATFORM = "discord" _CONTROL = re.compile("[\x00-\x1f\x7f]") # gallery-dl falls back to the response's type for a URL with no extension; # we never see the response before naming, and such URLs do not occur for # Discord attachments or embed proxies in practice. _NO_EXTENSION = "bin" def gdl_clean(segment: str) -> str: """One path segment as gallery-dl writes it on Linux.""" return _CONTROL.sub("", segment.replace("/", "_")) def message_date(post: dict) -> datetime | None: raw = post.get("timestamp") if not isinstance(raw, str) or not raw: return None try: dt = datetime.fromisoformat(raw.replace("Z", "+00:00")) except ValueError: return None return (dt if dt.tzinfo else dt.replace(tzinfo=UTC)).astimezone(UTC) def channel_dir(images_root: Path, artist_slug: str, post: dict) -> Path: """gallery-dl's `{channel}` directory; an empty name adds no segment.""" base = Path(images_root) / artist_slug / PLATFORM channel = gdl_clean(((post.get("_meta") or {}).get("channel") or "").strip()) return base / channel if channel else base def media_stem(post: dict, media) -> str: """`___` — the file's name minus `.`.""" when = message_date(post) day = f"{when:%Y%m%d}" if when else "None" return gdl_clean(f"{day}_{post.get('id')}_{media.num:>02}_{media.filename}") class DiscordDownloader(BaseNativeDownloader): """Download a message's files to gallery-dl's layout. The CDN gets gallery-dl's browser profile and no token — gallery-dl sends the token only to the API, and the CDN URLs are pre-signed.""" def __init__( self, images_root: Path, cookies_path: str | None = None, *, validate: bool = True, rate_limit: float = 0.0, session: requests.Session | None = None, ): super().__init__( images_root, None, platform=PLATFORM, validate=validate, rate_limit=rate_limit, session=session if session is not None else make_session(None, extra_headers={ "User-Agent": firefox_user_agent(), "Accept-Language": "en-US,en;q=0.5", "Referer": "https://discord.com/", }), ) def download_post( self, post: dict, media_items: list, artist_slug: str, *, is_seen: Callable[[object], bool] = lambda m: False, should_stop: Callable[[], bool] = lambda: False, recapture: bool = False, ) -> list[MediaOutcome]: """Every file of one message; per-file outcomes, one failure isolated.""" folder = channel_dir(self.images_root, artist_slug, post) outcomes: list[MediaOutcome] = [] for media in media_items: if should_stop(): break try: outcomes.append(self._download_one( post, media, folder, artist_slug, is_seen, recapture=recapture, )) except Exception as exc: # resilient: isolate one item's failure log.warning( "Discord media failed (message %s, file %d): %s", post.get("id"), media.num, exc, ) outcomes.append( MediaOutcome(media=media, status="error", path=None, error=str(exc)) ) return outcomes def _download_one( self, post: dict, media, folder: Path, artist_slug: str, is_seen: Callable[[object], bool], *, recapture: bool = False, ) -> MediaOutcome: seen = is_seen(media) if seen and not recapture: return MediaOutcome(media=media, status="skipped_seen", path=None, error=None) stem = media_stem(post, media) path = folder / f"{stem}.{media.extension or _NO_EXTENSION}" if path.exists(): # tier-2: gallery-dl (or an earlier walk) wrote it return MediaOutcome(media=media, status="skipped_disk", path=path, error=None) if seen: # recapture never re-fetches a seen file that is gone return MediaOutcome(media=media, status="skipped_seen", path=None, error=None) folder.mkdir(parents=True, exist_ok=True) if self._rate_limit > 0: time.sleep(self._rate_limit) out = self._fetch_get(media.url, path) reason, quarantined = self._validate_path(out, artist_slug, media.url) if reason is not None: return MediaOutcome(media=media, status="quarantined", path=quarantined, error=reason) sidecar = {"category": PLATFORM, "message_id": str(post.get("id") or "")} sidecar["source_url"] = media.url (folder / f"{stem}.json").write_text(json.dumps(sidecar, indent=2)) return MediaOutcome(media=media, status="downloaded", path=out, error=None) def write_post_record( self, post: dict, artist_slug: str, *, revisit: bool = False, ) -> PostRecordOutcome: """The message record — the one writer of a Discord post's body, date and permalink ids. `revisit` re-reads a message already captured (an edit); an empty re-read writes nothing, so it never blanks a body.""" mid = str(post.get("id") or "") body = message_text(post) if not mid or (revisit and not body.strip()): return PostRecordOutcome(path=None, post_type=None, title=None, body_chars=0) meta = post.get("_meta") or {} author = post.get("author") or {} record = { "category": PLATFORM, "message_id": mid, "server": meta.get("server"), "server_id": meta.get("server_id"), "channel": meta.get("channel"), "channel_id": meta.get("channel_id") or post.get("channel_id"), "parent": meta.get("parent"), "is_thread": meta.get("is_thread"), "author": author.get("username"), "author_id": author.get("id"), "message": body, "date": post.get("timestamp"), } folder = channel_dir(self.images_root, artist_slug, post) folder.mkdir(parents=True, exist_ok=True) when = message_date(post) day = f"{when:%Y%m%d}" if when else "None" path = folder / f"{day}_{mid}_post.json" path.write_text(json.dumps( {k: v for k, v in record.items() if v is not None}, indent=2, )) return PostRecordOutcome( path=path, post_type=None, title=None, body_chars=len(body), )