feat: Discord on the native core ingester — client, downloader, ledgers, wiring (milestone 428)

Discord was the last focus platform still on gallery-dl. This adds the native path, mirrored from gallery-dl 1.32.13's discord extractor: - discord_client: API v10 with the user token and gallery-dl's request profile (dated Firefox UA, Referer). Walks a server, category, forum, channel or thread in gallery-dl's order and pages each channel newest-first. Files are attachments, then embeds, then forwards, numbered across the message. The resume cursor is <channel>:<before>. Text-only messages are not posts, since gallery-dl never made them. - discord_downloader: gallery-dl's on-disk layout, cleaned the way it cleans names on Linux (only `/` and control characters change), so existing files are skipped_disk rather than fetched again. Sidecars carry identity only. The message record keeps gallery-dl's keys, so parse_sidecar, derive_post_url and the drop grouping read it unchanged. - The ledger keys on the attachment id (or a hash of an embed's URL path), not the file's position, which an edit can renumber. Migration 0111. - DiscordIngester: token auth, body canary off (files-only drops are normal). Registered as native, verified by token, and serialised per-platform, since every source shares one user token. - ingest_core: optional `skip_feed` client seam (#4413). A tick's early-out on a multi-channel source now ends the quiet channel, not the whole walk. Clients without the seam behave as before. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR
2026-09-24 19:42:00 -04:00
parent 058fa85606
commit 84e5448941
15 changed files with 1579 additions and 15 deletions
@@ -0,0 +1,210 @@
+"""Native Discord media downloader — the Discord counterpart to
+subscribestar_downloader.
+
+Writes files exactly where gallery-dl wrote them, so a cutover finds every
+existing file on disk (`skipped_disk`) instead of fetching it again:
+
+    <images>/<artist>/discord/<channel>/<YYYYMMDD>_<message_id>_<NN>_<name>.<ext>
+
+That is FC's gallery-dl config (`gallery_dl.DISCORD_DIRECTORY` /
+`DISCORD_FILENAME`) under the per-source base directory
+`<images>/<artist>/<platform>`. The name is cleaned the way gallery-dl cleans it
+on Linux — `/` becomes `_` and control characters are removed, nothing else
+(`path-restrict: auto`, `path-remove` defaults). It is NOT `sanitize_segment`,
+whose Windows set would turn a `:` in a channel or file name into `_` and miss
+the file gallery-dl wrote.
+
+Post-first (rule 120): each file gets a minimal sidecar named like it minus the
+extension (what `find_sidecar` pairs first), and the message itself gets one
+record, `<YYYYMMDD>_<message_id>_post.json`, carrying gallery-dl's metadata keys
+— `message_id` for the post id, `server_id`/`channel_id` for the permalink,
+`message` for the body, `date` — so `parse_sidecar` reads it exactly as it read
+the gallery-dl sidecars. Neither file carries an `id` or `post_id` key: both
+outrank `message_id` in the post-id chain (`platforms.base`).
+
+PURE: no DB; the seen-skip is an injected predicate.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+import time
+from collections.abc import Callable
+from datetime import UTC, datetime
+from pathlib import Path
+
+import requests
+
+from .discord_client import firefox_user_agent, message_text
+from .native_ingest_common import (
+    BaseNativeDownloader,
+    MediaOutcome,
+    PostRecordOutcome,
+    make_session,
+)
+
+log = logging.getLogger(__name__)
+
+PLATFORM = "discord"
+_CONTROL = re.compile("[\x00-\x1f\x7f]")
+# gallery-dl falls back to the response's type for a URL with no extension;
+# we never see the response before naming, and such URLs do not occur for
+# Discord attachments or embed proxies in practice.
+_NO_EXTENSION = "bin"
+
+
+def gdl_clean(segment: str) -> str:
+    """One path segment as gallery-dl writes it on Linux."""
+    return _CONTROL.sub("", segment.replace("/", "_"))
+
+
+def message_date(post: dict) -> datetime | None:
+    raw = post.get("timestamp")
+    if not isinstance(raw, str) or not raw:
+        return None
+    try:
+        dt = datetime.fromisoformat(raw.replace("Z", "+00:00"))
+    except ValueError:
+        return None
+    return (dt if dt.tzinfo else dt.replace(tzinfo=UTC)).astimezone(UTC)
+
+
+def channel_dir(images_root: Path, artist_slug: str, post: dict) -> Path:
+    """gallery-dl's `{channel}` directory; an empty name adds no segment."""
+    base = Path(images_root) / artist_slug / PLATFORM
+    channel = gdl_clean(((post.get("_meta") or {}).get("channel") or "").strip())
+    return base / channel if channel else base
+
+
+def media_stem(post: dict, media) -> str:
+    """`<YYYYMMDD>_<message_id>_<NN>_<name>` — the file's name minus `.<ext>`."""
+    when = message_date(post)
+    day = f"{when:%Y%m%d}" if when else "None"
+    return gdl_clean(f"{day}_{post.get('id')}_{media.num:>02}_{media.filename}")
+
+
+class DiscordDownloader(BaseNativeDownloader):
+    """Download a message's files to gallery-dl's layout. The CDN gets
+    gallery-dl's browser profile and no token — gallery-dl sends the token only
+    to the API, and the CDN URLs are pre-signed."""
+
+    def __init__(
+        self,
+        images_root: Path,
+        cookies_path: str | None = None,
+        *,
+        validate: bool = True,
+        rate_limit: float = 0.0,
+        session: requests.Session | None = None,
+    ):
+        super().__init__(
+            images_root, None, platform=PLATFORM,
+            validate=validate, rate_limit=rate_limit,
+            session=session if session is not None else make_session(None, extra_headers={
+                "User-Agent": firefox_user_agent(),
+                "Accept-Language": "en-US,en;q=0.5",
+                "Referer": "https://discord.com/",
+            }),
+        )
+
+    def download_post(
+        self,
+        post: dict,
+        media_items: list,
+        artist_slug: str,
+        *,
+        is_seen: Callable[[object], bool] = lambda m: False,
+        should_stop: Callable[[], bool] = lambda: False,
+        recapture: bool = False,
+    ) -> list[MediaOutcome]:
+        """Every file of one message; per-file outcomes, one failure isolated."""
+        folder = channel_dir(self.images_root, artist_slug, post)
+        outcomes: list[MediaOutcome] = []
+        for media in media_items:
+            if should_stop():
+                break
+            try:
+                outcomes.append(self._download_one(
+                    post, media, folder, artist_slug, is_seen, recapture=recapture,
+                ))
+            except Exception as exc:  # resilient: isolate one item's failure
+                log.warning(
+                    "Discord media failed (message %s, file %d): %s",
+                    post.get("id"), media.num, exc,
+                )
+                outcomes.append(
+                    MediaOutcome(media=media, status="error", path=None, error=str(exc))
+                )
+        return outcomes
+
+    def _download_one(
+        self,
+        post: dict,
+        media,
+        folder: Path,
+        artist_slug: str,
+        is_seen: Callable[[object], bool],
+        *,
+        recapture: bool = False,
+    ) -> MediaOutcome:
+        seen = is_seen(media)
+        if seen and not recapture:
+            return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
+        stem = media_stem(post, media)
+        path = folder / f"{stem}.{media.extension or _NO_EXTENSION}"
+        if path.exists():  # tier-2: gallery-dl (or an earlier walk) wrote it
+            return MediaOutcome(media=media, status="skipped_disk", path=path, error=None)
+        if seen:  # recapture never re-fetches a seen file that is gone
+            return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
+
+        folder.mkdir(parents=True, exist_ok=True)
+        if self._rate_limit > 0:
+            time.sleep(self._rate_limit)
+        out = self._fetch_get(media.url, path)
+        reason, quarantined = self._validate_path(out, artist_slug, media.url)
+        if reason is not None:
+            return MediaOutcome(media=media, status="quarantined", path=quarantined, error=reason)
+        sidecar = {"category": PLATFORM, "message_id": str(post.get("id") or "")}
+        sidecar["source_url"] = media.url
+        (folder / f"{stem}.json").write_text(json.dumps(sidecar, indent=2))
+        return MediaOutcome(media=media, status="downloaded", path=out, error=None)
+
+    def write_post_record(
+        self, post: dict, artist_slug: str, *, revisit: bool = False,
+    ) -> PostRecordOutcome:
+        """The message record — the one writer of a Discord post's body, date
+        and permalink ids. `revisit` re-reads a message already captured (an
+        edit); an empty re-read writes nothing, so it never blanks a body."""
+        mid = str(post.get("id") or "")
+        body = message_text(post)
+        if not mid or (revisit and not body.strip()):
+            return PostRecordOutcome(path=None, post_type=None, title=None, body_chars=0)
+        meta = post.get("_meta") or {}
+        author = post.get("author") or {}
+        record = {
+            "category": PLATFORM,
+            "message_id": mid,
+            "server": meta.get("server"),
+            "server_id": meta.get("server_id"),
+            "channel": meta.get("channel"),
+            "channel_id": meta.get("channel_id") or post.get("channel_id"),
+            "parent": meta.get("parent"),
+            "is_thread": meta.get("is_thread"),
+            "author": author.get("username"),
+            "author_id": author.get("id"),
+            "message": body,
+            "date": post.get("timestamp"),
+        }
+        folder = channel_dir(self.images_root, artist_slug, post)
+        folder.mkdir(parents=True, exist_ok=True)
+        when = message_date(post)
+        day = f"{when:%Y%m%d}" if when else "None"
+        path = folder / f"{day}_{mid}_post.json"
+        path.write_text(json.dumps(
+            {k: v for k, v in record.items() if v is not None}, indent=2,
+        ))
+        return PostRecordOutcome(
+            path=path, post_type=None, title=None, body_chars=len(body),
+        )