From 84e54489418658d8508dfd3a89b0c5c50bdfd186 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 24 Sep 2026 19:42:00 -0400 Subject: [PATCH] =?UTF-8?q?feat:=20Discord=20on=20the=20native=20core=20in?= =?UTF-8?q?gester=20=E2=80=94=20client,=20downloader,=20ledgers,=20wiring?= =?UTF-8?q?=20(milestone=20428)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Discord was the last focus platform still on gallery-dl. This adds the native path, mirrored from gallery-dl 1.32.13's discord extractor: - discord_client: API v10 with the user token and gallery-dl's request profile (dated Firefox UA, Referer). Walks a server, category, forum, channel or thread in gallery-dl's order and pages each channel newest-first. Files are attachments, then embeds, then forwards, numbered across the message. The resume cursor is :. Text-only messages are not posts, since gallery-dl never made them. - discord_downloader: gallery-dl's on-disk layout, cleaned the way it cleans names on Linux (only `/` and control characters change), so existing files are skipped_disk rather than fetched again. Sidecars carry identity only. The message record keeps gallery-dl's keys, so parse_sidecar, derive_post_url and the drop grouping read it unchanged. - The ledger keys on the attachment id (or a hash of an embed's URL path), not the file's position, which an edit can renumber. Migration 0111. - DiscordIngester: token auth, body canary off (files-only drops are normal). Registered as native, verified by token, and serialised per-platform, since every source shares one user token. - ingest_core: optional `skip_feed` client seam (#4413). A tick's early-out on a multi-channel source now ends the quiet channel, not the whole walk. Clients without the seam behave as before. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR --- .../versions/0111_discord_native_ledger.py | 75 +++ backend/app/models/__init__.py | 4 + backend/app/models/discord_failed_media.py | 36 ++ backend/app/models/discord_seen_media.py | 34 ++ backend/app/services/discord_client.py | 544 ++++++++++++++++++ backend/app/services/discord_downloader.py | 210 +++++++ backend/app/services/discord_ingester.py | 85 +++ backend/app/services/download_backends.py | 24 +- backend/app/services/ingest_core.py | 20 +- backend/app/services/platform_lock.py | 6 +- tests/test_discord_client.py | 339 +++++++++++ tests/test_discord_downloader.py | 135 +++++ tests/test_download_backends.py | 18 +- tests/test_patreon_ingester.py | 58 ++ tests/test_platform_lock.py | 6 +- 15 files changed, 1579 insertions(+), 15 deletions(-) create mode 100644 alembic/versions/0111_discord_native_ledger.py create mode 100644 backend/app/models/discord_failed_media.py create mode 100644 backend/app/models/discord_seen_media.py create mode 100644 backend/app/services/discord_client.py create mode 100644 backend/app/services/discord_downloader.py create mode 100644 backend/app/services/discord_ingester.py create mode 100644 tests/test_discord_client.py create mode 100644 tests/test_discord_downloader.py diff --git a/alembic/versions/0111_discord_native_ledger.py b/alembic/versions/0111_discord_native_ledger.py new file mode 100644 index 0000000..2637d2d --- /dev/null +++ b/alembic/versions/0111_discord_native_ledger.py @@ -0,0 +1,75 @@ +"""Discord native ingester ledgers — seen and dead-letter, per source. + +Milestone 428, #4415. Discord moves off gallery-dl onto the native core, which +keeps its memory of what a source has already fetched in these two tables +instead of gallery-dl's archive. Same shape as the SubscribeStar pair. + +Revision ID: 0111 +Revises: 0110 +Create Date: 2026-09-24 + +""" +import sqlalchemy as sa +from alembic import op + +revision = "0111" +down_revision = "0110" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "discord_seen_media", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("source_id", sa.Integer(), nullable=False), + sa.Column("filehash", sa.String(length=128), nullable=False), + sa.Column("post_id", sa.String(length=64), nullable=True), + sa.Column( + "seen_at", sa.DateTime(timezone=True), + server_default=sa.text("now()"), nullable=False, + ), + sa.ForeignKeyConstraint( + ["source_id"], ["source.id"], + name=op.f("fk_discord_seen_media_source_id_source"), ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_discord_seen_media")), + sa.UniqueConstraint("source_id", "filehash", name="uq_discord_seen_media_source_id"), + ) + op.create_index( + op.f("ix_discord_seen_media_source_id"), "discord_seen_media", ["source_id"], + ) + op.create_table( + "discord_failed_media", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("source_id", sa.Integer(), nullable=False), + sa.Column("filehash", sa.String(length=128), nullable=False), + sa.Column("attempts", sa.Integer(), server_default="1", nullable=False), + sa.Column("last_error", sa.Text(), nullable=True), + sa.Column( + "first_failed_at", sa.DateTime(timezone=True), + server_default=sa.text("now()"), nullable=False, + ), + sa.Column( + "last_failed_at", sa.DateTime(timezone=True), + server_default=sa.text("now()"), nullable=False, + ), + sa.ForeignKeyConstraint( + ["source_id"], ["source.id"], + name=op.f("fk_discord_failed_media_source_id_source"), ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_discord_failed_media")), + sa.UniqueConstraint( + "source_id", "filehash", name="uq_discord_failed_media_source_id", + ), + ) + op.create_index( + op.f("ix_discord_failed_media_source_id"), "discord_failed_media", ["source_id"], + ) + + +def downgrade(): + op.drop_index(op.f("ix_discord_failed_media_source_id"), table_name="discord_failed_media") + op.drop_table("discord_failed_media") + op.drop_index(op.f("ix_discord_seen_media_source_id"), table_name="discord_seen_media") + op.drop_table("discord_seen_media") diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index b925805..b890bfd 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -8,6 +8,8 @@ from .backup_run import BackupRun from .base import Base from .character_prototype import CcipPrototypeState, CharacterPrototype from .credential import Credential +from .discord_failed_media import DiscordFailedMedia +from .discord_seen_media import DiscordSeenMedia from .download_event import DownloadEvent from .external_link import ExternalLink from .gpu_job import GpuJob @@ -56,6 +58,8 @@ __all__ = [ "BackupRun", "Source", "Credential", + "DiscordFailedMedia", + "DiscordSeenMedia", "PatreonFailedMedia", "PatreonSeenMedia", "SubscribeStarFailedMedia", diff --git a/backend/app/models/discord_failed_media.py b/backend/app/models/discord_failed_media.py new file mode 100644 index 0000000..ad0b2d5 --- /dev/null +++ b/backend/app/models/discord_failed_media.py @@ -0,0 +1,36 @@ +"""DiscordFailedMedia — per-source dead-letter ledger of Discord files that +keep failing to download or validate. + +Mirror of SubscribeStarFailedMedia. After `attempts` reaches the dead-letter +threshold a routine walk skips the file (recovery still retries it); a later +clean download clears the row. `filehash` is the seen-ledger's key. +""" + +from datetime import datetime + +from sqlalchemy import ForeignKey, Integer, String, Text, UniqueConstraint, func +from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.types import DateTime + +from .base import Base + + +class DiscordFailedMedia(Base): + __tablename__ = "discord_failed_media" + __table_args__ = ( + UniqueConstraint("source_id", "filehash", name="uq_discord_failed_media_source_id"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + source_id: Mapped[int] = mapped_column( + ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True + ) + filehash: Mapped[str] = mapped_column(String(128), nullable=False) + attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default="1") + last_error: Mapped[str | None] = mapped_column(Text, nullable=True) + first_failed_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + last_failed_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) diff --git a/backend/app/models/discord_seen_media.py b/backend/app/models/discord_seen_media.py new file mode 100644 index 0000000..2757463 --- /dev/null +++ b/backend/app/models/discord_seen_media.py @@ -0,0 +1,34 @@ +"""DiscordSeenMedia — per-source ledger of Discord files already downloaded. + +Mirror of SubscribeStarSeenMedia for the native Discord ingester (milestone +428). `filehash` holds the ingester's per-file key, `:`: +the attachment id, or for an embed a hash of its URL path. Not the file's +position in the message — an edit that removes a file renumbers the rest +(see `discord_client.MediaItem`). The message record's own gate is the +synthetic `message:` key in the same column. +""" + +from datetime import datetime + +from sqlalchemy import ForeignKey, Integer, String, UniqueConstraint, func +from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.types import DateTime + +from .base import Base + + +class DiscordSeenMedia(Base): + __tablename__ = "discord_seen_media" + __table_args__ = ( + UniqueConstraint("source_id", "filehash", name="uq_discord_seen_media_source_id"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + source_id: Mapped[int] = mapped_column( + ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True + ) + filehash: Mapped[str] = mapped_column(String(128), nullable=False) + post_id: Mapped[str | None] = mapped_column(String(64), nullable=True) + seen_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) diff --git a/backend/app/services/discord_client.py b/backend/app/services/discord_client.py new file mode 100644 index 0000000..dfa5d50 --- /dev/null +++ b/backend/app/services/discord_client.py @@ -0,0 +1,544 @@ +"""Native Discord read client — the Discord counterpart to subscribestar_client. + +Mirrors gallery-dl 1.32.13's `extractor/discord.py` (rule 130: gallery-dl is the +known-working base), adapted to the native core's client contract +(`ingest_core` module docstring): `iter_posts` / `extract_media`, plus the +post-first `post_record_key` and the `post_meta` date the revisit window reads. + +What is mirrored exactly, because drift in any of it changes what we fetch or +where it lands on disk: + + - API v10, `Authorization: ` (a USER token, not a bot token). + - gallery-dl's request profile: its date-derived Firefox User-Agent, + `Accept: */*`, `Accept-Language`, `Referer: https://discord.com/`. + - `GET /channels/{id}/messages?limit=100&before=`, newest first, + stopping on a short page. Message types {0, 19, 21} only. + - The walk: a text/news channel's own messages then its threads, a forum's + threads, a category's children, a server's text/news/forum channels. + - Files: attachments, then embeds of type image/gifv/video (FC configures + `embeds: all`, which for files is the same three plus rich/link embeds that + carry an image), then forwarded `message_snapshots`, numbered from 1 across + the lot — the `num` in `{date}_{message_id}_{num}_{filename}`. + - Text: `content`, rich-embed author/title/description/fields/footer, poll. + +Two deliberate departures, both about the walk order, neither about content: + + - Threads are walked newest-CREATED first (by id), not by last-message time. + A backfill resumes from a checkpointed channel; last-message order shifts + between chunks whenever someone posts, which can move an unwalked thread + above the resume point and skip it. Creation order only ever grows at the + front, where the next tick finds it. + - A 403 on a thread or a nested channel skips that feed instead of failing + the walk. gallery-dl skips only nested channels; one private thread the + token cannot read would otherwise stop every channel after it. + +FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API. +""" + +from __future__ import annotations + +import hashlib +import logging +import re +import time +from collections.abc import Iterator +from dataclasses import dataclass +from datetime import date +from urllib.parse import unquote + +import requests + +from .native_ingest_common import ( + NativeAuthError, + NativeDriftError, + NativeIngestError, + retry_after_seconds, +) + +log = logging.getLogger(__name__) + +API_ROOT = "https://discord.com/api/v10" +_ROOT = "https://discord.com" + +_TIMEOUT_SECONDS = 60.0 +_MESSAGES_BATCH = 100 +_THREADS_BATCH = 25 +# gallery-dl retries a 429 up to its default 4 retries, waiting +# `request_interval_429` (60s) between them. Discord's Retry-After is exact, so +# it is honoured when present; 60s is the fallback and the cap. +_MAX_429_RETRIES = 4 +_429_WAIT_SECONDS = 60.0 + +# https://discord.com/developers/docs/resources/message#message-object-message-types +# DEFAULT, REPLY, CHAT_INPUT_COMMAND — the ones that carry user content. +MESSAGE_TYPES = frozenset({0, 19, 21}) +# https://discord.com/developers/docs/resources/channel#channel-object-channel-types +_TEXT = frozenset({0, 5}) # text, announcement: messages + threads +_DIRECT = frozenset({1, 3, 10, 11, 12}) # DMs and threads: messages only +_FORUM = frozenset({15, 16}) # forum, media: threads only +_CATEGORY = 4 +_SERVER_WALK = _TEXT | _FORUM +_EMBED_TYPES = frozenset({"image", "gifv", "video"}) + +_URL_RE = re.compile( + r"^(?:https?://)?(?:www\.|ptb\.|canary\.)?discord(?:app)?\.com/channels/" + r"(?P@me|\d+)(?:/(?:\d+/threads/)?(?P\d+))?(?P/.*)?/?$" +) + + +class DiscordAPIError(NativeIngestError): + """Base for native Discord client failures.""" + + +class DiscordAuthError(DiscordAPIError, NativeAuthError): + """401 (the token is invalid or expired) or a 403 on the channel the + source names. The fix is a new token, not a new client.""" + + +class DiscordDriftError(DiscordAPIError, NativeDriftError): + """A response did not have the shape the walk depends on.""" + + +def firefox_user_agent(today: date | None = None) -> str: + """gallery-dl's default User-Agent: a Firefox whose version advances every + four weeks (`util._ff_ver`, "147 on 2026-01-13"). Computed the same way so + the profile keeps matching the gallery-dl this replaced.""" + ver = ((today or date.today()).toordinal() - 735_513) // 28 + return ( + f"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:{ver}.0) " + f"Gecko/20100101 Firefox/{ver}.0" + ) + + +def nameext_from_url(url: str) -> tuple[str, str]: + """gallery-dl's `text.nameext_from_url`: the URL's last path segment, + unquoted, split at the last dot when the extension is at most 16 chars + (lowercased); otherwise the whole name and no extension.""" + filename = unquote(url.partition("?")[0].rpartition("/")[2]) + name, _, ext = filename.rpartition(".") + if name and len(ext) <= 16: + return name, ext.lower() + return filename, "" + + +def parse_source_url(url: str) -> tuple[str | None, str | None]: + """`(server_id, channel_id)` from a Discord channel/server URL. `server_id` + is None for a DM (`@me`); `channel_id` is None for a whole server. Raises + DiscordAPIError for anything else, including a link to a single message — + a message is not something a source can subscribe to.""" + m = _URL_RE.match((url or "").strip()) + if not m or (m.group("rest") or "").strip("/"): + raise DiscordAPIError( + f"Not a Discord channel or server link: {url!r} " + "(expected https://discord.com/channels/[/])" + ) + server = m.group("server") + channel = m.group("channel") + if server == "@me": + if not channel: + raise DiscordAPIError(f"A DM link needs a channel id: {url!r}") + return None, channel + return server, channel + + +def message_text(message: dict) -> str: + """gallery-dl's `extract_message_text`: the body plus the text of rich + embeds and polls, newline-joined, empties dropped.""" + parts = [message.get("content") or ""] + for embed in message.get("embeds") or []: + if embed.get("type") != "rich": + continue + parts.append((embed.get("author") or {}).get("name") or "") + parts.append(embed.get("title") or "") + parts.append(embed.get("description") or "") + for fld in embed.get("fields") or []: + parts.append(fld.get("name") or "") + parts.append(fld.get("value") or "") + parts.append((embed.get("footer") or {}).get("text") or "") + poll = message.get("poll") + if poll: + parts.append(((poll.get("question") or {}).get("text")) or "") + for answer in poll.get("answers") or []: + parts.append(((answer.get("poll_media") or {}).get("text")) or "") + return "\n".join(p for p in parts if p) + + +@dataclass +class MediaItem: + """One file of a Discord message. `filename`/`extension` are gallery-dl's + split of the URL; `num` is its 1-based position across the message's files, + which is what names it on disk. + + `media_id` is what the seen-ledger keys on, and it is deliberately NOT + `num`: an edit that removes a file renumbers the ones after it, and a + positional key would then call a different file seen. It is the + attachment's id, or for an embed (which has none) a hash of its URL path — + the query string is a signature that changes on every fetch. `filehash` is + always None; nothing in a signed CDN URL is a content hash.""" + + url: str + filename: str + extension: str + kind: str + post_id: str + num: int + media_id: str + filehash: str | None = None + + +class DiscordClient: + """Synchronous Discord API v10 read client for one user token.""" + + def __init__( + self, + token: str | None, + *, + request_sleep: float = 0.0, + max_retries: int = _MAX_429_RETRIES, + session: requests.Session | None = None, + ): + self._session = session or requests.Session() + self._session.headers.update({ + "User-Agent": firefox_user_agent(), + "Accept": "*/*", + "Accept-Language": "en-US,en;q=0.5", + "Referer": _ROOT + "/", + }) + if token: + self._session.headers["Authorization"] = token + self._token = token + self._request_sleep = request_sleep or 0.0 + self._max_retries = max_retries + self._server: dict = {} + self._channels: dict[str, dict] = {} + self._skip_feed = False + + # -- request ----------------------------------------------------------- + + def _get(self, endpoint: str, params: dict | None = None): + if not self._token: + raise DiscordAuthError("No Discord token is configured for this source") + if self._request_sleep > 0: + time.sleep(self._request_sleep) + url = API_ROOT + endpoint + attempt = 0 + while True: + try: + resp = self._session.get(url, params=params, timeout=_TIMEOUT_SECONDS) + except requests.RequestException as exc: + raise DiscordAPIError(f"Discord request failed ({endpoint}): {exc}") from exc + if resp.status_code == 429 and attempt < self._max_retries: + attempt += 1 + delay = retry_after_seconds( + resp, attempt, base=_429_WAIT_SECONDS, cap=_429_WAIT_SECONDS, + ) + log.warning( + "Discord 429 (%s) — waiting %.1fs (retry %d/%d)", + endpoint, delay, attempt, self._max_retries, + ) + time.sleep(delay) + continue + break + if resp.status_code == 401: + raise DiscordAuthError( + "Discord rejected the token (HTTP 401) — it is invalid or has " + "expired; copy a fresh one from the browser", + status_code=401, + ) + if resp.status_code != 200: + raise DiscordAPIError( + f"Discord returned HTTP {resp.status_code} ({endpoint})", + status_code=resp.status_code, + retry_after=_retry_after(resp), + ) + try: + return resp.json() + except ValueError as exc: + raise DiscordDriftError( + f"Discord returned non-JSON for {endpoint} ({len(resp.content)} bytes)" + ) from exc + + # -- metadata (gallery-dl parse_server / parse_channel) ----------------- + + def _load_server(self, server_id: str) -> None: + server = self._get(f"/guilds/{server_id}") + if not isinstance(server, dict) or "id" not in server: + raise DiscordDriftError(f"Discord server {server_id} came back without an id") + self._server = { + "server": server.get("name") or "", + "server_id": str(server["id"]), + "owner_id": server.get("owner_id"), + } + channels = self._get(f"/guilds/{server_id}/channels") + if not isinstance(channels, list): + raise DiscordDriftError(f"Discord server {server_id} channel list is not a list") + # Categories first, so every child can name its parent. + for channel in sorted(channels, key=lambda ch: ch.get("type") != _CATEGORY): + self._parse_channel(channel) + + def _parse_channel(self, channel: dict) -> dict: + parent_id = channel.get("parent_id") + meta = { + "channel": channel.get("name") or "", + "channel_id": str(channel.get("id")), + "channel_type": channel.get("type"), + "channel_topic": channel.get("topic") or "", + "parent_id": parent_id, + "is_thread": "thread_metadata" in channel, + } + parent = self._channels.get(parent_id) if parent_id else None + if parent: + meta["parent"] = parent["channel"] + meta["parent_type"] = parent["channel_type"] + if meta["channel_type"] in {1, 3}: + recipients = channel.get("recipients") or [] + meta["channel"] = "DMs" + meta["recipients"] = [u.get("username") for u in recipients] + meta["recipients_id"] = [u.get("id") for u in recipients] + self._channels[meta["channel_id"]] = meta + return meta + + def _channel_meta(self, channel_id: str) -> dict: + if channel_id not in self._channels: + self._parse_channel(self._get(f"/channels/{channel_id}")) + return self._channels[channel_id] + + def _threads(self, channel_id: str) -> list[dict]: + """Every thread of a channel or forum, newest-created first (see the + module docstring for why not last-message order).""" + threads: list[dict] = [] + offset = 0 + while True: + data = self._get(f"/channels/{channel_id}/threads/search", { + "sort_by": "last_message_time", + "sort_order": "desc", + "limit": _THREADS_BATCH, + "offset": offset, + }) + batch = (data.get("threads") or []) if isinstance(data, dict) else [] + threads.extend(batch) + if len(batch) < _THREADS_BATCH: + break + offset += len(batch) + threads.sort(key=lambda t: int(t.get("id") or 0), reverse=True) + return threads + + # -- the walk ------------------------------------------------------------ + + def _feeds(self, channel_id: str, *, safe: bool) -> Iterator[tuple[str, bool]]: + """`(channel_id, safe)` for every message feed under `channel_id`, in + gallery-dl's order. `safe` feeds are skipped on a 403.""" + try: + ctype = self._channel_meta(channel_id)["channel_type"] + except DiscordAPIError as exc: + if exc.status_code != 403: + raise + if not safe: + raise DiscordAuthError( + f"The Discord token cannot see channel {channel_id} (HTTP 403)", + status_code=403, + ) from exc + log.info("Discord: no access to channel %s — skipped", channel_id) + return + if ctype in _TEXT or ctype in _DIRECT: + yield channel_id, safe + if ctype in _TEXT or ctype in _FORUM: + try: + threads = self._threads(channel_id) + except DiscordAPIError as exc: + if exc.status_code != 403: + raise + log.info("Discord: cannot list threads of %s — skipped", channel_id) + threads = [] + for thread in threads: + yield self._parse_channel(thread)["channel_id"], True + elif ctype == _CATEGORY: + for child in list(self._channels.values()): + if child.get("parent_id") == channel_id: + yield from self._feeds(child["channel_id"], safe=True) + elif ctype not in _DIRECT and not safe: + raise DiscordAPIError( + f"Discord channel {channel_id} is of type {ctype}, which has no messages" + ) + + def _source_feeds(self, url: str) -> Iterator[tuple[str, bool]]: + server_id, channel_id = parse_source_url(url) + self._server, self._channels = {}, {} + if server_id is not None: + self._load_server(server_id) + if channel_id is not None: + yield from self._feeds(channel_id, safe=False) + return + for meta in list(self._channels.values()): + if meta["channel_type"] in _SERVER_WALK: + yield from self._feeds(meta["channel_id"], safe=True) + + def skip_feed(self) -> None: + """Optional core seam (#4413): end the current channel and go on to the + next one. A tick's early-out means THIS channel has nothing new, not + that the server has nothing new.""" + self._skip_feed = True + + def iter_posts( + self, campaign_id: str, cursor: str | None = None + ) -> Iterator[tuple[dict, dict, str | None]]: + """Yield `(message, channel_meta, page_cursor)` for every content + message the source reaches, channel by channel, each newest first. + + `campaign_id` is the source URL. The cursor is `:` + — the channel and the `before` id that fetched the page (empty for a + channel's first page) — so a backfill resumes inside the right channel + and re-fetches the page it was cut in. A cursor naming a channel the + walk no longer reaches (a deleted thread) restarts from the top rather + than walking nothing. + """ + resume_channel, _, resume_before = (cursor or "").partition(":") + resuming = bool(resume_channel) + feeds = list(self._source_feeds(campaign_id)) if resuming else None + if feeds is not None and resume_channel not in {cid for cid, _ in feeds}: + log.warning( + "Discord: resume channel %s is no longer in %s — restarting", + resume_channel, campaign_id, + ) + resuming = False + for channel_id, safe in feeds if feeds is not None else self._source_feeds(campaign_id): + before = None + if resuming: + if channel_id != resume_channel: + continue + resuming = False + before = resume_before or None + yield from self._iter_channel(channel_id, before, safe=safe) + + def _iter_channel( + self, channel_id: str, before: str | None, *, safe: bool + ) -> Iterator[tuple[dict, dict, str | None]]: + self._skip_feed = False + meta = {**self._server, **self._channels.get(channel_id, {})} + while True: + page_cursor = f"{channel_id}:{before or ''}" + try: + messages = self._get( + f"/channels/{channel_id}/messages", + {"limit": _MESSAGES_BATCH, "before": before}, + ) + except DiscordAPIError as exc: + if exc.status_code != 403: + raise + if not safe: + raise DiscordAuthError( + f"The Discord token cannot read channel {channel_id} (HTTP 403)", + status_code=403, + ) from exc + log.info("Discord: no access to messages of %s — skipped", channel_id) + return + if not isinstance(messages, list): + raise DiscordDriftError( + f"Discord messages of {channel_id} came back as " + f"{type(messages).__name__}, not a list" + ) + for message in messages: + if message.get("type") not in MESSAGE_TYPES: + continue + message["_meta"] = meta + yield message, meta, page_cursor + if self._skip_feed: + return + if len(messages) < _MESSAGES_BATCH: + return + before = str(messages[-1]["id"]) + + # -- per-message ------------------------------------------------------- + + @staticmethod + def extract_media(post: dict, included: dict | None = None) -> list[MediaItem]: + """gallery-dl's file list for one message: attachments, then the first + of video/image/thumbnail `proxy_url` of each file-bearing embed, then + the same for every forwarded snapshot; numbered from 1 across them.""" + mid = str(post.get("id") or "") + snapshots = [post] + [ + (s or {}).get("message") or {} + for s in post.get("message_snapshots") or [] + if ((s or {}).get("message") or {}).get("type", 0) in MESSAGE_TYPES + ] + found: list[tuple[str, str, str | None]] = [] + for snap in snapshots: + for att in snap.get("attachments") or []: + if att.get("url"): + aid = att.get("id") + found.append((att["url"], "attachment", str(aid) if aid else None)) + for embed in snap.get("embeds") or []: + if embed.get("type") not in _EMBED_TYPES: + continue + for fld in ("video", "image", "thumbnail"): + url = (embed.get(fld) or {}).get("proxy_url") + if url: + found.append((url, "embed", None)) + break + items = [] + for num, (url, kind, fid) in enumerate(found, start=1): + name, ext = nameext_from_url(url) + if fid is None: + path = url.partition("?")[0].encode() + fid = "u" + hashlib.sha1(path, usedforsecurity=False).hexdigest()[:32] + items.append(MediaItem( + url=url, filename=name, extension=ext, kind=kind, post_id=mid, + num=num, media_id=fid, + )) + return items + + @staticmethod + def post_meta(post: dict) -> dict: + """No title (Discord has none); `date` is the message timestamp, ISO + with an offset — what the core's revisit window reads.""" + return {"title": None, "date": post.get("timestamp")} + + @classmethod + def post_record_key(cls, post: dict) -> tuple[str, str] | None: + """`(message:, )` — gates the message record through the seen + ledger, like `post:` on the other platforms. + + None for a message with no files. gallery-dl wrote a sidecar only + beside a file, so a text-only chat line never became a post, and the + drop grouping (discord_grouping) is built on that: a channel's chatter + recorded as posts would bury the drops it exists to surface.""" + mid = post.get("id") + mid = str(mid) if mid is not None else "" + if not mid or not cls.extract_media(post): + return None + return (f"message:{mid}", mid) + + # -- verify ------------------------------------------------------------ + + def verify_auth(self, url: str) -> tuple[bool | None, str]: + """Is the token valid, and can it see what the source names?""" + try: + server_id, channel_id = parse_source_url(url) + except DiscordAPIError as exc: + return None, str(exc) + try: + me = self._get("/users/@me") + if channel_id is not None: + self._get(f"/channels/{channel_id}") + elif server_id is not None: + self._get(f"/guilds/{server_id}") + except DiscordAuthError as exc: + return False, f"Discord rejected the token — {exc}" + except DiscordAPIError as exc: + if exc.status_code in (403, 404): + return False, ( + "The token is valid, but its account cannot see " + f"{'this channel' if channel_id else 'this server'} " + f"(HTTP {exc.status_code})" + ) + return None, f"Couldn't verify (network/HTTP issue): {exc}" + who = (me or {}).get("username") if isinstance(me, dict) else None + return True, f"Token valid{f' ({who})' if who else ''} — the source is readable." + + +def _retry_after(resp: requests.Response) -> float | None: + hdr = resp.headers.get("Retry-After") + try: + return float(hdr) if hdr else None + except (TypeError, ValueError): + return None diff --git a/backend/app/services/discord_downloader.py b/backend/app/services/discord_downloader.py new file mode 100644 index 0000000..ffd8734 --- /dev/null +++ b/backend/app/services/discord_downloader.py @@ -0,0 +1,210 @@ +"""Native Discord media downloader — the Discord counterpart to +subscribestar_downloader. + +Writes files exactly where gallery-dl wrote them, so a cutover finds every +existing file on disk (`skipped_disk`) instead of fetching it again: + + //discord//___. + +That is FC's gallery-dl config (`gallery_dl.DISCORD_DIRECTORY` / +`DISCORD_FILENAME`) under the per-source base directory +`//`. The name is cleaned the way gallery-dl cleans it +on Linux — `/` becomes `_` and control characters are removed, nothing else +(`path-restrict: auto`, `path-remove` defaults). It is NOT `sanitize_segment`, +whose Windows set would turn a `:` in a channel or file name into `_` and miss +the file gallery-dl wrote. + +Post-first (rule 120): each file gets a minimal sidecar named like it minus the +extension (what `find_sidecar` pairs first), and the message itself gets one +record, `__post.json`, carrying gallery-dl's metadata keys +— `message_id` for the post id, `server_id`/`channel_id` for the permalink, +`message` for the body, `date` — so `parse_sidecar` reads it exactly as it read +the gallery-dl sidecars. Neither file carries an `id` or `post_id` key: both +outrank `message_id` in the post-id chain (`platforms.base`). + +PURE: no DB; the seen-skip is an injected predicate. +""" + +from __future__ import annotations + +import json +import logging +import re +import time +from collections.abc import Callable +from datetime import UTC, datetime +from pathlib import Path + +import requests + +from .discord_client import firefox_user_agent, message_text +from .native_ingest_common import ( + BaseNativeDownloader, + MediaOutcome, + PostRecordOutcome, + make_session, +) + +log = logging.getLogger(__name__) + +PLATFORM = "discord" +_CONTROL = re.compile("[\x00-\x1f\x7f]") +# gallery-dl falls back to the response's type for a URL with no extension; +# we never see the response before naming, and such URLs do not occur for +# Discord attachments or embed proxies in practice. +_NO_EXTENSION = "bin" + + +def gdl_clean(segment: str) -> str: + """One path segment as gallery-dl writes it on Linux.""" + return _CONTROL.sub("", segment.replace("/", "_")) + + +def message_date(post: dict) -> datetime | None: + raw = post.get("timestamp") + if not isinstance(raw, str) or not raw: + return None + try: + dt = datetime.fromisoformat(raw.replace("Z", "+00:00")) + except ValueError: + return None + return (dt if dt.tzinfo else dt.replace(tzinfo=UTC)).astimezone(UTC) + + +def channel_dir(images_root: Path, artist_slug: str, post: dict) -> Path: + """gallery-dl's `{channel}` directory; an empty name adds no segment.""" + base = Path(images_root) / artist_slug / PLATFORM + channel = gdl_clean(((post.get("_meta") or {}).get("channel") or "").strip()) + return base / channel if channel else base + + +def media_stem(post: dict, media) -> str: + """`___` — the file's name minus `.`.""" + when = message_date(post) + day = f"{when:%Y%m%d}" if when else "None" + return gdl_clean(f"{day}_{post.get('id')}_{media.num:>02}_{media.filename}") + + +class DiscordDownloader(BaseNativeDownloader): + """Download a message's files to gallery-dl's layout. The CDN gets + gallery-dl's browser profile and no token — gallery-dl sends the token only + to the API, and the CDN URLs are pre-signed.""" + + def __init__( + self, + images_root: Path, + cookies_path: str | None = None, + *, + validate: bool = True, + rate_limit: float = 0.0, + session: requests.Session | None = None, + ): + super().__init__( + images_root, None, platform=PLATFORM, + validate=validate, rate_limit=rate_limit, + session=session if session is not None else make_session(None, extra_headers={ + "User-Agent": firefox_user_agent(), + "Accept-Language": "en-US,en;q=0.5", + "Referer": "https://discord.com/", + }), + ) + + def download_post( + self, + post: dict, + media_items: list, + artist_slug: str, + *, + is_seen: Callable[[object], bool] = lambda m: False, + should_stop: Callable[[], bool] = lambda: False, + recapture: bool = False, + ) -> list[MediaOutcome]: + """Every file of one message; per-file outcomes, one failure isolated.""" + folder = channel_dir(self.images_root, artist_slug, post) + outcomes: list[MediaOutcome] = [] + for media in media_items: + if should_stop(): + break + try: + outcomes.append(self._download_one( + post, media, folder, artist_slug, is_seen, recapture=recapture, + )) + except Exception as exc: # resilient: isolate one item's failure + log.warning( + "Discord media failed (message %s, file %d): %s", + post.get("id"), media.num, exc, + ) + outcomes.append( + MediaOutcome(media=media, status="error", path=None, error=str(exc)) + ) + return outcomes + + def _download_one( + self, + post: dict, + media, + folder: Path, + artist_slug: str, + is_seen: Callable[[object], bool], + *, + recapture: bool = False, + ) -> MediaOutcome: + seen = is_seen(media) + if seen and not recapture: + return MediaOutcome(media=media, status="skipped_seen", path=None, error=None) + stem = media_stem(post, media) + path = folder / f"{stem}.{media.extension or _NO_EXTENSION}" + if path.exists(): # tier-2: gallery-dl (or an earlier walk) wrote it + return MediaOutcome(media=media, status="skipped_disk", path=path, error=None) + if seen: # recapture never re-fetches a seen file that is gone + return MediaOutcome(media=media, status="skipped_seen", path=None, error=None) + + folder.mkdir(parents=True, exist_ok=True) + if self._rate_limit > 0: + time.sleep(self._rate_limit) + out = self._fetch_get(media.url, path) + reason, quarantined = self._validate_path(out, artist_slug, media.url) + if reason is not None: + return MediaOutcome(media=media, status="quarantined", path=quarantined, error=reason) + sidecar = {"category": PLATFORM, "message_id": str(post.get("id") or "")} + sidecar["source_url"] = media.url + (folder / f"{stem}.json").write_text(json.dumps(sidecar, indent=2)) + return MediaOutcome(media=media, status="downloaded", path=out, error=None) + + def write_post_record( + self, post: dict, artist_slug: str, *, revisit: bool = False, + ) -> PostRecordOutcome: + """The message record — the one writer of a Discord post's body, date + and permalink ids. `revisit` re-reads a message already captured (an + edit); an empty re-read writes nothing, so it never blanks a body.""" + mid = str(post.get("id") or "") + body = message_text(post) + if not mid or (revisit and not body.strip()): + return PostRecordOutcome(path=None, post_type=None, title=None, body_chars=0) + meta = post.get("_meta") or {} + author = post.get("author") or {} + record = { + "category": PLATFORM, + "message_id": mid, + "server": meta.get("server"), + "server_id": meta.get("server_id"), + "channel": meta.get("channel"), + "channel_id": meta.get("channel_id") or post.get("channel_id"), + "parent": meta.get("parent"), + "is_thread": meta.get("is_thread"), + "author": author.get("username"), + "author_id": author.get("id"), + "message": body, + "date": post.get("timestamp"), + } + folder = channel_dir(self.images_root, artist_slug, post) + folder.mkdir(parents=True, exist_ok=True) + when = message_date(post) + day = f"{when:%Y%m%d}" if when else "None" + path = folder / f"{day}_{mid}_post.json" + path.write_text(json.dumps( + {k: v for k, v in record.items() if v is not None}, indent=2, + )) + return PostRecordOutcome( + path=path, post_type=None, title=None, body_chars=len(body), + ) diff --git a/backend/app/services/discord_ingester.py b/backend/app/services/discord_ingester.py new file mode 100644 index 0000000..f83721f --- /dev/null +++ b/backend/app/services/discord_ingester.py @@ -0,0 +1,85 @@ +"""Native Discord ingester — the Discord ADAPTER over `ingest_core.Ingester`. + +Thin counterpart to subscribestar_ingester (milestone 428). The walk's modes, +both ledgers, cursor checkpointing and the post-first capture live in the core; +this wires in the Discord client, downloader, ledger models and key. + +Two things differ from the cookie platforms: + + - Discord authenticates with a user TOKEN, so `auth_token` is the credential + here rather than an argument accepted and ignored. + - The body canary is off. It fails a walk whose first 30+ captured posts all + came back without text, on the theory that a creator nearly always writes + something; a Discord drop is routinely files and nothing else, so on + Discord that is an ordinary backfill, not a broken parser. + +`campaign_id` is the source URL (a server, channel, thread or category link). +FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Callable +from pathlib import Path + +from ..models import DiscordFailedMedia, DiscordSeenMedia +from .discord_client import DiscordAPIError, DiscordClient, MediaItem +from .discord_downloader import DiscordDownloader +from .ingest_core import Ingester + +_LEDGER_KEY_MAX = 128 + + +def _ledger_key(media: MediaItem) -> str: + """`:` — stable across edits (see MediaItem).""" + return f"{media.post_id}:{media.media_id}"[:_LEDGER_KEY_MAX] + + +class DiscordIngester(Ingester): + """Walk a Discord source's channels, download unseen files, return a + `DownloadResult`. `client` / `downloader` are injectable for tests.""" + + def __init__( + self, + images_root: Path, + cookies_path: str | None, + session_factory: Callable[[], object], + *, + validate: bool = True, + rate_limit: float = 0.0, + request_sleep: float = 0.0, + auth_token: str | None = None, + client: DiscordClient | None = None, + downloader: DiscordDownloader | None = None, + ): + del cookies_path # Discord authenticates by token (uniform signature) + self.images_root = Path(images_root) + super().__init__( + client=client if client is not None else DiscordClient( + auth_token, request_sleep=request_sleep, + ), + downloader=downloader if downloader is not None else DiscordDownloader( + self.images_root, validate=validate, rate_limit=rate_limit, + ), + session_factory=session_factory, + seen_model=DiscordSeenMedia, + failed_model=DiscordFailedMedia, + seen_constraint="uq_discord_seen_media_source_id", + failed_constraint="uq_discord_failed_media_source_id", + ledger_key=_ledger_key, + platform="discord", + error_base=DiscordAPIError, + drift_label="Discord API", + body_canary=False, + ) + + +async def verify_discord_credential(url: str, auth_token: str | None) -> tuple[bool | None, str]: + """The uniform `(ok, message)` probe: is the token valid, and can its + account see the channel or server the source names?""" + if not auth_token: + return False, "No Discord token is saved — add one under Credentials." + client = DiscordClient(auth_token) + loop = asyncio.get_running_loop() + return await loop.run_in_executor(None, client.verify_auth, url) diff --git a/backend/app/services/download_backends.py b/backend/app/services/download_backends.py index e2382f7..74fc88b 100644 --- a/backend/app/services/download_backends.py +++ b/backend/app/services/download_backends.py @@ -23,6 +23,7 @@ from __future__ import annotations import asyncio from pathlib import Path +from .discord_ingester import DiscordIngester from .gallery_dl import DownloadResult, ErrorType from .ingest_core import DEFAULT_REVISIT_DAYS from .patreon_ingester import PatreonIngester @@ -31,9 +32,13 @@ from .platforms import known_platform_keys from .subscribestar_ingester import SubscribeStarIngester # Platforms whose download + verify go through the native ingester rather than -# gallery-dl. gallery-dl still serves the rest (hentaifoundry, discord) until -# they migrate too. -NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar"}) +# gallery-dl. gallery-dl still serves the rest (hentaifoundry) until it +# migrates too. Discord joined in milestone 428. +NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar", "discord"}) + +# Native platforms whose feed id IS the source URL, so there is nothing to +# resolve: SubscribeStar's creator page, Discord's server/channel link. +_URL_IS_FEED = frozenset({"subscribestar", "discord"}) def _unsupported_platform_message(platform: str) -> str | None: @@ -67,6 +72,7 @@ def _native_ingester_cls(platform: str): return { "patreon": PatreonIngester, "subscribestar": SubscribeStarIngester, + "discord": DiscordIngester, }[platform] @@ -126,10 +132,10 @@ async def _resolve_native_campaign_id( platform: str, url: str, cookies_path: str | None, overrides: dict, ) -> tuple[str | None, str | None]: """`(campaign_id, resolved_campaign_id)` for a native source. SubscribeStar's - feed id IS the creator URL (no lookup → resolved None). Patreon resolves the - campaign id from the vanity URL (resolved non-None when a lookup actually ran, - so phase 3 caches it).""" - if platform == "subscribestar": + and Discord's feed id IS the source URL (no lookup → resolved None). Patreon + resolves the campaign id from the vanity URL (resolved non-None when a lookup + actually ran, so phase 3 caches it).""" + if platform in _URL_IS_FEED: return url, None return await resolve_campaign_id_for_source(url, cookies_path, overrides) @@ -252,6 +258,10 @@ async def verify_source_credential( from .subscribestar_ingester import verify_subscribestar_credential return await verify_subscribestar_credential(url, cookies_path, config_overrides) + if platform == "discord": + from .discord_ingester import verify_discord_credential + + return await verify_discord_credential(url, auth_token) from .patreon_ingester import verify_patreon_credential return await verify_patreon_credential(url, cookies_path, config_overrides) diff --git a/backend/app/services/ingest_core.py b/backend/app/services/ingest_core.py index 6030cad..ffc461a 100644 --- a/backend/app/services/ingest_core.py +++ b/backend/app/services/ingest_core.py @@ -264,6 +264,14 @@ class Ingester: # operator-driven "re-read every body" pass; a horizon there would be a # third overlapping answer to a question that has two. post_meta = getattr(self.client, "post_meta", None) + # #4413: optional client seam for a source that is several feeds walked + # one after another (a Discord server: every channel and thread). The + # tick early-out means "this feed has nothing new", and without the + # seam it ends the WHOLE walk — so the first quiet channel would hide + # every channel after it. With it, the early-out asks the client to + # move on and the walk continues. Absent → the early-out ends the walk, + # exactly as before (Patreon and SubscribeStar are one feed each). + skip_feed = getattr(self.client, "skip_feed", None) horizon: datetime | None = None if mode == "tick" and revisit_days > 0 and post_meta is not None: horizon = datetime.now(UTC) - timedelta(days=revisit_days) @@ -312,6 +320,7 @@ class Ingester: reached_bottom = False budget_hit = False early_out = False + feeds_caught_up = 0 # #4413: feeds a tick left early via skip_feed stopped = False # plan #708 B4: operator hit Stop mid-walk cancel_armed = False # latched once we observe a live "running" state @@ -674,9 +683,15 @@ class Ingester: }) if early_out: - break + if skip_feed is None: + break + skip_feed() + feeds_caught_up += 1 + early_out = False + consecutive_seen = 0 else: - reached_bottom = True + # A walk that left feeds early did not read to their ends. + reached_bottom = not feeds_caught_up except self._error_base as exc: # The platform's client-error base — _failure_result (adapter) # maps it to a typed error. @@ -724,6 +739,7 @@ class Ingester: f", {revisited} post(s) updated ({revisit_downloads} new file(s))" if revisited else "" ) + + (f", {feeds_caught_up} feed(s) caught up" if feeds_caught_up else "") + (", reached end" if reached_bottom else "") + (", time-boxed" if budget_hit else "") ) diff --git a/backend/app/services/platform_lock.py b/backend/app/services/platform_lock.py index 27d029e..d910016 100644 --- a/backend/app/services/platform_lock.py +++ b/backend/app/services/platform_lock.py @@ -23,8 +23,10 @@ log = logging.getLogger(__name__) # Platforms walked one-at-a-time. gallery-dl platforms are intentionally NOT # here: each runs as a self-pacing subprocess and they're lower-volume. The # native-ingester platforms are serialized (one paced scrape/API walk at a time). -# Add a platform here to cap it to a single concurrent walk. -SERIALIZED_PLATFORMS = frozenset({"patreon", "subscribestar"}) +# Add a platform here to cap it to a single concurrent walk. Discord most of +# all: every source walks on the operator's ONE user token, and parallel walks +# on a user account are both how its rate limit trips and what gets it flagged. +SERIALIZED_PLATFORMS = frozenset({"patreon", "subscribestar", "discord"}) _LOCK_PREFIX = "fc:download_lock:" diff --git a/tests/test_discord_client.py b/tests/test_discord_client.py new file mode 100644 index 0000000..230afde --- /dev/null +++ b/tests/test_discord_client.py @@ -0,0 +1,339 @@ +"""The native Discord client walks what gallery-dl walked, in its order (#4412). + +No network: a fake session answers by endpoint. What these pin is the part of +gallery-dl's behaviour that decides WHICH files exist and WHAT they are called — +the channel walk, the file list and its numbering, the name split — because a +difference there is a re-download or a missed file at cutover, not a style +choice. +""" + +from __future__ import annotations + +from datetime import date + +import pytest + +from backend.app.services import discord_client as dc +from backend.app.services.discord_client import ( + DiscordAPIError, + DiscordAuthError, + DiscordClient, + firefox_user_agent, + message_text, + nameext_from_url, + parse_source_url, +) + + +class _Resp: + def __init__(self, status, body=None, headers=None): + self.status_code = status + self._body = body + self.headers = headers or {} + self.content = b"x" + + def json(self): + return self._body + + +class _Session: + """Answers `GET API_ROOT + endpoint` from a dict. A value may be a list of + responses, served in order; a messages entry is keyed by (endpoint, before).""" + + def __init__(self, routes): + self.routes = routes + self.headers = {} + self.calls = [] + + def get(self, url, params=None, timeout=None): + endpoint = url[len(dc.API_ROOT):] + self.calls.append((endpoint, dict(params or {}))) + key = endpoint + if endpoint.endswith("/messages"): + key = (endpoint, (params or {}).get("before")) + elif endpoint.endswith("/threads/search"): + key = (endpoint, (params or {}).get("offset")) + answer = self.routes.get(key, _Resp(404, {})) + if isinstance(answer, list): + return answer.pop(0) + return answer + + +def _ok(body): + return _Resp(200, body) + + +def _msg(mid, *, content="", attachments=(), embeds=(), **extra): + return { + "id": str(mid), "type": 0, "content": content, + "timestamp": "2026-09-20T12:00:00.000000+00:00", + "author": {"id": "7", "username": "artist"}, + "attachments": list(attachments), "embeds": list(embeds), **extra, + } + + +def _client(routes, **kw): + return DiscordClient("tok", session=_Session(routes), **kw) + + +def _ids(client, url, cursor=None): + return [(m["id"], cur) for m, _meta, cur in client.iter_posts(url, cursor)] + + +# -- pure helpers -------------------------------------------------------------- + +def test_source_urls(): + assert parse_source_url("https://discord.com/channels/1/2") == ("1", "2") + assert parse_source_url("https://discord.com/channels/1") == ("1", None) + assert parse_source_url("https://discord.com/channels/1/2/threads/3") == ("1", "3") + assert parse_source_url("https://discord.com/channels/@me/5") == (None, "5") + assert parse_source_url("discord.com/channels/1/2/") == ("1", "2") + + +def test_a_message_link_is_not_a_source(): + with pytest.raises(DiscordAPIError): + parse_source_url("https://discord.com/channels/1/2/3") + with pytest.raises(DiscordAPIError): + parse_source_url("https://example.com/channels/1/2") + + +def test_name_split_matches_gallery_dl(): + url = "https://cdn.discordapp.com/attachments/1/2/My%20Pic.final.PNG?ex=a&hm=b" + assert nameext_from_url(url) == ("My Pic.final", "png") + assert nameext_from_url("https://x/y/noext") == ("noext", "") + assert nameext_from_url("https://x/y/a." + "b" * 17) == ("a." + "b" * 17, "") + + +def test_user_agent_is_gallery_dls_dated_firefox(): + """gallery-dl's own comment: "147 on 2026-01-13".""" + assert "Firefox/147.0" in firefox_user_agent(date(2026, 1, 13)) + + +def test_message_text_takes_rich_embeds_and_polls(): + m = _msg(1, content="hello", embeds=[ + {"type": "rich", "author": {"name": "A"}, "title": "T", + "fields": [{"name": "f", "value": "v"}], "footer": {"text": "ft"}}, + {"type": "image", "title": "not text"}, + ], poll={"question": {"text": "Q?"}, "answers": [{"poll_media": {"text": "yes"}}]}) + assert message_text(m) == "hello\nA\nT\nf\nv\nft\nQ?\nyes" + + +def test_files_are_attachments_then_embeds_then_snapshots_numbered_across(): + m = _msg(9, attachments=[{"url": "https://cdn/a/1.png"}], embeds=[ + {"type": "video", "video": {"proxy_url": "https://media/v.mp4"}, + "thumbnail": {"proxy_url": "https://media/t.jpg"}}, + {"type": "image", "thumbnail": {"proxy_url": "https://media/i.webp"}}, + {"type": "rich", "image": {"proxy_url": "https://media/rich.png"}}, + ], message_snapshots=[ + {"message": {"type": 0, "attachments": [{"url": "https://cdn/a/fwd.gif"}], + "embeds": []}}, + {"message": {"type": 7, "attachments": [{"url": "https://cdn/a/join.png"}]}}, + ]) + items = DiscordClient.extract_media(m) + assert [(i.num, i.filename, i.extension, i.kind) for i in items] == [ + (1, "1", "png", "attachment"), + (2, "v", "mp4", "embed"), + (3, "i", "webp", "embed"), + (4, "fwd", "gif", "attachment"), + ] + assert {i.post_id for i in items} == {"9"} + + +def test_the_ledger_identity_survives_a_renumbering_edit(): + """Removing the first file renumbers the second; its identity must not move.""" + a = {"id": "100", "url": "https://cdn/a/1.png?ex=1"} + b = {"id": "200", "url": "https://cdn/a/2.png?ex=1"} + before = DiscordClient.extract_media(_msg(9, attachments=[a, b])) + after = DiscordClient.extract_media(_msg(9, attachments=[b])) + assert (before[1].num, after[0].num) == (2, 1) + assert before[1].media_id == after[0].media_id == "200" + + +def test_an_embeds_identity_ignores_its_signature(): + def embed(sig): + return {"type": "image", "image": {"proxy_url": f"https://media/p/x.png?ex={sig}"}} + + one = DiscordClient.extract_media(_msg(9, embeds=[embed("a")]))[0].media_id + two = DiscordClient.extract_media(_msg(9, embeds=[embed("b")]))[0].media_id + assert one == two and len(one) <= 33 + + +def test_post_seams(): + with_file = _msg(5, attachments=[{"url": "https://cdn/a/1.png"}]) + assert DiscordClient.post_record_key(with_file) == ("message:5", "5") + assert DiscordClient.post_record_key({}) is None + + +def test_a_text_only_message_is_not_a_post(): + """gallery-dl never made one: chat lines would bury the drops.""" + assert DiscordClient.post_record_key(_msg(6, content="brb")) is None + assert DiscordClient.post_meta(_msg(1))["date"].startswith("2026-09-20") + + +# -- the walk -------------------------------------------------------------------- + +def test_a_channel_pages_newest_first_and_skips_system_messages(): + page1 = [_msg(i) for i in range(300, 200, -1)] + page1[3]["type"] = 7 # a member-join line: not content + routes = { + "/guilds/1": _ok({"id": "1", "name": "S"}), + "/guilds/1/channels": _ok([{"id": "2", "type": 0, "name": "art"}]), + ("/channels/2/messages", None): _ok(page1), + ("/channels/2/messages", "201"): _ok([_msg(150)]), + ("/channels/2/threads/search", 0): _ok({"threads": []}), + } + got = _ids(_client(routes), "https://discord.com/channels/1/2") + assert len(got) == 100 # 99 of page 1 + 1 of page 2 + assert "297" not in [mid for mid, _ in got] + assert got[0] == ("300", "2:") + assert got[-1] == ("150", "2:201") + + +def test_messages_carry_server_and_channel_metadata(): + routes = { + "/guilds/1": _ok({"id": "1", "name": "Studio", "owner_id": "9"}), + "/guilds/1/channels": _ok([ + {"id": "4", "type": 4, "name": "Art"}, + {"id": "2", "type": 0, "name": "drops", "parent_id": "4"}, + ]), + ("/channels/2/messages", None): _ok([_msg(10)]), + ("/channels/2/threads/search", 0): _ok({"threads": []}), + } + [(msg, meta, _)] = list(_client(routes).iter_posts("https://discord.com/channels/1/2")) + assert msg["_meta"] is meta + assert meta["server"] == "Studio" and meta["server_id"] == "1" + assert meta["channel"] == "drops" and meta["channel_id"] == "2" + assert meta["parent"] == "Art" + + +def test_a_server_walks_text_then_threads_newest_created_first_and_skips_private(): + routes = { + "/guilds/1": _ok({"id": "1", "name": "S"}), + "/guilds/1/channels": _ok([ + {"id": "2", "type": 0, "name": "text"}, + {"id": "3", "type": 2, "name": "voice"}, + {"id": "5", "type": 15, "name": "forum"}, + {"id": "6", "type": 0, "name": "private"}, + ]), + ("/channels/2/messages", None): _ok([_msg(20)]), + ("/channels/2/threads/search", 0): _ok({"threads": [ + {"id": "21", "type": 11, "name": "old", "parent_id": "2", "thread_metadata": {}}, + {"id": "22", "type": 11, "name": "new", "parent_id": "2", "thread_metadata": {}}, + ]}), + ("/channels/22/messages", None): _ok([_msg(220)]), + ("/channels/21/messages", None): _Resp(403, {}), # a private thread + ("/channels/5/threads/search", 0): _ok({"threads": [ + {"id": "51", "type": 11, "name": "post", "parent_id": "5", "thread_metadata": {}}, + ]}), + ("/channels/51/messages", None): _ok([_msg(510)]), + ("/channels/6/messages", None): _Resp(403, {}), + ("/channels/6/threads/search", 0): _Resp(403, {}), + } + got = [mid for mid, _ in _ids(_client(routes), "https://discord.com/channels/1")] + assert got == ["20", "220", "510"] + + +def test_a_resume_cursor_reenters_its_channel_at_its_page(): + routes = { + "/guilds/1": _ok({"id": "1", "name": "S"}), + "/guilds/1/channels": _ok([ + {"id": "2", "type": 0, "name": "a"}, + {"id": "3", "type": 0, "name": "b"}, + ]), + ("/channels/2/threads/search", 0): _ok({"threads": []}), + ("/channels/3/threads/search", 0): _ok({"threads": []}), + ("/channels/3/messages", "77"): _ok([_msg(70)]), + } + client = _client(routes) + assert _ids(client, "https://discord.com/channels/1", "3:77") == [("70", "3:77")] + fetched = [c for c in client._session.calls if c[0].endswith("/messages")] + assert fetched == [("/channels/3/messages", {"limit": 100, "before": "77"})] + + +def test_skip_feed_ends_the_channel_not_the_walk(): + routes = { + "/guilds/1": _ok({"id": "1", "name": "S"}), + "/guilds/1/channels": _ok([ + {"id": "2", "type": 0, "name": "a"}, + {"id": "3", "type": 0, "name": "b"}, + ]), + ("/channels/2/messages", None): _ok([_msg(29), _msg(28)]), + ("/channels/2/threads/search", 0): _ok({"threads": []}), + ("/channels/3/messages", None): _ok([_msg(39)]), + ("/channels/3/threads/search", 0): _ok({"threads": []}), + } + client = _client(routes) + seen = [] + for msg, _meta, _cur in client.iter_posts("https://discord.com/channels/1"): + seen.append(msg["id"]) + if msg["id"] == "29": + client.skip_feed() + assert seen == ["29", "39"] + + +def test_the_named_channel_refusing_the_token_is_an_auth_failure(): + routes = { + "/guilds/1": _ok({"id": "1", "name": "S"}), + "/guilds/1/channels": _ok([{"id": "2", "type": 0, "name": "a"}]), + ("/channels/2/messages", None): _Resp(403, {}), + } + with pytest.raises(DiscordAuthError): + list(_client(routes).iter_posts("https://discord.com/channels/1/2")) + + +def test_401_is_an_invalid_token(): + with pytest.raises(DiscordAuthError): + list(_client({"/guilds/1": _Resp(401, {})}).iter_posts( + "https://discord.com/channels/1")) + + +def test_no_token_fails_before_any_request(): + client = DiscordClient(None, session=_Session({})) + with pytest.raises(DiscordAuthError): + list(client.iter_posts("https://discord.com/channels/1")) + assert client._session.calls == [] + + +def test_429_waits_and_retries(monkeypatch): + waits = [] + monkeypatch.setattr(dc.time, "sleep", waits.append) + routes = {"/users/@me": [ + _Resp(429, {}, {"Retry-After": "1.5"}), _ok({"username": "me"}), + ], "/channels/2": _ok({"id": "2", "type": 0})} + ok, msg = _client(routes).verify_auth("https://discord.com/channels/1/2") + assert ok is True and "me" in msg + assert waits == [1.5] + + +def test_verify_tells_a_bad_token_from_a_hidden_channel(): + bad = _client({"/users/@me": _Resp(401, {})}) + assert bad.verify_auth("https://discord.com/channels/1/2")[0] is False + hidden = _client({"/users/@me": _ok({"username": "me"}), + "/channels/2": _Resp(403, {})}) + ok, msg = hidden.verify_auth("https://discord.com/channels/1/2") + assert ok is False and "cannot see this channel" in msg + assert _client({}).verify_auth("https://discord.com/channels/1/2/3")[0] is None + + +def test_the_request_profile_is_gallery_dls(): + client = _client({}) + h = client._session.headers + assert h["Authorization"] == "tok" + assert h["Referer"] == "https://discord.com/" + assert h["Accept"] == "*/*" + assert h["User-Agent"].startswith("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:") + + +# -- the adapter --------------------------------------------------------------- + +def test_the_adapter_authenticates_with_the_token_and_keys_by_identity(tmp_path): + from backend.app.services.discord_ingester import DiscordIngester, _ledger_key + + ing = DiscordIngester(tmp_path, None, session_factory=None, auth_token="tok") + assert ing.client._session.headers["Authorization"] == "tok" + # A files-only drop is ordinary on Discord, not a broken parser. + assert ing._body_canary is False + [media] = DiscordClient.extract_media( + _msg(9, attachments=[{"id": "300", "url": "https://cdn/a/1.png"}]) + ) + assert _ledger_key(media) == "9:300" diff --git a/tests/test_discord_downloader.py b/tests/test_discord_downloader.py new file mode 100644 index 0000000..711f4ca --- /dev/null +++ b/tests/test_discord_downloader.py @@ -0,0 +1,135 @@ +"""The native Discord downloader lands files where gallery-dl did (#4414). + +A cutover that names one file differently re-downloads it and imports a +duplicate, so these pin the path, the name cleaning and the sidecar pairing +against what gallery-dl's config produces — and that the records it writes read +back through `parse_sidecar` as the same post the gallery-dl sidecars made. +""" + +from __future__ import annotations + +import json + +from backend.app.services.discord_client import DiscordClient +from backend.app.services.discord_downloader import ( + DiscordDownloader, + channel_dir, + gdl_clean, + media_stem, +) +from backend.app.utils.sidecar import find_sidecar, parse_sidecar + +_PNG = b"\x89PNG\r\n\x1a\n" + b"\x00" * 16 + b"\x00\x00\x00\x00IEND\xaeB`\x82" + + +class _Resp: + status_code = 200 + headers: dict = {} + + def raise_for_status(self): + pass + + def iter_content(self, chunk_size=None): + yield _PNG + + +class _Media: + def __init__(self): + self.calls = [] + + def get(self, url, stream=None, timeout=None, headers=None): + self.calls.append(url) + return _Resp() + + +def _message(**extra): + return { + "id": "1234", "type": 0, "timestamp": "2026-09-20T23:30:00.000000+00:00", + "content": "new set!", "channel_id": "22", + "author": {"id": "7", "username": "artist"}, + "attachments": [{"url": "https://cdn.discordapp.com/attachments/2/3/Red%3AAlt.PNG?ex=1"}], + "embeds": [], + "_meta": {"server": "Studio", "server_id": "11", "channel": "drops/nsfw", + "channel_id": "22", "is_thread": False}, + **extra, + } + + +def _downloader(tmp_path, session=None): + return DiscordDownloader(tmp_path, validate=False, session=session or _Media()) + + +def test_names_follow_gallery_dls_pattern_and_linux_cleaning(tmp_path): + msg = _message() + [media] = DiscordClient.extract_media(msg) + assert channel_dir(tmp_path, "art", msg) == tmp_path / "art" / "discord" / "drops_nsfw" + # `:` survives: gallery-dl on Linux only replaces `/`. + assert media_stem(msg, media) == "20260920_1234_01_Red:Alt" + assert gdl_clean("a\x07b/c") == "ab_c" + + +def test_a_channel_with_no_name_adds_no_directory(tmp_path): + msg = _message(_meta={"channel": " "}) + assert channel_dir(tmp_path, "art", msg) == tmp_path / "art" / "discord" + + +def test_a_file_gallery_dl_already_wrote_is_not_fetched_again(tmp_path): + msg = _message() + media = DiscordClient.extract_media(msg) + existing = tmp_path / "art" / "discord" / "drops_nsfw" / "20260920_1234_01_Red:Alt.png" + existing.parent.mkdir(parents=True) + existing.write_bytes(_PNG) + session = _Media() + [out] = _downloader(tmp_path, session).download_post(msg, media, "art") + assert out.status == "skipped_disk" and out.path == existing + assert session.calls == [] + + +def test_a_new_file_gets_a_sidecar_the_importer_pairs_to_its_message(tmp_path): + msg = _message() + [out] = _downloader(tmp_path).download_post(msg, DiscordClient.extract_media(msg), "art") + assert out.status == "downloaded" + assert out.path.name == "20260920_1234_01_Red:Alt.png" + sidecar = find_sidecar(out.path) + assert sidecar is not None + data = json.loads(sidecar.read_text()) + # No `id`/`post_id`: either would outrank message_id as the post id. + assert "id" not in data and "post_id" not in data + sd = parse_sidecar(data) + assert sd.external_post_id == "1234" + assert sd.source_url.startswith("https://cdn.discordapp.com/") + + +def test_a_seen_file_is_skipped_without_a_request(tmp_path): + msg = _message() + session = _Media() + [out] = _downloader(tmp_path, session).download_post( + msg, DiscordClient.extract_media(msg), "art", is_seen=lambda m: True, + ) + assert out.status == "skipped_seen" and session.calls == [] + + +def test_the_message_record_reads_back_as_the_gallery_dl_post(tmp_path): + rec = _downloader(tmp_path).write_post_record(_message(), "art") + assert rec.path.name == "20260920_1234_post.json" + assert rec.body_chars == len("new set!") + sd = parse_sidecar(json.loads(rec.path.read_text())) + assert sd.platform == "discord" + assert sd.external_post_id == "1234" + assert sd.post_url == "https://discord.com/channels/11/22/1234" + assert sd.description == "new set!" + assert sd.post_date.isoformat().startswith("2026-09-20T23:30") + + +def test_the_record_is_not_a_media_sidecar(tmp_path): + """It must not pair with any file of the message.""" + msg = _message() + dl = _downloader(tmp_path) + [out] = dl.download_post(msg, DiscordClient.extract_media(msg), "art") + rec = dl.write_post_record(msg, "art") + assert find_sidecar(out.path) != rec.path + + +def test_an_empty_re_read_never_blanks_a_stored_body(tmp_path): + rec = _downloader(tmp_path).write_post_record(_message(content=""), "art", revisit=True) + assert rec.path is None diff --git a/tests/test_download_backends.py b/tests/test_download_backends.py index 5ee3f6b..932a2f6 100644 --- a/tests/test_download_backends.py +++ b/tests/test_download_backends.py @@ -17,7 +17,7 @@ from backend.app.services.gallery_dl import ErrorType def test_native_platforms(): - for platform in ("patreon", "subscribestar"): + for platform in ("patreon", "subscribestar", "discord"): assert uses_native_ingester(platform) is True assert platform in NATIVE_INGESTER_PLATFORMS @@ -104,8 +104,20 @@ async def test_verifying_a_retired_platform_is_inconclusive_not_rejected(): def test_gallery_dl_platforms_are_not_native(): # The platforms still served by gallery-dl must NOT route to the native # ingester — guards an accidental over-broad migration. - for platform in ("hentaifoundry", "discord"): - assert uses_native_ingester(platform) is False + assert uses_native_ingester("hentaifoundry") is False + + +@pytest.mark.asyncio +async def test_discord_verify_without_a_token_is_a_rejection_not_a_request(): + """Discord authenticates by token (milestone 428); with none saved there is + nothing to send, and saying so beats an HTTP 401 from Discord.""" + ok, message = await verify_source_credential( + platform="discord", url="https://discord.com/channels/1/2", + artist_slug="someone", config_overrides=None, cookies_path=None, + auth_token=None, images_root=Path("/nonexistent"), + ) + assert ok is False + assert "token" in message.lower() def test_unknown_platform_is_not_native(): diff --git a/tests/test_patreon_ingester.py b/tests/test_patreon_ingester.py index 1e344d7..8d158bc 100644 --- a/tests/test_patreon_ingester.py +++ b/tests/test_patreon_ingester.py @@ -319,6 +319,64 @@ async def test_tick_early_out_after_threshold(source_id, sync_engine, tmp_path): assert client.consumed_posts == 2 +class _FeedsClient(_FakeClient): + """A source that is several feeds walked in turn (a Discord server's + channels), with the optional `skip_feed` seam (#4413). `feeds` is a list of + `pages` lists, one per feed.""" + + def __init__(self, feeds): + super().__init__([page for pages in feeds for page in pages]) + self._feeds = feeds + self._skip = False + self.skips = 0 + + def skip_feed(self): + self._skip = True + self.skips += 1 + + def iter_posts(self, campaign_id, cursor=None): + for pages in self._feeds: + self._skip = False + self._pages = pages + for item in super().iter_posts(campaign_id, cursor): + yield item + if self._skip: + break + + +def _seed_seen_media(sync_engine, source_id, media): + factory = sessionmaker(sync_engine, expire_on_commit=False) + with factory() as s: + for m in media: + s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m), post_id=m.post_id)) + s.commit() + + +@pytest.mark.asyncio +async def test_a_quiet_feed_ends_itself_not_the_walk(source_id, sync_engine, tmp_path): + """#4413: a Discord server's first channel is all seen; the tick must still + reach the second channel's new file, instead of stopping at the first.""" + quiet = [_media(f"a{i}", 1) for i in range(1, 5)] + _seed_seen_media(sync_engine, source_id, quiet) + fresh = _media("b1", 1) + client = _FeedsClient([ + [(None, [(m.post_id, [m]) for m in quiet])], + [(None, [("b1", [fresh])])], + ]) + downloader = _FakeDownloader(tmp_path) + result = _ingester(sync_engine, tmp_path, client, downloader).run( + source_id=source_id, campaign_id="c1", artist_slug="ingest", + url="https://patreon.com/ingest", mode="tick", seen_threshold=2, + ) + assert result.success is True + assert client.skips == 1 + # The quiet feed stopped after its 2nd seen post; the next feed was walked. + assert client.consumed_posts == 3 + assert downloader.download_calls == 1 + assert "1 feed(s) caught up" in result.stdout + assert "reached end" not in result.stdout + + # --- backfill ------------------------------------------------------------- diff --git a/tests/test_platform_lock.py b/tests/test_platform_lock.py index b677e1e..c4def8b 100644 --- a/tests/test_platform_lock.py +++ b/tests/test_platform_lock.py @@ -10,7 +10,11 @@ pytestmark = pytest.mark.integration def test_non_serialized_platform_has_no_lock(): # gallery-dl platforms aren't capped — they get no lock at all. assert platform_lock("hentaifoundry", ttl_seconds=60) is None - assert platform_lock("discord", ttl_seconds=60) is None + + +def test_discord_is_serialized(): + # Native since milestone 428, and every source shares one user token. + assert platform_lock("discord", ttl_seconds=60) is not None def test_subscribestar_is_serialized():