bvandeusenandClaude Opus 5.5
0928d2f914
CI and images / lint (push) Successful in 3s
CI and images / extension-version (push) Successful in 3s
CI and images / extension-test (push) Successful in 19s
CI and images / frontend-build (push) Successful in 21s
CI and images / backend-lint-and-test (push) Successful in 32s
CI and images / integration (push) Successful in 2m23s
CI and images / build-agent (push) Successful in 6s
CI and images / sign-extension (push) Successful in 6m2s
CI and images / build-web (push) Successful in 1m40s
CI and images / smoke-web (push) Successful in 55s
CI and images / promote (push) Successful in 1s
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
753 lines
32 KiB
Python
753 lines
32 KiB
Python
"""Native Discord read client — the Discord counterpart to subscribestar_client.
|
||
|
||
Mirrors gallery-dl 1.32.13's `extractor/discord.py` (rule 130: gallery-dl is the
|
||
known-working base), adapted to the native core's client contract
|
||
(`ingest_core` module docstring): `iter_posts` / `extract_media`, plus the
|
||
post-first `post_record_key` and the `post_meta` date the revisit window reads.
|
||
|
||
What is mirrored exactly, because drift in any of it changes what we fetch or
|
||
where it lands on disk:
|
||
|
||
- API v10, `Authorization: <user token>` (a USER token, not a bot token).
|
||
- gallery-dl's request profile: its date-derived Firefox User-Agent,
|
||
`Accept: */*`, `Accept-Language`, `Referer: https://discord.com/`.
|
||
- `GET /channels/{id}/messages?limit=100&before=<last id>`, newest first,
|
||
stopping on a short page. Message types {0, 19, 21} only.
|
||
- The walk: a text/news channel's own messages then its threads, a forum's
|
||
threads, a category's children, a server's text/news/forum channels.
|
||
- Files: attachments, then embeds of type image/gifv/video (FC configures
|
||
`embeds: all`, which for files is the same three plus rich/link embeds that
|
||
carry an image), then forwarded `message_snapshots`, numbered from 1 across
|
||
the lot — the `num` in `{date}_{message_id}_{num}_{filename}`.
|
||
- Text: `content`, rich-embed author/title/description/fields/footer, poll.
|
||
|
||
Two deliberate departures, both about the walk order, neither about content:
|
||
|
||
- Threads are walked newest-CREATED first (by id), not by last-message time.
|
||
A backfill resumes from a checkpointed channel; last-message order shifts
|
||
between chunks whenever someone posts, which can move an unwalked thread
|
||
above the resume point and skip it. Creation order only ever grows at the
|
||
front, where the next tick finds it.
|
||
- A 403 on a thread or a nested channel skips that feed instead of failing
|
||
the walk. gallery-dl skips only nested channels; one private thread the
|
||
token cannot read would otherwise stop every channel after it.
|
||
|
||
And one about content: a message is taken only if it has an image or video
|
||
ATTACHMENT (`has_visual_attachment`). Discord is where creators chat as well
|
||
as post, and the operator wants the art, not the conversation (2026-09-28): a
|
||
text line, a lone archive or PSD, a pasted link's preview or a Tenor GIF is
|
||
chat. A message that passes keeps every file gallery-dl would take, numbered
|
||
as gallery-dl numbers them, so the on-disk names still match.
|
||
|
||
FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import logging
|
||
import re
|
||
import time
|
||
from collections.abc import Iterator
|
||
from dataclasses import dataclass
|
||
from datetime import date
|
||
from urllib.parse import unquote
|
||
|
||
import requests
|
||
|
||
from .native_ingest_common import (
|
||
NativeAuthError,
|
||
NativeDriftError,
|
||
NativeIngestError,
|
||
retry_after_seconds,
|
||
)
|
||
|
||
log = logging.getLogger(__name__)
|
||
|
||
API_ROOT = "https://discord.com/api/v10"
|
||
_ROOT = "https://discord.com"
|
||
|
||
_TIMEOUT_SECONDS = 60.0
|
||
_MESSAGES_BATCH = 100
|
||
_THREADS_BATCH = 25
|
||
# gallery-dl retries a 429 up to its default 4 retries, waiting
|
||
# `request_interval_429` (60s) between them. Discord's Retry-After is exact, so
|
||
# it is honoured when present; 60s is the fallback and the cap.
|
||
_MAX_429_RETRIES = 4
|
||
_429_WAIT_SECONDS = 60.0
|
||
# How far back the poster picker looks: two pages. Enough to see who posts in
|
||
# a channel, few enough to answer while the operator waits (#4491).
|
||
_POSTER_SCAN = 200
|
||
|
||
# https://discord.com/developers/docs/resources/message#message-object-message-types
|
||
# DEFAULT, REPLY, CHAT_INPUT_COMMAND — the ones that carry user content.
|
||
MESSAGE_TYPES = frozenset({0, 19, 21})
|
||
# https://discord.com/developers/docs/resources/channel#channel-object-channel-types
|
||
_TEXT = frozenset({0, 5}) # text, announcement: messages + threads
|
||
_DIRECT = frozenset({1, 3, 10, 11, 12}) # DMs and threads: messages only
|
||
_FORUM = frozenset({15, 16}) # forum, media: threads only
|
||
_CATEGORY = 4
|
||
_SERVER_WALK = _TEXT | _FORUM
|
||
_EMBED_TYPES = frozenset({"image", "gifv", "video"})
|
||
# What makes a message art rather than chat: an attached file FC imports as an
|
||
# image or video. Mirrors importer.ALL_EXTS (not imported: that module is the
|
||
# import pipeline, and this one is a network client).
|
||
_VISUAL_EXTS = frozenset({
|
||
"png", "jpg", "jpeg", "gif", "webp", "bmp", "tif", "tiff",
|
||
"mp4", "mov", "avi", "mkv", "webm", "m4v", "wmv", "flv",
|
||
})
|
||
|
||
_URL_RE = re.compile(
|
||
r"^(?:https?://)?(?:www\.|ptb\.|canary\.)?discord(?:app)?\.com/channels/"
|
||
r"(?P<server>@me|\d+)(?:/(?:\d+/threads/)?(?P<channel>\d+))?(?P<rest>/.*)?/?$"
|
||
)
|
||
|
||
|
||
class DiscordAPIError(NativeIngestError):
|
||
"""Base for native Discord client failures."""
|
||
|
||
|
||
class DiscordAuthError(DiscordAPIError, NativeAuthError):
|
||
"""401 (the token is invalid or expired) or a 403 on the channel the
|
||
source names. The fix is a new token, not a new client."""
|
||
|
||
|
||
class DiscordDriftError(DiscordAPIError, NativeDriftError):
|
||
"""A response did not have the shape the walk depends on."""
|
||
|
||
|
||
def firefox_user_agent(today: date | None = None) -> str:
|
||
"""gallery-dl's default User-Agent: a Firefox whose version advances every
|
||
four weeks (`util._ff_ver`, "147 on 2026-01-13"). Computed the same way so
|
||
the profile keeps matching the gallery-dl this replaced."""
|
||
ver = ((today or date.today()).toordinal() - 735_513) // 28
|
||
return (
|
||
f"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:{ver}.0) "
|
||
f"Gecko/20100101 Firefox/{ver}.0"
|
||
)
|
||
|
||
|
||
def nameext_from_url(url: str) -> tuple[str, str]:
|
||
"""gallery-dl's `text.nameext_from_url`: the URL's last path segment,
|
||
unquoted, split at the last dot when the extension is at most 16 chars
|
||
(lowercased); otherwise the whole name and no extension."""
|
||
filename = unquote(url.partition("?")[0].rpartition("/")[2])
|
||
name, _, ext = filename.rpartition(".")
|
||
if name and len(ext) <= 16:
|
||
return name, ext.lower()
|
||
return filename, ""
|
||
|
||
|
||
def parse_source_url(url: str) -> tuple[str | None, str | None]:
|
||
"""`(server_id, channel_id)` from a Discord channel/server URL. `server_id`
|
||
is None for a DM (`@me`); `channel_id` is None for a whole server. Raises
|
||
DiscordAPIError for anything else, including a link to a single message —
|
||
a message is not something a source can subscribe to."""
|
||
m = _URL_RE.match((url or "").strip())
|
||
if not m or (m.group("rest") or "").strip("/"):
|
||
raise DiscordAPIError(
|
||
f"Not a Discord channel or server link: {url!r} "
|
||
"(expected https://discord.com/channels/<server>[/<channel>])"
|
||
)
|
||
server = m.group("server")
|
||
channel = m.group("channel")
|
||
if server == "@me":
|
||
if not channel:
|
||
raise DiscordAPIError(f"A DM link needs a channel id: {url!r}")
|
||
return None, channel
|
||
return server, channel
|
||
|
||
|
||
def message_text(message: dict) -> str:
|
||
"""gallery-dl's `extract_message_text`: the body plus the text of rich
|
||
embeds and polls, newline-joined, empties dropped."""
|
||
parts = [message.get("content") or ""]
|
||
for embed in message.get("embeds") or []:
|
||
if embed.get("type") != "rich":
|
||
continue
|
||
parts.append((embed.get("author") or {}).get("name") or "")
|
||
parts.append(embed.get("title") or "")
|
||
parts.append(embed.get("description") or "")
|
||
for fld in embed.get("fields") or []:
|
||
parts.append(fld.get("name") or "")
|
||
parts.append(fld.get("value") or "")
|
||
parts.append((embed.get("footer") or {}).get("text") or "")
|
||
poll = message.get("poll")
|
||
if poll:
|
||
parts.append(((poll.get("question") or {}).get("text")) or "")
|
||
for answer in poll.get("answers") or []:
|
||
parts.append(((answer.get("poll_media") or {}).get("text")) or "")
|
||
return "\n".join(p for p in parts if p)
|
||
|
||
|
||
def _message_and_snapshots(message: dict) -> list[dict]:
|
||
"""The message itself, then each forwarded message it carries."""
|
||
return [message] + [
|
||
(s or {}).get("message") or {}
|
||
for s in message.get("message_snapshots") or []
|
||
if ((s or {}).get("message") or {}).get("type", 0) in MESSAGE_TYPES
|
||
]
|
||
|
||
|
||
def _is_visual(attachment: dict) -> bool:
|
||
ctype = (attachment.get("content_type") or "").lower()
|
||
if ctype.startswith(("image/", "video/")):
|
||
return True
|
||
name = attachment.get("filename") or attachment.get("url") or ""
|
||
return nameext_from_url(name)[1] in _VISUAL_EXTS
|
||
|
||
|
||
def has_visual_attachment(message: dict) -> bool:
|
||
"""Does this message, or a message it forwards, have an image or video
|
||
attached? The one test for "art, not chat" — see the module docstring."""
|
||
return any(
|
||
att.get("url") and _is_visual(att)
|
||
for snap in _message_and_snapshots(message)
|
||
for att in snap.get("attachments") or []
|
||
)
|
||
|
||
|
||
_SQUASH = re.compile(r"[\W_]+", re.UNICODE)
|
||
# Words a server name wraps around its creator's: "Todding's Server",
|
||
# "The Official Todding Discord".
|
||
_SERVER_WORDS = frozenset({"server", "discord", "the", "official", "community", "s"})
|
||
|
||
|
||
def _squash(text: str | None) -> str:
|
||
return _SQUASH.sub("", (text or "").lower())
|
||
|
||
|
||
def _server_tokens(server_name: str | None) -> list[str]:
|
||
name = (server_name or "").lower().replace("'s", " ").replace("\u2019s", " ")
|
||
words = [w for w in _SQUASH.split(name) if w and w not in _SERVER_WORDS]
|
||
tokens = [w for w in words if len(w) >= 3]
|
||
joined = "".join(words)
|
||
if len(joined) >= 3 and joined not in tokens:
|
||
tokens.append(joined)
|
||
return tokens
|
||
|
||
|
||
def rank_posters(server_name: str | None, owner_id, posters: list[dict]) -> list[dict]:
|
||
"""Mark who is probably the creator a server is named for, and order the
|
||
list for picking: suggested first, then by images posted, then messages.
|
||
|
||
Two signals, each a reason the picker shows beside the name:
|
||
- they own the server;
|
||
- their username or display name matches the server's name, once
|
||
"'s Server", "Official", "Discord" and punctuation are taken off
|
||
("Todding's Server" and "todding").
|
||
Neither signal alone is proof, which is why this suggests and never
|
||
decides: the operator ticks the list. Bots are never suggested."""
|
||
tokens = _server_tokens(server_name)
|
||
owner = str(owner_id) if owner_id else None
|
||
ranked = []
|
||
for p in posters:
|
||
reasons = []
|
||
if owner and p.get("id") == owner:
|
||
reasons.append("owns the server")
|
||
names = [n for n in (_squash(p.get("username")), _squash(p.get("global_name"))) if len(n) >= 3]
|
||
if any(t in n or n in t for t in tokens for n in names):
|
||
reasons.append("name matches the server")
|
||
ranked.append({**p, "reasons": reasons, "suggested": False})
|
||
best = max((len(r["reasons"]) for r in ranked if not r.get("bot")), default=0)
|
||
for r in ranked:
|
||
r["suggested"] = best > 0 and not r.get("bot") and len(r["reasons"]) == best
|
||
ranked.sort(key=lambda r: (not r["suggested"], -r.get("images", 0), -r.get("messages", 0)))
|
||
return ranked
|
||
|
||
|
||
@dataclass
|
||
class MediaItem:
|
||
"""One file of a Discord message. `filename`/`extension` are gallery-dl's
|
||
split of the URL; `num` is its 1-based position across the message's files,
|
||
which is what names it on disk.
|
||
|
||
`media_id` is what the seen-ledger keys on, and it is deliberately NOT
|
||
`num`: an edit that removes a file renumbers the ones after it, and a
|
||
positional key would then call a different file seen. It is the
|
||
attachment's id, or for an embed (which has none) a hash of its URL path —
|
||
the query string is a signature that changes on every fetch. `filehash` is
|
||
always None; nothing in a signed CDN URL is a content hash."""
|
||
|
||
url: str
|
||
filename: str
|
||
extension: str
|
||
kind: str
|
||
post_id: str
|
||
num: int
|
||
media_id: str
|
||
filehash: str | None = None
|
||
|
||
|
||
class DiscordClient:
|
||
"""Synchronous Discord API v10 read client for one user token."""
|
||
|
||
def __init__(
|
||
self,
|
||
token: str | None,
|
||
*,
|
||
request_sleep: float = 0.0,
|
||
max_retries: int = _MAX_429_RETRIES,
|
||
session: requests.Session | None = None,
|
||
):
|
||
self._session = session or requests.Session()
|
||
self._session.headers.update({
|
||
"User-Agent": firefox_user_agent(),
|
||
"Accept": "*/*",
|
||
"Accept-Language": "en-US,en;q=0.5",
|
||
"Referer": _ROOT + "/",
|
||
})
|
||
if token:
|
||
self._session.headers["Authorization"] = token
|
||
self._token = token
|
||
self._request_sleep = request_sleep or 0.0
|
||
self._max_retries = max_retries
|
||
self._server: dict = {}
|
||
self._channels: dict[str, dict] = {}
|
||
self._skip_feed = False
|
||
# Whose messages the walk takes (`only_from`); empty takes everyone's.
|
||
self._authors: frozenset[str] = frozenset()
|
||
|
||
# -- request -----------------------------------------------------------
|
||
|
||
def _get(self, endpoint: str, params: dict | None = None):
|
||
if not self._token:
|
||
raise DiscordAuthError("No Discord token is configured for this source")
|
||
if self._request_sleep > 0:
|
||
time.sleep(self._request_sleep)
|
||
url = API_ROOT + endpoint
|
||
attempt = 0
|
||
while True:
|
||
try:
|
||
resp = self._session.get(url, params=params, timeout=_TIMEOUT_SECONDS)
|
||
except requests.RequestException as exc:
|
||
raise DiscordAPIError(f"Discord request failed ({endpoint}): {exc}") from exc
|
||
if resp.status_code == 429 and attempt < self._max_retries:
|
||
attempt += 1
|
||
delay = retry_after_seconds(
|
||
resp, attempt, base=_429_WAIT_SECONDS, cap=_429_WAIT_SECONDS,
|
||
)
|
||
log.warning(
|
||
"Discord 429 (%s) — waiting %.1fs (retry %d/%d)",
|
||
endpoint, delay, attempt, self._max_retries,
|
||
)
|
||
time.sleep(delay)
|
||
continue
|
||
break
|
||
if resp.status_code == 401:
|
||
raise DiscordAuthError(
|
||
"Discord rejected the token (HTTP 401) — it is invalid or has "
|
||
"expired; copy a fresh one from the browser",
|
||
status_code=401,
|
||
)
|
||
if resp.status_code != 200:
|
||
raise DiscordAPIError(
|
||
f"Discord returned HTTP {resp.status_code} ({endpoint})",
|
||
status_code=resp.status_code,
|
||
retry_after=_retry_after(resp),
|
||
)
|
||
try:
|
||
return resp.json()
|
||
except ValueError as exc:
|
||
raise DiscordDriftError(
|
||
f"Discord returned non-JSON for {endpoint} ({len(resp.content)} bytes)"
|
||
) from exc
|
||
|
||
# -- metadata (gallery-dl parse_server / parse_channel) -----------------
|
||
|
||
def _load_server(self, server_id: str) -> None:
|
||
server = self._get(f"/guilds/{server_id}")
|
||
if not isinstance(server, dict) or "id" not in server:
|
||
raise DiscordDriftError(f"Discord server {server_id} came back without an id")
|
||
self._server = {
|
||
"server": server.get("name") or "",
|
||
"server_id": str(server["id"]),
|
||
"owner_id": server.get("owner_id"),
|
||
}
|
||
channels = self._get(f"/guilds/{server_id}/channels")
|
||
if not isinstance(channels, list):
|
||
raise DiscordDriftError(f"Discord server {server_id} channel list is not a list")
|
||
# Categories first, so every child can name its parent.
|
||
for channel in sorted(channels, key=lambda ch: ch.get("type") != _CATEGORY):
|
||
self._parse_channel(channel)
|
||
|
||
def _parse_channel(self, channel: dict) -> dict:
|
||
parent_id = channel.get("parent_id")
|
||
meta = {
|
||
"channel": channel.get("name") or "",
|
||
"channel_id": str(channel.get("id")),
|
||
"channel_type": channel.get("type"),
|
||
"channel_topic": channel.get("topic") or "",
|
||
"parent_id": parent_id,
|
||
"is_thread": "thread_metadata" in channel,
|
||
}
|
||
parent = self._channels.get(parent_id) if parent_id else None
|
||
if parent:
|
||
meta["parent"] = parent["channel"]
|
||
meta["parent_type"] = parent["channel_type"]
|
||
if meta["channel_type"] in {1, 3}:
|
||
recipients = channel.get("recipients") or []
|
||
meta["channel"] = "DMs"
|
||
meta["recipients"] = [u.get("username") for u in recipients]
|
||
meta["recipients_id"] = [u.get("id") for u in recipients]
|
||
self._channels[meta["channel_id"]] = meta
|
||
return meta
|
||
|
||
def _channel_meta(self, channel_id: str) -> dict:
|
||
if channel_id not in self._channels:
|
||
self._parse_channel(self._get(f"/channels/{channel_id}"))
|
||
return self._channels[channel_id]
|
||
|
||
def _threads(self, channel_id: str) -> list[dict]:
|
||
"""Every thread of a channel or forum, newest-created first (see the
|
||
module docstring for why not last-message order)."""
|
||
threads: list[dict] = []
|
||
offset = 0
|
||
while True:
|
||
data = self._get(f"/channels/{channel_id}/threads/search", {
|
||
"sort_by": "last_message_time",
|
||
"sort_order": "desc",
|
||
"limit": _THREADS_BATCH,
|
||
"offset": offset,
|
||
})
|
||
batch = (data.get("threads") or []) if isinstance(data, dict) else []
|
||
threads.extend(batch)
|
||
if len(batch) < _THREADS_BATCH:
|
||
break
|
||
offset += len(batch)
|
||
threads.sort(key=lambda t: int(t.get("id") or 0), reverse=True)
|
||
return threads
|
||
|
||
# -- the walk ------------------------------------------------------------
|
||
|
||
def _feeds(self, channel_id: str, *, safe: bool) -> Iterator[tuple[str, bool]]:
|
||
"""`(channel_id, safe)` for every message feed under `channel_id`, in
|
||
gallery-dl's order. `safe` feeds are skipped on a 403."""
|
||
try:
|
||
ctype = self._channel_meta(channel_id)["channel_type"]
|
||
except DiscordAPIError as exc:
|
||
if exc.status_code != 403:
|
||
raise
|
||
if not safe:
|
||
raise DiscordAuthError(
|
||
f"The Discord token cannot see channel {channel_id} (HTTP 403)",
|
||
status_code=403,
|
||
) from exc
|
||
log.info("Discord: no access to channel %s — skipped", channel_id)
|
||
return
|
||
if ctype in _TEXT or ctype in _DIRECT:
|
||
yield channel_id, safe
|
||
if ctype in _TEXT or ctype in _FORUM:
|
||
try:
|
||
threads = self._threads(channel_id)
|
||
except DiscordAPIError as exc:
|
||
if exc.status_code != 403:
|
||
raise
|
||
log.info("Discord: cannot list threads of %s — skipped", channel_id)
|
||
threads = []
|
||
for thread in threads:
|
||
yield self._parse_channel(thread)["channel_id"], True
|
||
elif ctype == _CATEGORY:
|
||
for child in list(self._channels.values()):
|
||
if child.get("parent_id") == channel_id:
|
||
yield from self._feeds(child["channel_id"], safe=True)
|
||
elif ctype not in _DIRECT and not safe:
|
||
raise DiscordAPIError(
|
||
f"Discord channel {channel_id} is of type {ctype}, which has no messages"
|
||
)
|
||
|
||
def _source_feeds(self, url: str) -> Iterator[tuple[str, bool]]:
|
||
server_id, channel_id = parse_source_url(url)
|
||
self._server, self._channels = {}, {}
|
||
if server_id is not None:
|
||
self._load_server(server_id)
|
||
if channel_id is not None:
|
||
yield from self._feeds(channel_id, safe=False)
|
||
return
|
||
for meta in list(self._channels.values()):
|
||
if meta["channel_type"] in _SERVER_WALK:
|
||
yield from self._feeds(meta["channel_id"], safe=True)
|
||
|
||
def only_from(self, authors) -> None:
|
||
"""Take only messages posted by these people: user ids, usernames or
|
||
display names, any case. A creator's server is full of other members
|
||
posting their own pictures; the source is subscribed to the creator.
|
||
Empty or None takes everyone's, as before."""
|
||
self._authors = frozenset(
|
||
str(a).strip().lower() for a in authors or () if str(a).strip()
|
||
)
|
||
|
||
def _author_wanted(self, message: dict) -> bool:
|
||
if not self._authors:
|
||
return True
|
||
author = message.get("author") or {}
|
||
names = (author.get("id"), author.get("username"), author.get("global_name"))
|
||
return any(str(n).lower() in self._authors for n in names if n)
|
||
|
||
def source_label(self, url: str) -> str | None:
|
||
"""What the source is called in Discord — `Server · #channel`, a
|
||
thread as `#parent › thread`, a whole server by its name — from the
|
||
metadata the walk loaded. None when the walk never got that far."""
|
||
try:
|
||
_, channel_id = parse_source_url(url)
|
||
except DiscordAPIError:
|
||
return None
|
||
server = self._server.get("server") or None
|
||
if channel_id is None:
|
||
return server
|
||
meta = self._channels.get(channel_id) or {}
|
||
name = meta.get("channel")
|
||
if not name:
|
||
return None
|
||
if name != "DMs":
|
||
name = f"#{name}"
|
||
if meta.get("is_thread") and meta.get("parent"):
|
||
name = f"#{meta['parent']} › {meta['channel']}"
|
||
return f"{server} · {name}" if server else name
|
||
|
||
def skip_feed(self) -> None:
|
||
"""Optional core seam (#4413): end the current channel and go on to the
|
||
next one. A tick's early-out means THIS channel has nothing new, not
|
||
that the server has nothing new."""
|
||
self._skip_feed = True
|
||
|
||
def iter_posts(
|
||
self, campaign_id: str, cursor: str | None = None
|
||
) -> Iterator[tuple[dict, dict, str | None]]:
|
||
"""Yield `(message, channel_meta, page_cursor)` for every content
|
||
message the source reaches, channel by channel, each newest first.
|
||
|
||
`campaign_id` is the source URL. The cursor is `<channel_id>:<before>`
|
||
— the channel and the `before` id that fetched the page (empty for a
|
||
channel's first page) — so a backfill resumes inside the right channel
|
||
and re-fetches the page it was cut in. A cursor naming a channel the
|
||
walk no longer reaches (a deleted thread) restarts from the top rather
|
||
than walking nothing.
|
||
"""
|
||
resume_channel, _, resume_before = (cursor or "").partition(":")
|
||
resuming = bool(resume_channel)
|
||
feeds = list(self._source_feeds(campaign_id)) if resuming else None
|
||
if feeds is not None and resume_channel not in {cid for cid, _ in feeds}:
|
||
log.warning(
|
||
"Discord: resume channel %s is no longer in %s — restarting",
|
||
resume_channel, campaign_id,
|
||
)
|
||
resuming = False
|
||
for channel_id, safe in feeds if feeds is not None else self._source_feeds(campaign_id):
|
||
before = None
|
||
if resuming:
|
||
if channel_id != resume_channel:
|
||
continue
|
||
resuming = False
|
||
before = resume_before or None
|
||
yield from self._iter_channel(channel_id, before, safe=safe)
|
||
|
||
def _iter_channel(
|
||
self, channel_id: str, before: str | None, *, safe: bool
|
||
) -> Iterator[tuple[dict, dict, str | None]]:
|
||
self._skip_feed = False
|
||
meta = {**self._server, **self._channels.get(channel_id, {})}
|
||
while True:
|
||
page_cursor = f"{channel_id}:{before or ''}"
|
||
try:
|
||
messages = self._get(
|
||
f"/channels/{channel_id}/messages",
|
||
{"limit": _MESSAGES_BATCH, "before": before},
|
||
)
|
||
except DiscordAPIError as exc:
|
||
if exc.status_code != 403:
|
||
raise
|
||
if not safe:
|
||
raise DiscordAuthError(
|
||
f"The Discord token cannot read channel {channel_id} (HTTP 403)",
|
||
status_code=403,
|
||
) from exc
|
||
log.info("Discord: no access to messages of %s — skipped", channel_id)
|
||
return
|
||
if not isinstance(messages, list):
|
||
raise DiscordDriftError(
|
||
f"Discord messages of {channel_id} came back as "
|
||
f"{type(messages).__name__}, not a list"
|
||
)
|
||
for message in messages:
|
||
if message.get("type") not in MESSAGE_TYPES:
|
||
continue
|
||
if not self._author_wanted(message):
|
||
continue
|
||
message["_meta"] = meta
|
||
yield message, meta, page_cursor
|
||
if self._skip_feed:
|
||
return
|
||
if len(messages) < _MESSAGES_BATCH:
|
||
return
|
||
before = str(messages[-1]["id"])
|
||
|
||
# -- per-message -------------------------------------------------------
|
||
|
||
@staticmethod
|
||
def extract_media(post: dict, included: dict | None = None) -> list[MediaItem]:
|
||
"""gallery-dl's file list for one message: attachments, then the first
|
||
of video/image/thumbnail `proxy_url` of each file-bearing embed, then
|
||
the same for every forwarded snapshot; numbered from 1 across them.
|
||
|
||
Empty for a message with no image or video attached: that is chat,
|
||
whatever else it carries (`has_visual_attachment`)."""
|
||
if not has_visual_attachment(post):
|
||
return []
|
||
mid = str(post.get("id") or "")
|
||
snapshots = _message_and_snapshots(post)
|
||
found: list[tuple[str, str, str | None]] = []
|
||
for snap in snapshots:
|
||
for att in snap.get("attachments") or []:
|
||
if att.get("url"):
|
||
aid = att.get("id")
|
||
found.append((att["url"], "attachment", str(aid) if aid else None))
|
||
for embed in snap.get("embeds") or []:
|
||
if embed.get("type") not in _EMBED_TYPES:
|
||
continue
|
||
for fld in ("video", "image", "thumbnail"):
|
||
url = (embed.get(fld) or {}).get("proxy_url")
|
||
if url:
|
||
found.append((url, "embed", None))
|
||
break
|
||
items = []
|
||
for num, (url, kind, fid) in enumerate(found, start=1):
|
||
name, ext = nameext_from_url(url)
|
||
if fid is None:
|
||
path = url.partition("?")[0].encode()
|
||
fid = "u" + hashlib.sha1(path, usedforsecurity=False).hexdigest()[:32]
|
||
items.append(MediaItem(
|
||
url=url, filename=name, extension=ext, kind=kind, post_id=mid,
|
||
num=num, media_id=fid,
|
||
))
|
||
return items
|
||
|
||
@staticmethod
|
||
def post_meta(post: dict) -> dict:
|
||
"""No title (Discord has none); `date` is the message timestamp, ISO
|
||
with an offset — what the core's revisit window reads."""
|
||
return {"title": None, "date": post.get("timestamp")}
|
||
|
||
@classmethod
|
||
def post_record_key(cls, post: dict) -> tuple[str, str] | None:
|
||
"""`(message:<id>, <id>)` — gates the message record through the seen
|
||
ledger, like `post:<id>` on the other platforms.
|
||
|
||
None for a message `extract_media` takes nothing from, i.e. one with
|
||
no image or video attached. The drop grouping (discord_grouping) is
|
||
built on that: a channel's chatter recorded as posts would bury the
|
||
drops it exists to surface."""
|
||
mid = post.get("id")
|
||
mid = str(mid) if mid is not None else ""
|
||
if not mid or not cls.extract_media(post):
|
||
return None
|
||
return (f"message:{mid}", mid)
|
||
|
||
# -- verify ------------------------------------------------------------
|
||
|
||
def describe(self, server_id: str | None, channel_id: str | None) -> dict:
|
||
"""The display names behind a server/channel pair, for the browser
|
||
extension's Add panel. Best-effort per name: one that can't be read
|
||
comes back None, and the other is still returned."""
|
||
out: dict = {"server": None, "channel": None, "parent": None}
|
||
if server_id:
|
||
try:
|
||
out["server"] = (self._get(f"/guilds/{server_id}") or {}).get("name") or None
|
||
except DiscordAPIError:
|
||
pass
|
||
if channel_id:
|
||
try:
|
||
meta = self._parse_channel(self._get(f"/channels/{channel_id}"))
|
||
out["channel"] = meta.get("channel") or None
|
||
except (DiscordAPIError, AttributeError):
|
||
pass
|
||
return out
|
||
|
||
def recent_posters(
|
||
self, server_id: str | None, channel_id: str, *, max_messages: int = _POSTER_SCAN,
|
||
) -> dict:
|
||
"""Who has posted lately in a channel, for picking a source's poster
|
||
list (#4491): the newest `max_messages` messages, tallied per author.
|
||
Deliberately shallow — it answers "who posts here", not "who ever did".
|
||
Every poster comes back with their stable id, which is what the list
|
||
stores: a username can change on a whim, an id can't."""
|
||
server: dict = {}
|
||
if server_id:
|
||
try:
|
||
server = self._get(f"/guilds/{server_id}") or {}
|
||
except DiscordAPIError:
|
||
server = {}
|
||
tally: dict[str, dict] = {}
|
||
before = None
|
||
seen = 0
|
||
while seen < max_messages:
|
||
page = self._get(
|
||
f"/channels/{channel_id}/messages",
|
||
{"limit": min(_MESSAGES_BATCH, max_messages - seen), "before": before},
|
||
)
|
||
if not isinstance(page, list) or not page:
|
||
break
|
||
for message in page:
|
||
seen += 1
|
||
if message.get("type") not in MESSAGE_TYPES:
|
||
continue
|
||
author = message.get("author") or {}
|
||
aid = str(author.get("id") or "")
|
||
if not aid:
|
||
continue
|
||
row = tally.setdefault(aid, {
|
||
"id": aid,
|
||
"username": author.get("username"),
|
||
"global_name": author.get("global_name"),
|
||
"bot": bool(author.get("bot")),
|
||
"messages": 0,
|
||
"images": 0,
|
||
})
|
||
row["messages"] += 1
|
||
if has_visual_attachment(message):
|
||
row["images"] += 1
|
||
if len(page) < _MESSAGES_BATCH:
|
||
break
|
||
before = str(page[-1]["id"])
|
||
return {
|
||
"server": server.get("name") or None,
|
||
"owner_id": str(server["owner_id"]) if server.get("owner_id") else None,
|
||
"scanned": seen,
|
||
"posters": rank_posters(
|
||
server.get("name"), server.get("owner_id"), list(tally.values()),
|
||
),
|
||
}
|
||
|
||
def verify_auth(self, url: str) -> tuple[bool | None, str]:
|
||
"""Is the token valid, and can it see what the source names?"""
|
||
try:
|
||
server_id, channel_id = parse_source_url(url)
|
||
except DiscordAPIError as exc:
|
||
return None, str(exc)
|
||
try:
|
||
me = self._get("/users/@me")
|
||
if channel_id is not None:
|
||
self._get(f"/channels/{channel_id}")
|
||
elif server_id is not None:
|
||
self._get(f"/guilds/{server_id}")
|
||
except DiscordAuthError as exc:
|
||
return False, f"Discord rejected the token — {exc}"
|
||
except DiscordAPIError as exc:
|
||
if exc.status_code in (403, 404):
|
||
return False, (
|
||
"The token is valid, but its account cannot see "
|
||
f"{'this channel' if channel_id else 'this server'} "
|
||
f"(HTTP {exc.status_code})"
|
||
)
|
||
return None, f"Couldn't verify (network/HTTP issue): {exc}"
|
||
who = (me or {}).get("username") if isinstance(me, dict) else None
|
||
return True, f"Token valid{f' ({who})' if who else ''} — the source is readable."
|
||
|
||
|
||
def _retry_after(resp: requests.Response) -> float | None:
|
||
hdr = resp.headers.get("Retry-After")
|
||
try:
|
||
return float(hdr) if hdr else None
|
||
except (TypeError, ValueError):
|
||
return None
|