fix: a Discord source takes only messages with an image attached, from the posters it names, and knows its channel's name (#4481)
CI and images / lint (push) Successful in 4s
CI and images / extension-version (push) Successful in 3s
CI and images / frontend-build (push) Successful in 21s
CI and images / extension-test (push) Successful in 20s
CI and images / backend-lint-and-test (push) Successful in 34s
CI and images / integration (push) Successful in 2m24s
CI and images / sign-extension (push) Successful in 3s
CI and images / build-agent (push) Successful in 7s
CI and images / build-web (push) Successful in 1m42s
CI and images / smoke-web (push) Successful in 59s
CI and images / promote (push) Successful in 2s

- A message with no image or video attachment is chat: extract_media returns
  nothing for it, so nothing downloads and no post record is written. A
  message that passes keeps every file, numbered as gallery-dl numbers them.
- `discord_authors` in a source's config limits the walk to those posters
  (id, username or display name); the edit dialog has a field for it and no
  longer drops config keys it has no field for.
- source.display_name (migration 0115), refreshed by every Discord walk, shows
  on Subscriptions in place of the two-id URL; a Discord post card names its
  channel from the record's `channel`.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-09-28 07:03:47 -04:00
co-authored by Claude Opus 5.5
parent 42a60e3e74
commit 29c22afcb7
13 changed files with 348 additions and 25 deletions
+5
View File
@@ -46,6 +46,11 @@ class Source(Base):
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default="true")
config_overrides: Mapped[dict | None] = mapped_column(JSON, nullable=True)
# alembic 0115: what the source is called on its platform, where the URL
# alone doesn't say — a Discord link is two numbers. The ingester that
# walks it refreshes it every walk, so a rename reads as the new name.
# NULL until a walk has read it, and on platforms that don't write it.
display_name: Mapped[str | None] = mapped_column(Text, nullable=True)
last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
+93 -10
View File
@@ -32,6 +32,13 @@ Two deliberate departures, both about the walk order, neither about content:
the walk. gallery-dl skips only nested channels; one private thread the
token cannot read would otherwise stop every channel after it.
And one about content: a message is taken only if it has an image or video
ATTACHMENT (`has_visual_attachment`). Discord is where creators chat as well
as post, and the operator wants the art, not the conversation (2026-09-28): a
text line, a lone archive or PSD, a pasted link's preview or a Tenor GIF is
chat. A message that passes keeps every file gallery-dl would take, numbered
as gallery-dl numbers them, so the on-disk names still match.
FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
"""
@@ -79,6 +86,13 @@ _FORUM = frozenset({15, 16}) # forum, media: threads only
_CATEGORY = 4
_SERVER_WALK = _TEXT | _FORUM
_EMBED_TYPES = frozenset({"image", "gifv", "video"})
# What makes a message art rather than chat: an attached file FC imports as an
# image or video. Mirrors importer.ALL_EXTS (not imported: that module is the
# import pipeline, and this one is a network client).
_VISUAL_EXTS = frozenset({
"png", "jpg", "jpeg", "gif", "webp", "bmp", "tif", "tiff",
"mp4", "mov", "avi", "mkv", "webm", "m4v", "wmv", "flv",
})
_URL_RE = re.compile(
r"^(?:https?://)?(?:www\.|ptb\.|canary\.)?discord(?:app)?\.com/channels/"
@@ -163,6 +177,33 @@ def message_text(message: dict) -> str:
return "\n".join(p for p in parts if p)
def _message_and_snapshots(message: dict) -> list[dict]:
"""The message itself, then each forwarded message it carries."""
return [message] + [
(s or {}).get("message") or {}
for s in message.get("message_snapshots") or []
if ((s or {}).get("message") or {}).get("type", 0) in MESSAGE_TYPES
]
def _is_visual(attachment: dict) -> bool:
ctype = (attachment.get("content_type") or "").lower()
if ctype.startswith(("image/", "video/")):
return True
name = attachment.get("filename") or attachment.get("url") or ""
return nameext_from_url(name)[1] in _VISUAL_EXTS
def has_visual_attachment(message: dict) -> bool:
"""Does this message, or a message it forwards, have an image or video
attached? The one test for "art, not chat" — see the module docstring."""
return any(
att.get("url") and _is_visual(att)
for snap in _message_and_snapshots(message)
for att in snap.get("attachments") or []
)
@dataclass
class MediaItem:
"""One file of a Discord message. `filename`/`extension` are gallery-dl's
@@ -212,6 +253,8 @@ class DiscordClient:
self._server: dict = {}
self._channels: dict[str, dict] = {}
self._skip_feed = False
# Whose messages the walk takes (`only_from`); empty takes everyone's.
self._authors: frozenset[str] = frozenset()
# -- request -----------------------------------------------------------
@@ -373,6 +416,43 @@ class DiscordClient:
if meta["channel_type"] in _SERVER_WALK:
yield from self._feeds(meta["channel_id"], safe=True)
def only_from(self, authors) -> None:
"""Take only messages posted by these people: user ids, usernames or
display names, any case. A creator's server is full of other members
posting their own pictures; the source is subscribed to the creator.
Empty or None takes everyone's, as before."""
self._authors = frozenset(
str(a).strip().lower() for a in authors or () if str(a).strip()
)
def _author_wanted(self, message: dict) -> bool:
if not self._authors:
return True
author = message.get("author") or {}
names = (author.get("id"), author.get("username"), author.get("global_name"))
return any(str(n).lower() in self._authors for n in names if n)
def source_label(self, url: str) -> str | None:
"""What the source is called in Discord — `Server · #channel`, a
thread as `#parent › thread`, a whole server by its name — from the
metadata the walk loaded. None when the walk never got that far."""
try:
_, channel_id = parse_source_url(url)
except DiscordAPIError:
return None
server = self._server.get("server") or None
if channel_id is None:
return server
meta = self._channels.get(channel_id) or {}
name = meta.get("channel")
if not name:
return None
if name != "DMs":
name = f"#{name}"
if meta.get("is_thread") and meta.get("parent"):
name = f"#{meta['parent']} › {meta['channel']}"
return f"{server} · {name}" if server else name
def skip_feed(self) -> None:
"""Optional core seam (#4413): end the current channel and go on to the
next one. A tick's early-out means THIS channel has nothing new, not
@@ -440,6 +520,8 @@ class DiscordClient:
for message in messages:
if message.get("type") not in MESSAGE_TYPES:
continue
if not self._author_wanted(message):
continue
message["_meta"] = meta
yield message, meta, page_cursor
if self._skip_feed:
@@ -454,13 +536,14 @@ class DiscordClient:
def extract_media(post: dict, included: dict | None = None) -> list[MediaItem]:
"""gallery-dl's file list for one message: attachments, then the first
of video/image/thumbnail `proxy_url` of each file-bearing embed, then
the same for every forwarded snapshot; numbered from 1 across them."""
the same for every forwarded snapshot; numbered from 1 across them.
Empty for a message with no image or video attached: that is chat,
whatever else it carries (`has_visual_attachment`)."""
if not has_visual_attachment(post):
return []
mid = str(post.get("id") or "")
snapshots = [post] + [
(s or {}).get("message") or {}
for s in post.get("message_snapshots") or []
if ((s or {}).get("message") or {}).get("type", 0) in MESSAGE_TYPES
]
snapshots = _message_and_snapshots(post)
found: list[tuple[str, str, str | None]] = []
for snap in snapshots:
for att in snap.get("attachments") or []:
@@ -498,10 +581,10 @@ class DiscordClient:
"""`(message:<id>, <id>)` — gates the message record through the seen
ledger, like `post:<id>` on the other platforms.
None for a message with no files. gallery-dl wrote a sidecar only
beside a file, so a text-only chat line never became a post, and the
drop grouping (discord_grouping) is built on that: a channel's chatter
recorded as posts would bury the drops it exists to surface."""
None for a message `extract_media` takes nothing from, i.e. one with
no image or video attached. The drop grouping (discord_grouping) is
built on that: a channel's chatter recorded as posts would bury the
drops it exists to surface."""
mid = post.get("id")
mid = str(mid) if mid is not None else ""
if not mid or not cls.extract_media(post):
+54 -1
View File
@@ -13,6 +13,11 @@ Two things differ from the cookie platforms:
something; a Discord drop is routinely files and nothing else, so on
Discord that is an ordinary backfill, not a broken parser.
And two things only a Discord source has: `discord_authors` in its
config_overrides limits it to the creator's own messages (a server's other
members post pictures too), and it learns its own name — the server and
channel it walks — into `source.display_name`.
`campaign_id` is the source URL (a server, channel, thread or category link).
FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
"""
@@ -20,15 +25,23 @@ FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
from __future__ import annotations
import asyncio
import logging
from collections.abc import Callable
from pathlib import Path
from ..models import DiscordFailedMedia, DiscordSeenMedia
from sqlalchemy import select, update
from ..models import DiscordFailedMedia, DiscordSeenMedia, Source
from .discord_client import DiscordAPIError, DiscordClient, MediaItem
from .discord_downloader import DiscordDownloader
from .ingest_core import Ingester
log = logging.getLogger(__name__)
_LEDGER_KEY_MAX = 128
# `config_overrides` key: the people whose messages this source takes (user
# ids, usernames or display names). Absent or empty takes everyone's.
AUTHORS_KEY = "discord_authors"
def _ledger_key(media: MediaItem) -> str:
@@ -74,6 +87,46 @@ class DiscordIngester(Ingester):
body_canary=False,
)
def run(self, **kwargs):
"""The core walk, bracketed by the two things only a Discord source
has: whose messages it takes, read before the walk, and the name of
what it walks, written after it (the walk is what loads that name)."""
source_id = kwargs["source_id"]
self.client.only_from(self._source_authors(source_id))
try:
return super().run(**kwargs)
finally:
self._record_display_name(source_id, kwargs["campaign_id"])
def _source_authors(self, source_id: int) -> list:
if self.session_factory is None:
return []
with self.session_factory() as session:
overrides = session.execute(
select(Source.config_overrides).where(Source.id == source_id)
).scalar_one_or_none() or {}
authors = overrides.get(AUTHORS_KEY) or []
return authors if isinstance(authors, list) else []
def _record_display_name(self, source_id: int, url: str) -> None:
"""Refreshed on every walk, never written once: a renamed channel
should read as its new name. A name the walk couldn't read leaves the
stored one alone, and a failure here never fails the walk."""
label = self.client.source_label(url)
if not label or self.session_factory is None:
return
try:
with self.session_factory() as session:
session.execute(
update(Source)
.where(Source.id == source_id)
.where(Source.display_name.is_distinct_from(label))
.values(display_name=label)
)
session.commit()
except Exception as exc: # a name is decoration — never fail the walk
log.warning("Discord: couldn't record source %s's name: %s", source_id, exc)
async def verify_discord_credential(url: str, auth_token: str | None) -> tuple[bool | None, str]:
"""The uniform `(ok, message)` probe: is the token valid, and can its
+14
View File
@@ -479,6 +479,10 @@ class PostFeedService:
# the UI uses this to explain why a post it linked to is not in the
# stream.
"absorbed_by_post_id": post.absorbed_by_post_id,
# #4481. The Discord channel a message was posted in: both the native
# post record and gallery-dl's sidecars carry it as `channel`. None
# on every other platform, and on a Discord post that never said.
"channel": _discord_channel(post, source),
"artist": {"id": artist.id, "name": artist.name, "slug": artist.slug},
"source": (
{"id": source.id, "platform": source.platform}
@@ -488,3 +492,13 @@ class PostFeedService:
"thumbnails_more": thumbs_entry["more"],
"attachments": atts_map.get(post.id, []),
}
def _discord_channel(post: Post, source: Source | None) -> str | None:
if source is None or source.platform != "discord":
return None
raw = post.raw_metadata if isinstance(post.raw_metadata, dict) else {}
channel = raw.get("channel")
if not isinstance(channel, str):
return None
return channel.strip() or None
+5
View File
@@ -68,6 +68,9 @@ class SourceRecord:
artist_slug: str
platform: str
url: str
# alembic 0115: the name the platform gives it, where the URL is opaque
# (a Discord link is two ids). None until a walk has read it.
display_name: str | None
enabled: bool
config_overrides: dict | None
last_checked_at: str | None
@@ -110,6 +113,7 @@ class SourceRecord:
"artist_slug": self.artist_slug,
"platform": self.platform,
"url": self.url,
"display_name": self.display_name,
"enabled": self.enabled,
"config_overrides": self.config_overrides,
"last_checked_at": self.last_checked_at,
@@ -301,6 +305,7 @@ class SourceService:
artist_slug=artist.slug,
platform=source.platform,
url=source.url,
display_name=source.display_name,
enabled=source.enabled,
config_overrides=source.config_overrides,
last_checked_at=source.last_checked_at.isoformat() if source.last_checked_at else None,