diff --git a/alembic/versions/0115_source_display_name.py b/alembic/versions/0115_source_display_name.py new file mode 100644 index 0000000..2f113df --- /dev/null +++ b/alembic/versions/0115_source_display_name.py @@ -0,0 +1,27 @@ +"""source.display_name — what a source is called on its platform. + +#4481. A Discord source's URL is a server id and a channel id, so the +Subscriptions page could only show two numbers. The Discord ingester already +loads the server and channel names to walk them; it now keeps them here, +refreshed on every walk. NULL until a walk has read it. + +Revision ID: 0115 +Revises: 0114 +Create Date: 2026-09-28 + +""" +import sqlalchemy as sa +from alembic import op + +revision = "0115" +down_revision = "0114" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("source", sa.Column("display_name", sa.Text(), nullable=True)) + + +def downgrade() -> None: + op.drop_column("source", "display_name") diff --git a/backend/app/models/source.py b/backend/app/models/source.py index 3501505..96901fb 100644 --- a/backend/app/models/source.py +++ b/backend/app/models/source.py @@ -46,6 +46,11 @@ class Source(Base): enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default="true") config_overrides: Mapped[dict | None] = mapped_column(JSON, nullable=True) + # alembic 0115: what the source is called on its platform, where the URL + # alone doesn't say — a Discord link is two numbers. The ingester that + # walks it refreshes it every walk, so a rename reads as the new name. + # NULL until a walk has read it, and on platforms that don't write it. + display_name: Mapped[str | None] = mapped_column(Text, nullable=True) last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) last_error: Mapped[str | None] = mapped_column(Text, nullable=True) diff --git a/backend/app/services/discord_client.py b/backend/app/services/discord_client.py index 9d5b154..d2f4f77 100644 --- a/backend/app/services/discord_client.py +++ b/backend/app/services/discord_client.py @@ -32,6 +32,13 @@ Two deliberate departures, both about the walk order, neither about content: the walk. gallery-dl skips only nested channels; one private thread the token cannot read would otherwise stop every channel after it. +And one about content: a message is taken only if it has an image or video +ATTACHMENT (`has_visual_attachment`). Discord is where creators chat as well +as post, and the operator wants the art, not the conversation (2026-09-28): a +text line, a lone archive or PSD, a pasted link's preview or a Tenor GIF is +chat. A message that passes keeps every file gallery-dl would take, numbered +as gallery-dl numbers them, so the on-disk names still match. + FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API. """ @@ -79,6 +86,13 @@ _FORUM = frozenset({15, 16}) # forum, media: threads only _CATEGORY = 4 _SERVER_WALK = _TEXT | _FORUM _EMBED_TYPES = frozenset({"image", "gifv", "video"}) +# What makes a message art rather than chat: an attached file FC imports as an +# image or video. Mirrors importer.ALL_EXTS (not imported: that module is the +# import pipeline, and this one is a network client). +_VISUAL_EXTS = frozenset({ + "png", "jpg", "jpeg", "gif", "webp", "bmp", "tif", "tiff", + "mp4", "mov", "avi", "mkv", "webm", "m4v", "wmv", "flv", +}) _URL_RE = re.compile( r"^(?:https?://)?(?:www\.|ptb\.|canary\.)?discord(?:app)?\.com/channels/" @@ -163,6 +177,33 @@ def message_text(message: dict) -> str: return "\n".join(p for p in parts if p) +def _message_and_snapshots(message: dict) -> list[dict]: + """The message itself, then each forwarded message it carries.""" + return [message] + [ + (s or {}).get("message") or {} + for s in message.get("message_snapshots") or [] + if ((s or {}).get("message") or {}).get("type", 0) in MESSAGE_TYPES + ] + + +def _is_visual(attachment: dict) -> bool: + ctype = (attachment.get("content_type") or "").lower() + if ctype.startswith(("image/", "video/")): + return True + name = attachment.get("filename") or attachment.get("url") or "" + return nameext_from_url(name)[1] in _VISUAL_EXTS + + +def has_visual_attachment(message: dict) -> bool: + """Does this message, or a message it forwards, have an image or video + attached? The one test for "art, not chat" — see the module docstring.""" + return any( + att.get("url") and _is_visual(att) + for snap in _message_and_snapshots(message) + for att in snap.get("attachments") or [] + ) + + @dataclass class MediaItem: """One file of a Discord message. `filename`/`extension` are gallery-dl's @@ -212,6 +253,8 @@ class DiscordClient: self._server: dict = {} self._channels: dict[str, dict] = {} self._skip_feed = False + # Whose messages the walk takes (`only_from`); empty takes everyone's. + self._authors: frozenset[str] = frozenset() # -- request ----------------------------------------------------------- @@ -373,6 +416,43 @@ class DiscordClient: if meta["channel_type"] in _SERVER_WALK: yield from self._feeds(meta["channel_id"], safe=True) + def only_from(self, authors) -> None: + """Take only messages posted by these people: user ids, usernames or + display names, any case. A creator's server is full of other members + posting their own pictures; the source is subscribed to the creator. + Empty or None takes everyone's, as before.""" + self._authors = frozenset( + str(a).strip().lower() for a in authors or () if str(a).strip() + ) + + def _author_wanted(self, message: dict) -> bool: + if not self._authors: + return True + author = message.get("author") or {} + names = (author.get("id"), author.get("username"), author.get("global_name")) + return any(str(n).lower() in self._authors for n in names if n) + + def source_label(self, url: str) -> str | None: + """What the source is called in Discord — `Server · #channel`, a + thread as `#parent › thread`, a whole server by its name — from the + metadata the walk loaded. None when the walk never got that far.""" + try: + _, channel_id = parse_source_url(url) + except DiscordAPIError: + return None + server = self._server.get("server") or None + if channel_id is None: + return server + meta = self._channels.get(channel_id) or {} + name = meta.get("channel") + if not name: + return None + if name != "DMs": + name = f"#{name}" + if meta.get("is_thread") and meta.get("parent"): + name = f"#{meta['parent']} › {meta['channel']}" + return f"{server} · {name}" if server else name + def skip_feed(self) -> None: """Optional core seam (#4413): end the current channel and go on to the next one. A tick's early-out means THIS channel has nothing new, not @@ -440,6 +520,8 @@ class DiscordClient: for message in messages: if message.get("type") not in MESSAGE_TYPES: continue + if not self._author_wanted(message): + continue message["_meta"] = meta yield message, meta, page_cursor if self._skip_feed: @@ -454,13 +536,14 @@ class DiscordClient: def extract_media(post: dict, included: dict | None = None) -> list[MediaItem]: """gallery-dl's file list for one message: attachments, then the first of video/image/thumbnail `proxy_url` of each file-bearing embed, then - the same for every forwarded snapshot; numbered from 1 across them.""" + the same for every forwarded snapshot; numbered from 1 across them. + + Empty for a message with no image or video attached: that is chat, + whatever else it carries (`has_visual_attachment`).""" + if not has_visual_attachment(post): + return [] mid = str(post.get("id") or "") - snapshots = [post] + [ - (s or {}).get("message") or {} - for s in post.get("message_snapshots") or [] - if ((s or {}).get("message") or {}).get("type", 0) in MESSAGE_TYPES - ] + snapshots = _message_and_snapshots(post) found: list[tuple[str, str, str | None]] = [] for snap in snapshots: for att in snap.get("attachments") or []: @@ -498,10 +581,10 @@ class DiscordClient: """`(message:, )` — gates the message record through the seen ledger, like `post:` on the other platforms. - None for a message with no files. gallery-dl wrote a sidecar only - beside a file, so a text-only chat line never became a post, and the - drop grouping (discord_grouping) is built on that: a channel's chatter - recorded as posts would bury the drops it exists to surface.""" + None for a message `extract_media` takes nothing from, i.e. one with + no image or video attached. The drop grouping (discord_grouping) is + built on that: a channel's chatter recorded as posts would bury the + drops it exists to surface.""" mid = post.get("id") mid = str(mid) if mid is not None else "" if not mid or not cls.extract_media(post): diff --git a/backend/app/services/discord_ingester.py b/backend/app/services/discord_ingester.py index f83721f..0e2dff2 100644 --- a/backend/app/services/discord_ingester.py +++ b/backend/app/services/discord_ingester.py @@ -13,6 +13,11 @@ Two things differ from the cookie platforms: something; a Discord drop is routinely files and nothing else, so on Discord that is an ordinary backfill, not a broken parser. +And two things only a Discord source has: `discord_authors` in its +config_overrides limits it to the creator's own messages (a server's other +members post pictures too), and it learns its own name — the server and +channel it walks — into `source.display_name`. + `campaign_id` is the source URL (a server, channel, thread or category link). FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API. """ @@ -20,15 +25,23 @@ FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API. from __future__ import annotations import asyncio +import logging from collections.abc import Callable from pathlib import Path -from ..models import DiscordFailedMedia, DiscordSeenMedia +from sqlalchemy import select, update + +from ..models import DiscordFailedMedia, DiscordSeenMedia, Source from .discord_client import DiscordAPIError, DiscordClient, MediaItem from .discord_downloader import DiscordDownloader from .ingest_core import Ingester +log = logging.getLogger(__name__) + _LEDGER_KEY_MAX = 128 +# `config_overrides` key: the people whose messages this source takes (user +# ids, usernames or display names). Absent or empty takes everyone's. +AUTHORS_KEY = "discord_authors" def _ledger_key(media: MediaItem) -> str: @@ -74,6 +87,46 @@ class DiscordIngester(Ingester): body_canary=False, ) + def run(self, **kwargs): + """The core walk, bracketed by the two things only a Discord source + has: whose messages it takes, read before the walk, and the name of + what it walks, written after it (the walk is what loads that name).""" + source_id = kwargs["source_id"] + self.client.only_from(self._source_authors(source_id)) + try: + return super().run(**kwargs) + finally: + self._record_display_name(source_id, kwargs["campaign_id"]) + + def _source_authors(self, source_id: int) -> list: + if self.session_factory is None: + return [] + with self.session_factory() as session: + overrides = session.execute( + select(Source.config_overrides).where(Source.id == source_id) + ).scalar_one_or_none() or {} + authors = overrides.get(AUTHORS_KEY) or [] + return authors if isinstance(authors, list) else [] + + def _record_display_name(self, source_id: int, url: str) -> None: + """Refreshed on every walk, never written once: a renamed channel + should read as its new name. A name the walk couldn't read leaves the + stored one alone, and a failure here never fails the walk.""" + label = self.client.source_label(url) + if not label or self.session_factory is None: + return + try: + with self.session_factory() as session: + session.execute( + update(Source) + .where(Source.id == source_id) + .where(Source.display_name.is_distinct_from(label)) + .values(display_name=label) + ) + session.commit() + except Exception as exc: # a name is decoration — never fail the walk + log.warning("Discord: couldn't record source %s's name: %s", source_id, exc) + async def verify_discord_credential(url: str, auth_token: str | None) -> tuple[bool | None, str]: """The uniform `(ok, message)` probe: is the token valid, and can its diff --git a/backend/app/services/post_feed_service.py b/backend/app/services/post_feed_service.py index e41b5d2..36deb48 100644 --- a/backend/app/services/post_feed_service.py +++ b/backend/app/services/post_feed_service.py @@ -479,6 +479,10 @@ class PostFeedService: # the UI uses this to explain why a post it linked to is not in the # stream. "absorbed_by_post_id": post.absorbed_by_post_id, + # #4481. The Discord channel a message was posted in: both the native + # post record and gallery-dl's sidecars carry it as `channel`. None + # on every other platform, and on a Discord post that never said. + "channel": _discord_channel(post, source), "artist": {"id": artist.id, "name": artist.name, "slug": artist.slug}, "source": ( {"id": source.id, "platform": source.platform} @@ -488,3 +492,13 @@ class PostFeedService: "thumbnails_more": thumbs_entry["more"], "attachments": atts_map.get(post.id, []), } + + +def _discord_channel(post: Post, source: Source | None) -> str | None: + if source is None or source.platform != "discord": + return None + raw = post.raw_metadata if isinstance(post.raw_metadata, dict) else {} + channel = raw.get("channel") + if not isinstance(channel, str): + return None + return channel.strip() or None diff --git a/backend/app/services/source_service.py b/backend/app/services/source_service.py index 42201a7..4b5130a 100644 --- a/backend/app/services/source_service.py +++ b/backend/app/services/source_service.py @@ -68,6 +68,9 @@ class SourceRecord: artist_slug: str platform: str url: str + # alembic 0115: the name the platform gives it, where the URL is opaque + # (a Discord link is two ids). None until a walk has read it. + display_name: str | None enabled: bool config_overrides: dict | None last_checked_at: str | None @@ -110,6 +113,7 @@ class SourceRecord: "artist_slug": self.artist_slug, "platform": self.platform, "url": self.url, + "display_name": self.display_name, "enabled": self.enabled, "config_overrides": self.config_overrides, "last_checked_at": self.last_checked_at, @@ -301,6 +305,7 @@ class SourceService: artist_slug=artist.slug, platform=source.platform, url=source.url, + display_name=source.display_name, enabled=source.enabled, config_overrides=source.config_overrides, last_checked_at=source.last_checked_at.isoformat() if source.last_checked_at else None, diff --git a/frontend/src/components/posts/PostCard.vue b/frontend/src/components/posts/PostCard.vue index d8a7981..07a22ef 100644 --- a/frontend/src/components/posts/PostCard.vue +++ b/frontend/src/components/posts/PostCard.vue @@ -26,6 +26,7 @@ :to="{ name: 'artist', params: { slug: post.artist.slug } }" class="fc-post-card__artist" >{{ post.artist.name }} + #{{ post.channel }} {{ relativeDate }} · {{ totalImages }} image{{ totalImages === 1 ? '' : 's' }} diff --git a/frontend/src/components/subscriptions/SourceCard.vue b/frontend/src/components/subscriptions/SourceCard.vue index c4526b5..c381f4f 100644 --- a/frontend/src/components/subscriptions/SourceCard.vue +++ b/frontend/src/components/subscriptions/SourceCard.vue @@ -16,7 +16,7 @@ {{ source.url }} + >{{ source.display_name || source.url }} + +

More per-platform fields land here over time. Use Advanced JSON for everything else.

@@ -98,6 +106,24 @@ const urlError = ref('') const configTab = ref('structured') const structuredVideos = ref(true) const structuredSince = ref('') +const structuredAuthors = ref('') +// Keys the structured view has no field for, carried through its saves so +// switching tabs never drops what the JSON view set. +const otherConfig = ref({}) +const STRUCTURED_KEYS = ['videos', 'since', 'discord_authors'] + +function splitAuthors(txt) { + return (txt || '').split(',').map(s => s.trim()).filter(Boolean) +} + +function takeConfig(co) { + structuredVideos.value = co.videos !== false + structuredSince.value = co.since ?? '' + structuredAuthors.value = Array.isArray(co.discord_authors) ? co.discord_authors.join(', ') : '' + otherConfig.value = Object.fromEntries( + Object.entries(co).filter(([k]) => !STRUCTURED_KEYS.includes(k)), + ) +} const jsonText = ref('{}') const jsonError = ref('') @@ -105,13 +131,15 @@ const busy = ref(false) // Sync config_overrides between the two views. const config = computed(() => { - const out = {} + const out = { ...otherConfig.value } if (!structuredVideos.value) out.videos = false if (structuredSince.value) out.since = structuredSince.value + const authors = splitAuthors(structuredAuthors.value) + if (authors.length) out.discord_authors = authors return out }) -watch([structuredVideos, structuredSince], () => { +watch([structuredVideos, structuredSince, structuredAuthors], () => { if (configTab.value === 'structured') { jsonText.value = JSON.stringify(config.value, null, 2) jsonError.value = '' @@ -128,8 +156,7 @@ watch(jsonText, (txt) => { } jsonError.value = '' // Reflect recognized keys into the structured view. - structuredVideos.value = parsed.videos !== false - structuredSince.value = parsed.since ?? '' + takeConfig(parsed) } catch { jsonError.value = 'Invalid JSON' } @@ -146,14 +173,13 @@ watch(() => props.modelValue, async (open) => { url.value = props.source.url enabled.value = props.source.enabled const co = props.source.config_overrides || {} - structuredVideos.value = co.videos !== false - structuredSince.value = co.since ?? '' + takeConfig(co) jsonText.value = JSON.stringify(co, null, 2) artistChoice.value = { id: props.source.artist_id, name: props.source.artist_name } } else { platform.value = platformsStore.list[0]?.key || 'patreon' url.value = ''; enabled.value = true - structuredVideos.value = true; structuredSince.value = '' + takeConfig({}) jsonText.value = '{}' artistChoice.value = props.initialArtist ? { id: props.initialArtist.id, name: props.initialArtist.name } diff --git a/frontend/src/components/subscriptions/SourceRow.vue b/frontend/src/components/subscriptions/SourceRow.vue index 74cfb0d..4964175 100644 --- a/frontend/src/components/subscriptions/SourceRow.vue +++ b/frontend/src/components/subscriptions/SourceRow.vue @@ -10,7 +10,7 @@
- {{ source.url }} + {{ source.display_name || source.url }} diff --git a/frontend/src/components/subscriptions/SubscriptionsTab.vue b/frontend/src/components/subscriptions/SubscriptionsTab.vue index 5c0aa35..d0e0dc7 100644 --- a/frontend/src/components/subscriptions/SubscriptionsTab.vue +++ b/frontend/src/components/subscriptions/SubscriptionsTab.vue @@ -108,7 +108,7 @@ v-if="item.singleSource" :href="item.singleSource.url" target="_blank" rel="noopener" class="fc-subs__sub-url" @click.stop - >{{ item.singleSource.url }} + >{{ item.singleSource.display_name || item.singleSource.url }}