feat: the extension picks a Discord source's posters from who posts in the channel, stored by id (#4488)
CI and images / lint (push) Successful in 3s
CI and images / extension-version (push) Successful in 3s
CI and images / extension-test (push) Successful in 18s
CI and images / frontend-build (push) Successful in 26s
CI and images / backend-lint-and-test (push) Successful in 32s
CI and images / integration (push) Successful in 2m24s
CI and images / build-agent (push) Successful in 5s
CI and images / sign-extension (push) Successful in 6m13s
CI and images / build-web (push) Successful in 1m38s
CI and images / smoke-web (push) Successful in 54s
CI and images / promote (push) Successful in 2s
CI and images / lint (push) Successful in 3s
CI and images / extension-version (push) Successful in 3s
CI and images / extension-test (push) Successful in 18s
CI and images / frontend-build (push) Successful in 26s
CI and images / backend-lint-and-test (push) Successful in 32s
CI and images / integration (push) Successful in 2m24s
CI and images / build-agent (push) Successful in 5s
CI and images / sign-extension (push) Successful in 6m13s
CI and images / build-web (push) Successful in 1m38s
CI and images / smoke-web (push) Successful in 54s
CI and images / promote (push) Successful in 2s
On a Discord channel page the Add panel lists who posted in the newest 200 messages, the creator the server is named for (or owned by) ticked. On a channel FabledCurator already follows, the chip opens the same list with Save and Open artist. The list is kept as user ids, so a rename never quietly stops it matching; the name each was picked under rides beside it for display, and the Subscriptions dialog shows it. - DiscordClient.recent_posters + rank_posters (owner / name-matches-server; bots never suggested; it suggests, the operator ticks). - GET/POST /api/extension/discord/posters; quick-add takes discord_authors. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -75,6 +75,9 @@ _THREADS_BATCH = 25
|
||||
# it is honoured when present; 60s is the fallback and the cap.
|
||||
_MAX_429_RETRIES = 4
|
||||
_429_WAIT_SECONDS = 60.0
|
||||
# How far back the poster picker looks: two pages. Enough to see who posts in
|
||||
# a channel, few enough to answer while the operator waits (#4488).
|
||||
_POSTER_SCAN = 200
|
||||
|
||||
# https://discord.com/developers/docs/resources/message#message-object-message-types
|
||||
# DEFAULT, REPLY, CHAT_INPUT_COMMAND — the ones that carry user content.
|
||||
@@ -204,6 +207,55 @@ def has_visual_attachment(message: dict) -> bool:
|
||||
)
|
||||
|
||||
|
||||
_SQUASH = re.compile(r"[\W_]+", re.UNICODE)
|
||||
# Words a server name wraps around its creator's: "Todding's Server",
|
||||
# "The Official Todding Discord".
|
||||
_SERVER_WORDS = frozenset({"server", "discord", "the", "official", "community", "s"})
|
||||
|
||||
|
||||
def _squash(text: str | None) -> str:
|
||||
return _SQUASH.sub("", (text or "").lower())
|
||||
|
||||
|
||||
def _server_tokens(server_name: str | None) -> list[str]:
|
||||
name = (server_name or "").lower().replace("'s", " ").replace("\u2019s", " ")
|
||||
words = [w for w in _SQUASH.split(name) if w and w not in _SERVER_WORDS]
|
||||
tokens = [w for w in words if len(w) >= 3]
|
||||
joined = "".join(words)
|
||||
if len(joined) >= 3 and joined not in tokens:
|
||||
tokens.append(joined)
|
||||
return tokens
|
||||
|
||||
|
||||
def rank_posters(server_name: str | None, owner_id, posters: list[dict]) -> list[dict]:
|
||||
"""Mark who is probably the creator a server is named for, and order the
|
||||
list for picking: suggested first, then by images posted, then messages.
|
||||
|
||||
Two signals, each a reason the picker shows beside the name:
|
||||
- they own the server;
|
||||
- their username or display name matches the server's name, once
|
||||
"'s Server", "Official", "Discord" and punctuation are taken off
|
||||
("Todding's Server" and "todding").
|
||||
Neither signal alone is proof, which is why this suggests and never
|
||||
decides: the operator ticks the list. Bots are never suggested."""
|
||||
tokens = _server_tokens(server_name)
|
||||
owner = str(owner_id) if owner_id else None
|
||||
ranked = []
|
||||
for p in posters:
|
||||
reasons = []
|
||||
if owner and p.get("id") == owner:
|
||||
reasons.append("owns the server")
|
||||
names = [n for n in (_squash(p.get("username")), _squash(p.get("global_name"))) if len(n) >= 3]
|
||||
if any(t in n or n in t for t in tokens for n in names):
|
||||
reasons.append("name matches the server")
|
||||
ranked.append({**p, "reasons": reasons, "suggested": False})
|
||||
best = max((len(r["reasons"]) for r in ranked if not r.get("bot")), default=0)
|
||||
for r in ranked:
|
||||
r["suggested"] = best > 0 and not r.get("bot") and len(r["reasons"]) == best
|
||||
ranked.sort(key=lambda r: (not r["suggested"], -r.get("images", 0), -r.get("messages", 0)))
|
||||
return ranked
|
||||
|
||||
|
||||
@dataclass
|
||||
class MediaItem:
|
||||
"""One file of a Discord message. `filename`/`extension` are gallery-dl's
|
||||
@@ -611,6 +663,61 @@ class DiscordClient:
|
||||
pass
|
||||
return out
|
||||
|
||||
def recent_posters(
|
||||
self, server_id: str | None, channel_id: str, *, max_messages: int = _POSTER_SCAN,
|
||||
) -> dict:
|
||||
"""Who has posted lately in a channel, for picking a source's poster
|
||||
list (#4488): the newest `max_messages` messages, tallied per author.
|
||||
Deliberately shallow — it answers "who posts here", not "who ever did".
|
||||
Every poster comes back with their stable id, which is what the list
|
||||
stores: a username can change on a whim, an id can't."""
|
||||
server: dict = {}
|
||||
if server_id:
|
||||
try:
|
||||
server = self._get(f"/guilds/{server_id}") or {}
|
||||
except DiscordAPIError:
|
||||
server = {}
|
||||
tally: dict[str, dict] = {}
|
||||
before = None
|
||||
seen = 0
|
||||
while seen < max_messages:
|
||||
page = self._get(
|
||||
f"/channels/{channel_id}/messages",
|
||||
{"limit": min(_MESSAGES_BATCH, max_messages - seen), "before": before},
|
||||
)
|
||||
if not isinstance(page, list) or not page:
|
||||
break
|
||||
for message in page:
|
||||
seen += 1
|
||||
if message.get("type") not in MESSAGE_TYPES:
|
||||
continue
|
||||
author = message.get("author") or {}
|
||||
aid = str(author.get("id") or "")
|
||||
if not aid:
|
||||
continue
|
||||
row = tally.setdefault(aid, {
|
||||
"id": aid,
|
||||
"username": author.get("username"),
|
||||
"global_name": author.get("global_name"),
|
||||
"bot": bool(author.get("bot")),
|
||||
"messages": 0,
|
||||
"images": 0,
|
||||
})
|
||||
row["messages"] += 1
|
||||
if has_visual_attachment(message):
|
||||
row["images"] += 1
|
||||
if len(page) < _MESSAGES_BATCH:
|
||||
break
|
||||
before = str(page[-1]["id"])
|
||||
return {
|
||||
"server": server.get("name") or None,
|
||||
"owner_id": str(server["owner_id"]) if server.get("owner_id") else None,
|
||||
"scanned": seen,
|
||||
"posters": rank_posters(
|
||||
server.get("name"), server.get("owner_id"), list(tally.values()),
|
||||
),
|
||||
}
|
||||
|
||||
def verify_auth(self, url: str) -> tuple[bool | None, str]:
|
||||
"""Is the token valid, and can it see what the source names?"""
|
||||
try:
|
||||
|
||||
@@ -23,6 +23,12 @@ log = logging.getLogger(__name__)
|
||||
|
||||
# The probe runs while the chip is drawing; names that take longer are skipped.
|
||||
_NAME_LOOKUP_SECONDS = 6.0
|
||||
# The poster picker waits on two pages of a channel's messages (#4488).
|
||||
_POSTER_LOOKUP_SECONDS = 20.0
|
||||
|
||||
# A source's poster list (discord_ingester.AUTHORS_KEY) holds user ids; this
|
||||
# keeps the name each id was picked under, for showing the list to a person.
|
||||
AUTHOR_LABELS_KEY = "discord_author_labels"
|
||||
|
||||
|
||||
class UnknownPlatformError(Exception):
|
||||
@@ -37,6 +43,15 @@ class UnknownArtistError(Exception):
|
||||
"""quick-add named an `artist_id` that does not exist."""
|
||||
|
||||
|
||||
class PosterLookupError(Exception):
|
||||
"""The Discord poster picker couldn't read the channel (no token, no
|
||||
access, not a channel page, or Discord was slow)."""
|
||||
|
||||
|
||||
class UnknownSourceError(Exception):
|
||||
"""No source follows this Discord channel or its server."""
|
||||
|
||||
|
||||
# Mirrored byte-for-byte from extension/lib/platforms.js
|
||||
# PLATFORM_ARTIST_PATTERNS. Keep these two copies in sync by hand —
|
||||
# reviewers catch drift.
|
||||
@@ -110,6 +125,7 @@ class ExtensionService:
|
||||
artist_id: int | None = None,
|
||||
artist_name: str | None = None,
|
||||
use_platform_name: bool = False,
|
||||
discord_authors: list[dict] | None = None,
|
||||
) -> dict:
|
||||
"""Add `url` as a source. `artist_id` connects it to an existing
|
||||
artist, `artist_name` to that artist (created if new); with neither,
|
||||
@@ -119,7 +135,10 @@ class ExtensionService:
|
||||
name is canon: a Patreon source added to an existing artist renames
|
||||
that artist to the creator's Patreon display name. Name only — the
|
||||
slug, and every path keyed off it, never moves (#130). Ignored on every
|
||||
other platform, and when the name can't be read."""
|
||||
other platform, and when the name can't be read.
|
||||
|
||||
`discord_authors` ([{id, name}], #4488) sets a Discord source's poster
|
||||
list in the same step, on a new source or one already there."""
|
||||
platform, raw_slug = self._derive(url)
|
||||
url = canonical_source_url(platform, url, raw_slug)
|
||||
renamed_from = None
|
||||
@@ -133,6 +152,8 @@ class ExtensionService:
|
||||
artist = (await self.session.execute(
|
||||
select(Artist).where(Artist.id == existing.artist_id)
|
||||
)).scalar_one()
|
||||
if platform == DISCORD and discord_authors is not None:
|
||||
await self._apply_posters(existing, discord_authors)
|
||||
return self._shape(existing, artist, created_source=False, created_artist=False)
|
||||
|
||||
if artist_id is not None:
|
||||
@@ -154,11 +175,98 @@ class ExtensionService:
|
||||
source, created_source = await self._find_or_create_source(
|
||||
artist_id=artist.id, platform=platform, url=url,
|
||||
)
|
||||
if platform == DISCORD and discord_authors is not None:
|
||||
await self._apply_posters(source, discord_authors)
|
||||
shaped = self._shape(source, artist, created_source, created_artist)
|
||||
if renamed_from is not None:
|
||||
shaped["renamed_from"] = renamed_from
|
||||
return shaped
|
||||
|
||||
# -- Discord poster picker (#4488) ------------------------------------------
|
||||
|
||||
def _discord_page(self, url: str) -> tuple[str, str]:
|
||||
platform, raw_slug = self._derive(url)
|
||||
if platform != DISCORD:
|
||||
raise InvalidUrlError("not a Discord page")
|
||||
server_id, _, channel_id = raw_slug.partition("/")
|
||||
if not channel_id:
|
||||
raise PosterLookupError("Open a channel to see who posts in it.")
|
||||
return server_id, channel_id
|
||||
|
||||
async def _source_for_page(self, server_id: str, channel_id: str) -> Source | None:
|
||||
"""The source this channel's posts land on: the channel's own, else a
|
||||
whole-server source that walks it."""
|
||||
base = f"https://discord.com/channels/{server_id}"
|
||||
return (
|
||||
await self._existing_source(DISCORD, f"{base}/{channel_id}")
|
||||
or await self._existing_source(DISCORD, base)
|
||||
)
|
||||
|
||||
async def discord_posters(self, url: str) -> dict:
|
||||
"""Who posts in the channel `url` shows, newest `_POSTER_SCAN`
|
||||
messages deep, with the likely creator marked — and, when a source
|
||||
already follows it, the poster list that source has now."""
|
||||
server_id, channel_id = self._discord_page(url)
|
||||
import asyncio
|
||||
|
||||
from .credential_service import CredentialService
|
||||
from .discord_client import DiscordAPIError, DiscordClient
|
||||
token = None
|
||||
if self._crypto is not None:
|
||||
token = await CredentialService(self.session, self._crypto).get_token(DISCORD)
|
||||
if not token:
|
||||
raise PosterLookupError("No Discord token is saved in FabledCurator.")
|
||||
client = DiscordClient(token, max_retries=0)
|
||||
loop = asyncio.get_running_loop()
|
||||
try:
|
||||
found = await asyncio.wait_for(
|
||||
loop.run_in_executor(None, client.recent_posters, server_id, channel_id),
|
||||
timeout=_POSTER_LOOKUP_SECONDS,
|
||||
)
|
||||
except TimeoutError as exc:
|
||||
raise PosterLookupError("Discord took too long to answer; try again.") from exc
|
||||
except DiscordAPIError as exc:
|
||||
raise PosterLookupError(f"Couldn't read this channel: {exc}") from exc
|
||||
source = await self._source_for_page(server_id, channel_id)
|
||||
co = (source.config_overrides or {}) if source is not None else {}
|
||||
from .discord_ingester import AUTHORS_KEY
|
||||
return {
|
||||
**found,
|
||||
"source_id": source.id if source is not None else None,
|
||||
"selected": list(co.get(AUTHORS_KEY) or []),
|
||||
# So someone on the list who hasn't posted lately still shows by
|
||||
# name, and can be unticked.
|
||||
"selected_labels": dict(co.get(AUTHOR_LABELS_KEY) or {}),
|
||||
}
|
||||
|
||||
async def set_discord_posters(self, url: str, authors: list[dict]) -> dict:
|
||||
"""Store the picked posters on the source that follows `url`'s channel."""
|
||||
server_id, channel_id = self._discord_page(url)
|
||||
source = await self._source_for_page(server_id, channel_id)
|
||||
if source is None:
|
||||
raise UnknownSourceError("No source follows this channel yet.")
|
||||
return await self._apply_posters(source, authors)
|
||||
|
||||
async def _apply_posters(self, source: Source, authors: list[dict]) -> dict:
|
||||
"""Ids are what the list keys on — a username can change on a whim,
|
||||
an id can't — and the name each was picked under rides beside it for
|
||||
showing the list to a person. An empty pick clears the list, which
|
||||
takes everyone's posts again."""
|
||||
from .discord_ingester import AUTHORS_KEY
|
||||
picked = [a for a in authors if str(a.get("id") or "").strip()]
|
||||
co = dict(source.config_overrides or {})
|
||||
if picked:
|
||||
co[AUTHORS_KEY] = [str(a["id"]).strip() for a in picked]
|
||||
co[AUTHOR_LABELS_KEY] = {
|
||||
str(a["id"]).strip(): str(a.get("name") or a["id"]) for a in picked
|
||||
}
|
||||
else:
|
||||
co.pop(AUTHORS_KEY, None)
|
||||
co.pop(AUTHOR_LABELS_KEY, None)
|
||||
source.config_overrides = co or None
|
||||
await self.session.commit()
|
||||
return {"source_id": source.id, "discord_authors": co.get(AUTHORS_KEY, [])}
|
||||
|
||||
async def _adopt_patreon_name(self, artist, raw_slug: str, url: str) -> str | None:
|
||||
"""Rename `artist` to the Patreon display name; the old name when it
|
||||
changed, else None. Unreadable name → no rename, never the handle."""
|
||||
|
||||
Reference in New Issue
Block a user