Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ef591edf10 | ||
|
|
1efece0f21 | ||
|
|
20aeac81df | ||
|
|
d6d3184361 | ||
|
|
29c22afcb7 |
@@ -0,0 +1,27 @@
|
||||
"""source.display_name — what a source is called on its platform.
|
||||
|
||||
#4481. A Discord source's URL is a server id and a channel id, so the
|
||||
Subscriptions page could only show two numbers. The Discord ingester already
|
||||
loads the server and channel names to walk them; it now keeps them here,
|
||||
refreshed on every walk. NULL until a walk has read it.
|
||||
|
||||
Revision ID: 0115
|
||||
Revises: 0114
|
||||
Create Date: 2026-09-28
|
||||
|
||||
"""
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0115"
|
||||
down_revision = "0114"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("source", sa.Column("display_name", sa.Text(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("source", "display_name")
|
||||
@@ -0,0 +1,34 @@
|
||||
"""The single-color filter's default becomes near-total: 0.95 → 0.995.
|
||||
|
||||
#4483. At 0.95 the filter rejected line art: a pencil doodle on white is
|
||||
mostly white, and five of Todding's Discord doodles were skipped on import as
|
||||
"single color", leaving their posts with text and no image. The predicate now
|
||||
samples real pixels instead of a blurred thumbnail, and the default only
|
||||
calls an image blank when it essentially is one.
|
||||
|
||||
A stored 0.95 is the old default, so it moves with it. Any other value was
|
||||
chosen by hand and is left alone.
|
||||
|
||||
Revision ID: 0116
|
||||
Revises: 0115
|
||||
Create Date: 2026-09-28
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
|
||||
revision = "0116"
|
||||
down_revision = "0115"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.alter_column("import_settings", "single_color_threshold", server_default="0.995")
|
||||
op.execute(
|
||||
"UPDATE import_settings SET single_color_threshold = 0.995 "
|
||||
"WHERE single_color_threshold = 0.95"
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.alter_column("import_settings", "single_color_threshold", server_default="0.95")
|
||||
@@ -19,8 +19,10 @@ from ..models import AppSetting
|
||||
from ..services.extension_service import (
|
||||
ExtensionService,
|
||||
InvalidUrlError,
|
||||
PosterLookupError,
|
||||
UnknownArtistError,
|
||||
UnknownPlatformError,
|
||||
UnknownSourceError,
|
||||
)
|
||||
from ..services.source_service import KNOWN_PLATFORMS
|
||||
from ._responses import error_response as _bad
|
||||
@@ -121,6 +123,10 @@ async def quick_add_source():
|
||||
# Patreon is canon: adding a Patreon source to an existing artist can take
|
||||
# the creator's Patreon display name (name only; the slug never moves).
|
||||
use_platform_name = body.get("use_platform_name") is True
|
||||
# #4488: a Discord add can carry the posters picked in the panel.
|
||||
discord_authors = body.get("discord_authors")
|
||||
if discord_authors is not None and not _valid_authors(discord_authors):
|
||||
return _bad("invalid_body", detail="discord_authors must be a list of {id, name}")
|
||||
|
||||
from .credentials import _get_crypto
|
||||
|
||||
@@ -133,6 +139,7 @@ async def quick_add_source():
|
||||
result = await ExtensionService(session, _get_crypto()).quick_add_source(
|
||||
url, artist_id=artist_id, artist_name=artist_name,
|
||||
use_platform_name=use_platform_name,
|
||||
discord_authors=discord_authors,
|
||||
)
|
||||
except UnknownArtistError as exc:
|
||||
return _bad("not_found", detail=str(exc), status=404)
|
||||
@@ -147,6 +154,59 @@ async def quick_add_source():
|
||||
return jsonify(result), (201 if result["created_source"] else 200)
|
||||
|
||||
|
||||
def _valid_authors(value) -> bool:
|
||||
"""[{id, name}]: the id is what the list keys on, the name only shows it."""
|
||||
return isinstance(value, list) and all(
|
||||
isinstance(a, dict) and isinstance(a.get("id"), str | int)
|
||||
and not isinstance(a.get("id"), bool)
|
||||
and (a.get("name") is None or isinstance(a.get("name"), str))
|
||||
for a in value
|
||||
)
|
||||
|
||||
|
||||
@extension_bp.route("/discord/posters", methods=["GET"])
|
||||
async def discord_posters():
|
||||
"""Who posts in the Discord channel `url` shows, lately, with the likely
|
||||
creator marked (#4488). Reads the channel with the stored token; writes
|
||||
nothing."""
|
||||
url = (request.args.get("url") or "").strip()
|
||||
if not url:
|
||||
return _bad("invalid_body", detail="url query parameter is required")
|
||||
from .credentials import _get_crypto
|
||||
|
||||
async with get_session() as session:
|
||||
if not await _ext_key_required(session):
|
||||
return _bad("unauthorized", status=401)
|
||||
try:
|
||||
result = await ExtensionService(session, _get_crypto()).discord_posters(url)
|
||||
except (InvalidUrlError, UnknownPlatformError) as exc:
|
||||
return _bad("invalid_url", detail=str(exc))
|
||||
except PosterLookupError as exc:
|
||||
return _bad("posters_unavailable", detail=str(exc), status=409)
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@extension_bp.route("/discord/posters", methods=["POST"])
|
||||
async def set_discord_posters():
|
||||
"""Set the poster list of the source following `url`'s channel."""
|
||||
body = await request.get_json(silent=True)
|
||||
if not isinstance(body, dict) or not isinstance(body.get("url"), str):
|
||||
return _bad("invalid_body", detail="url is required")
|
||||
authors = body.get("authors")
|
||||
if not _valid_authors(authors):
|
||||
return _bad("invalid_body", detail="authors must be a list of {id, name}")
|
||||
async with get_session() as session:
|
||||
if not await _ext_key_required(session):
|
||||
return _bad("unauthorized", status=401)
|
||||
try:
|
||||
result = await ExtensionService(session).set_discord_posters(body["url"], authors)
|
||||
except (InvalidUrlError, UnknownPlatformError, PosterLookupError) as exc:
|
||||
return _bad("invalid_url", detail=str(exc))
|
||||
except UnknownSourceError as exc:
|
||||
return _bad("not_found", detail=str(exc), status=404)
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
def _read_manifest_sync() -> dict | None:
|
||||
"""All the filesystem-touching work for /api/extension/manifest,
|
||||
in a sync helper so the async route can dispatch it via
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
"""FC-3a: CRUD over Source rows. FC-3c adds POST /<id>/check."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import DownloadEvent, MembershipSync, PlatformMembership, Source
|
||||
from ..services import discord_poster_cleanup as poster_cleanup
|
||||
from ..services.artist_membership_service import ArtistMembershipService
|
||||
from ..services.artist_membership_service import rescan as membership_rescan
|
||||
from ..services.artist_service import ArtistService
|
||||
@@ -463,3 +466,49 @@ async def adopt_membership():
|
||||
return jsonify({"already_tracked": exc.existing_id})
|
||||
artist_id = artist.id
|
||||
return jsonify({"source_id": record.id, "artist_id": artist_id}), 201
|
||||
|
||||
|
||||
# -- Discord: remove posts by people outside the poster list (#4486) --------------
|
||||
|
||||
_IMAGES_ROOT = Path("/images")
|
||||
|
||||
|
||||
async def _poster_cleanup_preview(source_id: int):
|
||||
async with get_session() as session:
|
||||
return await session.run_sync(
|
||||
lambda s: poster_cleanup.preview(s, source_id=source_id)
|
||||
)
|
||||
|
||||
|
||||
@sources_bp.route("/<int:source_id>/discord/other-posters", methods=["GET"])
|
||||
async def other_posters_preview(source_id: int):
|
||||
"""What removing other posters' posts would delete. Nothing is touched."""
|
||||
try:
|
||||
projection = await _poster_cleanup_preview(source_id)
|
||||
except LookupError:
|
||||
return _bad("not_found", status=404)
|
||||
except poster_cleanup.PosterCleanupError as exc:
|
||||
return _bad("not_applicable", detail=str(exc), status=409)
|
||||
projection["confirm_token"] = poster_cleanup.confirm_token(projection)
|
||||
return jsonify(projection)
|
||||
|
||||
|
||||
@sources_bp.route("/<int:source_id>/discord/other-posters/remove", methods=["POST"])
|
||||
async def other_posters_remove(source_id: int):
|
||||
"""Delete them. `confirm` must be the token of the preview the operator saw:
|
||||
a poster list edited since then changes the set, and the token with it."""
|
||||
body = await request.get_json(silent=True) or {}
|
||||
try:
|
||||
projection = await _poster_cleanup_preview(source_id)
|
||||
except LookupError:
|
||||
return _bad("not_found", status=404)
|
||||
except poster_cleanup.PosterCleanupError as exc:
|
||||
return _bad("not_applicable", detail=str(exc), status=409)
|
||||
expected = poster_cleanup.confirm_token(projection)
|
||||
if body.get("confirm") != expected:
|
||||
return _bad("confirm_mismatch", expected=expected)
|
||||
async with get_session() as session:
|
||||
result = await session.run_sync(
|
||||
lambda s: poster_cleanup.apply(s, source_id=source_id, images_root=_IMAGES_ROOT)
|
||||
)
|
||||
return jsonify(result)
|
||||
|
||||
@@ -39,7 +39,7 @@ class ImportSettings(Base):
|
||||
transparency_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.9, server_default="0.9")
|
||||
|
||||
skip_single_color: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default="false")
|
||||
single_color_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.95, server_default="0.95")
|
||||
single_color_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.995, server_default="0.995")
|
||||
single_color_tolerance: Mapped[int] = mapped_column(Integer, nullable=False, default=30, server_default="30")
|
||||
|
||||
# Hamming distance over a 256-bit pHash (utils.phash, hash_size=16). The
|
||||
|
||||
@@ -46,6 +46,11 @@ class Source(Base):
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default="true")
|
||||
|
||||
config_overrides: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
# alembic 0115: what the source is called on its platform, where the URL
|
||||
# alone doesn't say — a Discord link is two numbers. The ingester that
|
||||
# walks it refreshes it every walk, so a rename reads as the new name.
|
||||
# NULL until a walk has read it, and on platforms that don't write it.
|
||||
display_name: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
|
||||
last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
|
||||
@@ -4,11 +4,19 @@ predicate for BOTH surfaces: FC-Cleanup's retroactive audit and — since
|
||||
2026-07-02 — the import-side filter (Importer._single_color_hit /
|
||||
SkipReason.single_color), so what the audit flags and what the import
|
||||
skips can never disagree.
|
||||
|
||||
It is meant to catch the blank: a placeholder, an error tile, a solid fill.
|
||||
Line art is the case it must not catch (#4483): a pencil doodle on white can be
|
||||
well over 95% white even at full size, and a smoothing downsample blends its strokes
|
||||
into the paper until it measures as blank. So the sample is taken by NEAREST
|
||||
(each sampled pixel is a real pixel, strokes keep their contrast), at 256px
|
||||
so thin strokes are still hit, and the default threshold is near-total
|
||||
(0.995) — a few percent of ink is a drawing, not an empty image.
|
||||
"""
|
||||
|
||||
from PIL import Image
|
||||
|
||||
_THUMB_SIZE = (64, 64)
|
||||
_THUMB_SIZE = (256, 256)
|
||||
|
||||
|
||||
def evaluate(
|
||||
@@ -20,7 +28,8 @@ def evaluate(
|
||||
"""True iff the fraction of pixels within `tolerance` (Euclidean RGB
|
||||
distance) of the dominant color exceeds `threshold`.
|
||||
|
||||
Downsamples to 64x64 for speed (~4ms regardless of source size).
|
||||
Samples 256x256 by NEAREST (see the module docstring for why not a
|
||||
smoothing resample).
|
||||
Alpha channels are stripped; only RGB is considered. Animated images
|
||||
use frame 0 (PIL's default after Image.open without seek).
|
||||
"""
|
||||
@@ -30,7 +39,7 @@ def evaluate(
|
||||
elif im.mode not in ("RGB", "L"):
|
||||
im = im.convert("RGB")
|
||||
if im.size != _THUMB_SIZE:
|
||||
im = im.resize(_THUMB_SIZE, Image.Resampling.BILINEAR)
|
||||
im = im.resize(_THUMB_SIZE, Image.Resampling.NEAREST)
|
||||
pixels = list(im.getdata())
|
||||
if not pixels:
|
||||
return False
|
||||
|
||||
@@ -32,6 +32,13 @@ Two deliberate departures, both about the walk order, neither about content:
|
||||
the walk. gallery-dl skips only nested channels; one private thread the
|
||||
token cannot read would otherwise stop every channel after it.
|
||||
|
||||
And one about content: a message is taken only if it has an image or video
|
||||
ATTACHMENT (`has_visual_attachment`). Discord is where creators chat as well
|
||||
as post, and the operator wants the art, not the conversation (2026-09-28): a
|
||||
text line, a lone archive or PSD, a pasted link's preview or a Tenor GIF is
|
||||
chat. A message that passes keeps every file gallery-dl would take, numbered
|
||||
as gallery-dl numbers them, so the on-disk names still match.
|
||||
|
||||
FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
|
||||
"""
|
||||
|
||||
@@ -68,6 +75,9 @@ _THREADS_BATCH = 25
|
||||
# it is honoured when present; 60s is the fallback and the cap.
|
||||
_MAX_429_RETRIES = 4
|
||||
_429_WAIT_SECONDS = 60.0
|
||||
# How far back the poster picker looks: two pages. Enough to see who posts in
|
||||
# a channel, few enough to answer while the operator waits (#4488).
|
||||
_POSTER_SCAN = 200
|
||||
|
||||
# https://discord.com/developers/docs/resources/message#message-object-message-types
|
||||
# DEFAULT, REPLY, CHAT_INPUT_COMMAND — the ones that carry user content.
|
||||
@@ -79,6 +89,13 @@ _FORUM = frozenset({15, 16}) # forum, media: threads only
|
||||
_CATEGORY = 4
|
||||
_SERVER_WALK = _TEXT | _FORUM
|
||||
_EMBED_TYPES = frozenset({"image", "gifv", "video"})
|
||||
# What makes a message art rather than chat: an attached file FC imports as an
|
||||
# image or video. Mirrors importer.ALL_EXTS (not imported: that module is the
|
||||
# import pipeline, and this one is a network client).
|
||||
_VISUAL_EXTS = frozenset({
|
||||
"png", "jpg", "jpeg", "gif", "webp", "bmp", "tif", "tiff",
|
||||
"mp4", "mov", "avi", "mkv", "webm", "m4v", "wmv", "flv",
|
||||
})
|
||||
|
||||
_URL_RE = re.compile(
|
||||
r"^(?:https?://)?(?:www\.|ptb\.|canary\.)?discord(?:app)?\.com/channels/"
|
||||
@@ -163,6 +180,82 @@ def message_text(message: dict) -> str:
|
||||
return "\n".join(p for p in parts if p)
|
||||
|
||||
|
||||
def _message_and_snapshots(message: dict) -> list[dict]:
|
||||
"""The message itself, then each forwarded message it carries."""
|
||||
return [message] + [
|
||||
(s or {}).get("message") or {}
|
||||
for s in message.get("message_snapshots") or []
|
||||
if ((s or {}).get("message") or {}).get("type", 0) in MESSAGE_TYPES
|
||||
]
|
||||
|
||||
|
||||
def _is_visual(attachment: dict) -> bool:
|
||||
ctype = (attachment.get("content_type") or "").lower()
|
||||
if ctype.startswith(("image/", "video/")):
|
||||
return True
|
||||
name = attachment.get("filename") or attachment.get("url") or ""
|
||||
return nameext_from_url(name)[1] in _VISUAL_EXTS
|
||||
|
||||
|
||||
def has_visual_attachment(message: dict) -> bool:
|
||||
"""Does this message, or a message it forwards, have an image or video
|
||||
attached? The one test for "art, not chat" — see the module docstring."""
|
||||
return any(
|
||||
att.get("url") and _is_visual(att)
|
||||
for snap in _message_and_snapshots(message)
|
||||
for att in snap.get("attachments") or []
|
||||
)
|
||||
|
||||
|
||||
_SQUASH = re.compile(r"[\W_]+", re.UNICODE)
|
||||
# Words a server name wraps around its creator's: "Todding's Server",
|
||||
# "The Official Todding Discord".
|
||||
_SERVER_WORDS = frozenset({"server", "discord", "the", "official", "community", "s"})
|
||||
|
||||
|
||||
def _squash(text: str | None) -> str:
|
||||
return _SQUASH.sub("", (text or "").lower())
|
||||
|
||||
|
||||
def _server_tokens(server_name: str | None) -> list[str]:
|
||||
name = (server_name or "").lower().replace("'s", " ").replace("\u2019s", " ")
|
||||
words = [w for w in _SQUASH.split(name) if w and w not in _SERVER_WORDS]
|
||||
tokens = [w for w in words if len(w) >= 3]
|
||||
joined = "".join(words)
|
||||
if len(joined) >= 3 and joined not in tokens:
|
||||
tokens.append(joined)
|
||||
return tokens
|
||||
|
||||
|
||||
def rank_posters(server_name: str | None, owner_id, posters: list[dict]) -> list[dict]:
|
||||
"""Mark who is probably the creator a server is named for, and order the
|
||||
list for picking: suggested first, then by images posted, then messages.
|
||||
|
||||
Two signals, each a reason the picker shows beside the name:
|
||||
- they own the server;
|
||||
- their username or display name matches the server's name, once
|
||||
"'s Server", "Official", "Discord" and punctuation are taken off
|
||||
("Todding's Server" and "todding").
|
||||
Neither signal alone is proof, which is why this suggests and never
|
||||
decides: the operator ticks the list. Bots are never suggested."""
|
||||
tokens = _server_tokens(server_name)
|
||||
owner = str(owner_id) if owner_id else None
|
||||
ranked = []
|
||||
for p in posters:
|
||||
reasons = []
|
||||
if owner and p.get("id") == owner:
|
||||
reasons.append("owns the server")
|
||||
names = [n for n in (_squash(p.get("username")), _squash(p.get("global_name"))) if len(n) >= 3]
|
||||
if any(t in n or n in t for t in tokens for n in names):
|
||||
reasons.append("name matches the server")
|
||||
ranked.append({**p, "reasons": reasons, "suggested": False})
|
||||
best = max((len(r["reasons"]) for r in ranked if not r.get("bot")), default=0)
|
||||
for r in ranked:
|
||||
r["suggested"] = best > 0 and not r.get("bot") and len(r["reasons"]) == best
|
||||
ranked.sort(key=lambda r: (not r["suggested"], -r.get("images", 0), -r.get("messages", 0)))
|
||||
return ranked
|
||||
|
||||
|
||||
@dataclass
|
||||
class MediaItem:
|
||||
"""One file of a Discord message. `filename`/`extension` are gallery-dl's
|
||||
@@ -212,6 +305,8 @@ class DiscordClient:
|
||||
self._server: dict = {}
|
||||
self._channels: dict[str, dict] = {}
|
||||
self._skip_feed = False
|
||||
# Whose messages the walk takes (`only_from`); empty takes everyone's.
|
||||
self._authors: frozenset[str] = frozenset()
|
||||
|
||||
# -- request -----------------------------------------------------------
|
||||
|
||||
@@ -373,6 +468,43 @@ class DiscordClient:
|
||||
if meta["channel_type"] in _SERVER_WALK:
|
||||
yield from self._feeds(meta["channel_id"], safe=True)
|
||||
|
||||
def only_from(self, authors) -> None:
|
||||
"""Take only messages posted by these people: user ids, usernames or
|
||||
display names, any case. A creator's server is full of other members
|
||||
posting their own pictures; the source is subscribed to the creator.
|
||||
Empty or None takes everyone's, as before."""
|
||||
self._authors = frozenset(
|
||||
str(a).strip().lower() for a in authors or () if str(a).strip()
|
||||
)
|
||||
|
||||
def _author_wanted(self, message: dict) -> bool:
|
||||
if not self._authors:
|
||||
return True
|
||||
author = message.get("author") or {}
|
||||
names = (author.get("id"), author.get("username"), author.get("global_name"))
|
||||
return any(str(n).lower() in self._authors for n in names if n)
|
||||
|
||||
def source_label(self, url: str) -> str | None:
|
||||
"""What the source is called in Discord — `Server · #channel`, a
|
||||
thread as `#parent › thread`, a whole server by its name — from the
|
||||
metadata the walk loaded. None when the walk never got that far."""
|
||||
try:
|
||||
_, channel_id = parse_source_url(url)
|
||||
except DiscordAPIError:
|
||||
return None
|
||||
server = self._server.get("server") or None
|
||||
if channel_id is None:
|
||||
return server
|
||||
meta = self._channels.get(channel_id) or {}
|
||||
name = meta.get("channel")
|
||||
if not name:
|
||||
return None
|
||||
if name != "DMs":
|
||||
name = f"#{name}"
|
||||
if meta.get("is_thread") and meta.get("parent"):
|
||||
name = f"#{meta['parent']} › {meta['channel']}"
|
||||
return f"{server} · {name}" if server else name
|
||||
|
||||
def skip_feed(self) -> None:
|
||||
"""Optional core seam (#4413): end the current channel and go on to the
|
||||
next one. A tick's early-out means THIS channel has nothing new, not
|
||||
@@ -440,6 +572,8 @@ class DiscordClient:
|
||||
for message in messages:
|
||||
if message.get("type") not in MESSAGE_TYPES:
|
||||
continue
|
||||
if not self._author_wanted(message):
|
||||
continue
|
||||
message["_meta"] = meta
|
||||
yield message, meta, page_cursor
|
||||
if self._skip_feed:
|
||||
@@ -454,13 +588,14 @@ class DiscordClient:
|
||||
def extract_media(post: dict, included: dict | None = None) -> list[MediaItem]:
|
||||
"""gallery-dl's file list for one message: attachments, then the first
|
||||
of video/image/thumbnail `proxy_url` of each file-bearing embed, then
|
||||
the same for every forwarded snapshot; numbered from 1 across them."""
|
||||
the same for every forwarded snapshot; numbered from 1 across them.
|
||||
|
||||
Empty for a message with no image or video attached: that is chat,
|
||||
whatever else it carries (`has_visual_attachment`)."""
|
||||
if not has_visual_attachment(post):
|
||||
return []
|
||||
mid = str(post.get("id") or "")
|
||||
snapshots = [post] + [
|
||||
(s or {}).get("message") or {}
|
||||
for s in post.get("message_snapshots") or []
|
||||
if ((s or {}).get("message") or {}).get("type", 0) in MESSAGE_TYPES
|
||||
]
|
||||
snapshots = _message_and_snapshots(post)
|
||||
found: list[tuple[str, str, str | None]] = []
|
||||
for snap in snapshots:
|
||||
for att in snap.get("attachments") or []:
|
||||
@@ -498,10 +633,10 @@ class DiscordClient:
|
||||
"""`(message:<id>, <id>)` — gates the message record through the seen
|
||||
ledger, like `post:<id>` on the other platforms.
|
||||
|
||||
None for a message with no files. gallery-dl wrote a sidecar only
|
||||
beside a file, so a text-only chat line never became a post, and the
|
||||
drop grouping (discord_grouping) is built on that: a channel's chatter
|
||||
recorded as posts would bury the drops it exists to surface."""
|
||||
None for a message `extract_media` takes nothing from, i.e. one with
|
||||
no image or video attached. The drop grouping (discord_grouping) is
|
||||
built on that: a channel's chatter recorded as posts would bury the
|
||||
drops it exists to surface."""
|
||||
mid = post.get("id")
|
||||
mid = str(mid) if mid is not None else ""
|
||||
if not mid or not cls.extract_media(post):
|
||||
@@ -528,6 +663,61 @@ class DiscordClient:
|
||||
pass
|
||||
return out
|
||||
|
||||
def recent_posters(
|
||||
self, server_id: str | None, channel_id: str, *, max_messages: int = _POSTER_SCAN,
|
||||
) -> dict:
|
||||
"""Who has posted lately in a channel, for picking a source's poster
|
||||
list (#4488): the newest `max_messages` messages, tallied per author.
|
||||
Deliberately shallow — it answers "who posts here", not "who ever did".
|
||||
Every poster comes back with their stable id, which is what the list
|
||||
stores: a username can change on a whim, an id can't."""
|
||||
server: dict = {}
|
||||
if server_id:
|
||||
try:
|
||||
server = self._get(f"/guilds/{server_id}") or {}
|
||||
except DiscordAPIError:
|
||||
server = {}
|
||||
tally: dict[str, dict] = {}
|
||||
before = None
|
||||
seen = 0
|
||||
while seen < max_messages:
|
||||
page = self._get(
|
||||
f"/channels/{channel_id}/messages",
|
||||
{"limit": min(_MESSAGES_BATCH, max_messages - seen), "before": before},
|
||||
)
|
||||
if not isinstance(page, list) or not page:
|
||||
break
|
||||
for message in page:
|
||||
seen += 1
|
||||
if message.get("type") not in MESSAGE_TYPES:
|
||||
continue
|
||||
author = message.get("author") or {}
|
||||
aid = str(author.get("id") or "")
|
||||
if not aid:
|
||||
continue
|
||||
row = tally.setdefault(aid, {
|
||||
"id": aid,
|
||||
"username": author.get("username"),
|
||||
"global_name": author.get("global_name"),
|
||||
"bot": bool(author.get("bot")),
|
||||
"messages": 0,
|
||||
"images": 0,
|
||||
})
|
||||
row["messages"] += 1
|
||||
if has_visual_attachment(message):
|
||||
row["images"] += 1
|
||||
if len(page) < _MESSAGES_BATCH:
|
||||
break
|
||||
before = str(page[-1]["id"])
|
||||
return {
|
||||
"server": server.get("name") or None,
|
||||
"owner_id": str(server["owner_id"]) if server.get("owner_id") else None,
|
||||
"scanned": seen,
|
||||
"posters": rank_posters(
|
||||
server.get("name"), server.get("owner_id"), list(tally.values()),
|
||||
),
|
||||
}
|
||||
|
||||
def verify_auth(self, url: str) -> tuple[bool | None, str]:
|
||||
"""Is the token valid, and can it see what the source names?"""
|
||||
try:
|
||||
|
||||
@@ -195,6 +195,7 @@ class DiscordDownloader(BaseNativeDownloader):
|
||||
"parent": meta.get("parent"),
|
||||
"is_thread": meta.get("is_thread"),
|
||||
"author": author.get("username"),
|
||||
"author_name": author.get("global_name"),
|
||||
"author_id": author.get("id"),
|
||||
"message": body,
|
||||
"date": post.get("timestamp"),
|
||||
|
||||
@@ -13,6 +13,11 @@ Two things differ from the cookie platforms:
|
||||
something; a Discord drop is routinely files and nothing else, so on
|
||||
Discord that is an ordinary backfill, not a broken parser.
|
||||
|
||||
And two things only a Discord source has: `discord_authors` in its
|
||||
config_overrides limits it to the creator's own messages (a server's other
|
||||
members post pictures too), and it learns its own name — the server and
|
||||
channel it walks — into `source.display_name`.
|
||||
|
||||
`campaign_id` is the source URL (a server, channel, thread or category link).
|
||||
FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
|
||||
"""
|
||||
@@ -20,15 +25,23 @@ FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
from ..models import DiscordFailedMedia, DiscordSeenMedia
|
||||
from sqlalchemy import select, update
|
||||
|
||||
from ..models import DiscordFailedMedia, DiscordSeenMedia, Source
|
||||
from .discord_client import DiscordAPIError, DiscordClient, MediaItem
|
||||
from .discord_downloader import DiscordDownloader
|
||||
from .ingest_core import Ingester
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_LEDGER_KEY_MAX = 128
|
||||
# `config_overrides` key: the people whose messages this source takes (user
|
||||
# ids, usernames or display names). Absent or empty takes everyone's.
|
||||
AUTHORS_KEY = "discord_authors"
|
||||
|
||||
|
||||
def _ledger_key(media: MediaItem) -> str:
|
||||
@@ -74,6 +87,46 @@ class DiscordIngester(Ingester):
|
||||
body_canary=False,
|
||||
)
|
||||
|
||||
def run(self, **kwargs):
|
||||
"""The core walk, bracketed by the two things only a Discord source
|
||||
has: whose messages it takes, read before the walk, and the name of
|
||||
what it walks, written after it (the walk is what loads that name)."""
|
||||
source_id = kwargs["source_id"]
|
||||
self.client.only_from(self._source_authors(source_id))
|
||||
try:
|
||||
return super().run(**kwargs)
|
||||
finally:
|
||||
self._record_display_name(source_id, kwargs["campaign_id"])
|
||||
|
||||
def _source_authors(self, source_id: int) -> list:
|
||||
if self.session_factory is None:
|
||||
return []
|
||||
with self.session_factory() as session:
|
||||
overrides = session.execute(
|
||||
select(Source.config_overrides).where(Source.id == source_id)
|
||||
).scalar_one_or_none() or {}
|
||||
authors = overrides.get(AUTHORS_KEY) or []
|
||||
return authors if isinstance(authors, list) else []
|
||||
|
||||
def _record_display_name(self, source_id: int, url: str) -> None:
|
||||
"""Refreshed on every walk, never written once: a renamed channel
|
||||
should read as its new name. A name the walk couldn't read leaves the
|
||||
stored one alone, and a failure here never fails the walk."""
|
||||
label = self.client.source_label(url)
|
||||
if not label or self.session_factory is None:
|
||||
return
|
||||
try:
|
||||
with self.session_factory() as session:
|
||||
session.execute(
|
||||
update(Source)
|
||||
.where(Source.id == source_id)
|
||||
.where(Source.display_name.is_distinct_from(label))
|
||||
.values(display_name=label)
|
||||
)
|
||||
session.commit()
|
||||
except Exception as exc: # a name is decoration — never fail the walk
|
||||
log.warning("Discord: couldn't record source %s's name: %s", source_id, exc)
|
||||
|
||||
|
||||
async def verify_discord_credential(url: str, auth_token: str | None) -> tuple[bool | None, str]:
|
||||
"""The uniform `(ok, message)` probe: is the token valid, and can its
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
"""Remove a Discord source's posts by people outside its poster list (#4486).
|
||||
|
||||
A creator's server is full of other members posting their own pictures. The
|
||||
poster list (`discord_authors`, #4481) stops a walk taking them; this removes
|
||||
the ones taken before the list existed. The operator sets the list first, then
|
||||
previews what would go, then applies.
|
||||
|
||||
## What goes
|
||||
|
||||
A post of this source whose record names its poster (`author_id` / `author`,
|
||||
written by `DiscordDownloader.write_post_record`) and names someone NOT on the
|
||||
list. Two kinds of post are never touched:
|
||||
|
||||
* a post whose record names no poster — a message recorded before the native
|
||||
ingester, whose poster FC cannot tell. Counted as `unknown`, never guessed;
|
||||
* a synthetic drop post (`synthesized_by`) — FC authored it, and it is handled
|
||||
below as a consequence, not matched as a poster's post.
|
||||
|
||||
An image goes when every REAL post it belongs to is going. A link to a
|
||||
synthetic drop does not keep an image: the drop only references its members'
|
||||
images. An image also on a kept post — the creator re-posting a piece someone
|
||||
else shared — stays.
|
||||
|
||||
A drop that absorbed a removed message is deleted outright. Deleting a drop is
|
||||
its undo (discord_grouping's honesty rule): its remaining members return to
|
||||
the feed and the grouping sweep regroups them on its next run, so no half-
|
||||
rebuilt drop keeps text or thumbnails from someone it no longer contains.
|
||||
|
||||
Preview and apply spread the same predicates (rule 93, snippet #3087).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import and_, delete, exists, func, or_, select
|
||||
from sqlalchemy.orm import Session, aliased
|
||||
|
||||
from ..models import ImageProvenance, ImageRecord, Post, PostAttachment, Source
|
||||
from .cleanup_service import delete_images
|
||||
from .discord_ingester import AUTHORS_KEY
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
PLATFORM = "discord"
|
||||
# The record keys that name a message's poster. `author_name` (the display
|
||||
# name) is only on records written after 2026-09-28.
|
||||
_POSTER_KEYS = ("author_id", "author", "author_name")
|
||||
|
||||
|
||||
class PosterCleanupError(ValueError):
|
||||
"""The source can't be cleaned this way (not Discord, or no list)."""
|
||||
|
||||
|
||||
def source_authors(source: Source) -> list[str]:
|
||||
"""The source's poster list, lowercased; empty means everyone's wanted."""
|
||||
authors = (source.config_overrides or {}).get(AUTHORS_KEY) or []
|
||||
if not isinstance(authors, list):
|
||||
return []
|
||||
return sorted({str(a).strip().lower() for a in authors if str(a).strip()})
|
||||
|
||||
|
||||
def _poster(key: str):
|
||||
return func.lower(Post.raw_metadata[key].as_string())
|
||||
|
||||
|
||||
def _has_poster():
|
||||
return or_(*(Post.raw_metadata[k].as_string().isnot(None) for k in ("author_id", "author")))
|
||||
|
||||
|
||||
def _real_post_conditions(source_id: int) -> list:
|
||||
return [Post.source_id == source_id, Post.synthesized_by.is_(None)]
|
||||
|
||||
|
||||
def _named_poster_conditions(source_id: int) -> list:
|
||||
"""A real post of this source whose record says who posted it."""
|
||||
return [*_real_post_conditions(source_id), _has_poster()]
|
||||
|
||||
|
||||
def _other_poster_post_conditions(source_id: int, authors: list[str]) -> list:
|
||||
"""The posts that go: a named poster none of whose names is on the list."""
|
||||
return [
|
||||
*_named_poster_conditions(source_id),
|
||||
*(func.coalesce(_poster(k), "").notin_(authors) for k in _POSTER_KEYS),
|
||||
]
|
||||
|
||||
|
||||
def _doomed_post_ids(source_id: int, authors: list[str]):
|
||||
# correlate(None): used inside queries that are themselves over `post` (the
|
||||
# drops, the delete), where auto-correlation would bind this to the outer
|
||||
# row instead of scanning the table.
|
||||
return (
|
||||
select(Post.id)
|
||||
.where(*_other_poster_post_conditions(source_id, authors))
|
||||
.correlate(None)
|
||||
)
|
||||
|
||||
|
||||
def _image_conditions(source_id: int, authors: list[str]) -> list:
|
||||
"""Images that belong to a removed post and to no kept real post."""
|
||||
doomed = _doomed_post_ids(source_id, authors)
|
||||
keeper = aliased(Post)
|
||||
on_doomed = or_(
|
||||
ImageRecord.primary_post_id.in_(doomed),
|
||||
exists().where(
|
||||
ImageProvenance.image_record_id == ImageRecord.id,
|
||||
ImageProvenance.post_id.in_(doomed),
|
||||
),
|
||||
)
|
||||
on_kept = or_(
|
||||
and_(
|
||||
ImageRecord.primary_post_id.isnot(None),
|
||||
ImageRecord.primary_post_id.notin_(doomed),
|
||||
),
|
||||
exists().where(
|
||||
ImageProvenance.image_record_id == ImageRecord.id,
|
||||
ImageProvenance.post_id == keeper.id,
|
||||
keeper.synthesized_by.is_(None),
|
||||
keeper.id.notin_(doomed),
|
||||
),
|
||||
)
|
||||
return [on_doomed, ~on_kept]
|
||||
|
||||
|
||||
def _drop_conditions(source_id: int, authors: list[str]) -> list:
|
||||
"""The synthetic drops that absorbed a removed message."""
|
||||
member = aliased(Post)
|
||||
return [
|
||||
Post.source_id == source_id,
|
||||
Post.synthesized_by.isnot(None),
|
||||
exists().where(
|
||||
member.absorbed_by_post_id == Post.id,
|
||||
member.id.in_(_doomed_post_ids(source_id, authors)),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _attachment_conditions(source_id: int, authors: list[str]) -> list:
|
||||
return [PostAttachment.post_id.in_(_doomed_post_ids(source_id, authors))]
|
||||
|
||||
|
||||
def _load(session: Session, source_id: int) -> tuple[Source, list[str]]:
|
||||
source = session.get(Source, source_id)
|
||||
if source is None:
|
||||
raise LookupError(source_id)
|
||||
if source.platform != PLATFORM:
|
||||
raise PosterCleanupError("Only a Discord source has posters to filter by.")
|
||||
authors = source_authors(source)
|
||||
if not authors:
|
||||
raise PosterCleanupError(
|
||||
"Set this source's 'Only posts by' list first — with no list, "
|
||||
"every poster is wanted and nothing would be removed."
|
||||
)
|
||||
return source, authors
|
||||
|
||||
|
||||
def _count(session: Session, stmt) -> int:
|
||||
return session.execute(stmt).scalar_one()
|
||||
|
||||
|
||||
def preview(session: Session, *, source_id: int) -> dict:
|
||||
"""What `apply` would remove, per poster, without touching anything."""
|
||||
_, authors = _load(session, source_id)
|
||||
who = func.coalesce(
|
||||
Post.raw_metadata["author_name"].as_string(),
|
||||
Post.raw_metadata["author"].as_string(),
|
||||
Post.raw_metadata["author_id"].as_string(),
|
||||
)
|
||||
rows = session.execute(
|
||||
select(who, func.count(Post.id))
|
||||
.where(*_other_poster_post_conditions(source_id, authors))
|
||||
.group_by(who)
|
||||
.order_by(func.count(Post.id).desc())
|
||||
).all()
|
||||
posters = [{"poster": name, "posts": n} for name, n in rows]
|
||||
unknown = _count(session, select(func.count(Post.id)).where(
|
||||
*_real_post_conditions(source_id), ~_has_poster(),
|
||||
))
|
||||
return {
|
||||
"authors": authors,
|
||||
"posters": posters,
|
||||
"posts": sum(p["posts"] for p in posters),
|
||||
"images": _count(session, select(func.count(ImageRecord.id)).where(
|
||||
*_image_conditions(source_id, authors))),
|
||||
"attachments": _count(session, select(func.count(PostAttachment.id)).where(
|
||||
*_attachment_conditions(source_id, authors))),
|
||||
"drops": _count(session, select(func.count(Post.id)).where(
|
||||
*_drop_conditions(source_id, authors))),
|
||||
"unknown_posts": unknown,
|
||||
}
|
||||
|
||||
|
||||
def confirm_token(projection: dict) -> str:
|
||||
"""What the operator's apply must echo: the preview it was shown, so a
|
||||
list edited between preview and apply can't delete an unseen set."""
|
||||
return f"remove-{projection['posts']}-posts-{projection['images']}-images"
|
||||
|
||||
|
||||
def apply(session: Session, *, source_id: int, images_root: Path) -> dict:
|
||||
"""Remove them. Counts are taken before the deletes they describe — once
|
||||
the rows are gone there is nothing left to count (snippet #3087 note 2)."""
|
||||
projection = preview(session, source_id=source_id)
|
||||
_, authors = _load(session, source_id)
|
||||
|
||||
image_ids = session.execute(
|
||||
select(ImageRecord.id).where(*_image_conditions(source_id, authors))
|
||||
).scalars().all()
|
||||
# Drops first-class: found BEFORE their members go, since the link that
|
||||
# finds them is the member's absorbed_by_post_id.
|
||||
drop_ids = session.execute(
|
||||
select(Post.id).where(*_drop_conditions(source_id, authors))
|
||||
).scalars().all()
|
||||
|
||||
deleted = delete_images(session, image_ids=list(image_ids), images_root=images_root)
|
||||
# Attachments before posts: post_attachment.post_id is SET NULL, and a
|
||||
# NULL-post attachment collides on its partial unique index (see
|
||||
# cleanup_service.delete_artist_cascade).
|
||||
attachments = session.execute(
|
||||
delete(PostAttachment).where(*_attachment_conditions(source_id, authors))
|
||||
).rowcount or 0
|
||||
posts = session.execute(
|
||||
Post.__table__.delete().where(Post.id.in_(_doomed_post_ids(source_id, authors)))
|
||||
).rowcount or 0
|
||||
drops = 0
|
||||
if drop_ids:
|
||||
drops = session.execute(
|
||||
Post.__table__.delete().where(Post.id.in_(drop_ids))
|
||||
).rowcount or 0
|
||||
session.commit()
|
||||
log.info(
|
||||
"discord poster cleanup (source %s, keeping %s): %d post(s), %d image(s), "
|
||||
"%d attachment(s), %d drop(s) removed",
|
||||
source_id, authors, posts, deleted["images_deleted"], attachments, drops,
|
||||
)
|
||||
return {
|
||||
**projection,
|
||||
"posts_deleted": posts,
|
||||
"images_deleted": deleted["images_deleted"],
|
||||
"files_deleted": deleted["files_deleted"],
|
||||
"attachments_deleted": attachments,
|
||||
"drops_deleted": drops,
|
||||
}
|
||||
@@ -23,6 +23,12 @@ log = logging.getLogger(__name__)
|
||||
|
||||
# The probe runs while the chip is drawing; names that take longer are skipped.
|
||||
_NAME_LOOKUP_SECONDS = 6.0
|
||||
# The poster picker waits on two pages of a channel's messages (#4488).
|
||||
_POSTER_LOOKUP_SECONDS = 20.0
|
||||
|
||||
# A source's poster list (discord_ingester.AUTHORS_KEY) holds user ids; this
|
||||
# keeps the name each id was picked under, for showing the list to a person.
|
||||
AUTHOR_LABELS_KEY = "discord_author_labels"
|
||||
|
||||
|
||||
class UnknownPlatformError(Exception):
|
||||
@@ -37,6 +43,15 @@ class UnknownArtistError(Exception):
|
||||
"""quick-add named an `artist_id` that does not exist."""
|
||||
|
||||
|
||||
class PosterLookupError(Exception):
|
||||
"""The Discord poster picker couldn't read the channel (no token, no
|
||||
access, not a channel page, or Discord was slow)."""
|
||||
|
||||
|
||||
class UnknownSourceError(Exception):
|
||||
"""No source follows this Discord channel or its server."""
|
||||
|
||||
|
||||
# Mirrored byte-for-byte from extension/lib/platforms.js
|
||||
# PLATFORM_ARTIST_PATTERNS. Keep these two copies in sync by hand —
|
||||
# reviewers catch drift.
|
||||
@@ -110,6 +125,7 @@ class ExtensionService:
|
||||
artist_id: int | None = None,
|
||||
artist_name: str | None = None,
|
||||
use_platform_name: bool = False,
|
||||
discord_authors: list[dict] | None = None,
|
||||
) -> dict:
|
||||
"""Add `url` as a source. `artist_id` connects it to an existing
|
||||
artist, `artist_name` to that artist (created if new); with neither,
|
||||
@@ -119,7 +135,10 @@ class ExtensionService:
|
||||
name is canon: a Patreon source added to an existing artist renames
|
||||
that artist to the creator's Patreon display name. Name only — the
|
||||
slug, and every path keyed off it, never moves (#130). Ignored on every
|
||||
other platform, and when the name can't be read."""
|
||||
other platform, and when the name can't be read.
|
||||
|
||||
`discord_authors` ([{id, name}], #4488) sets a Discord source's poster
|
||||
list in the same step, on a new source or one already there."""
|
||||
platform, raw_slug = self._derive(url)
|
||||
url = canonical_source_url(platform, url, raw_slug)
|
||||
renamed_from = None
|
||||
@@ -133,6 +152,8 @@ class ExtensionService:
|
||||
artist = (await self.session.execute(
|
||||
select(Artist).where(Artist.id == existing.artist_id)
|
||||
)).scalar_one()
|
||||
if platform == DISCORD and discord_authors is not None:
|
||||
await self._apply_posters(existing, discord_authors)
|
||||
return self._shape(existing, artist, created_source=False, created_artist=False)
|
||||
|
||||
if artist_id is not None:
|
||||
@@ -154,11 +175,98 @@ class ExtensionService:
|
||||
source, created_source = await self._find_or_create_source(
|
||||
artist_id=artist.id, platform=platform, url=url,
|
||||
)
|
||||
if platform == DISCORD and discord_authors is not None:
|
||||
await self._apply_posters(source, discord_authors)
|
||||
shaped = self._shape(source, artist, created_source, created_artist)
|
||||
if renamed_from is not None:
|
||||
shaped["renamed_from"] = renamed_from
|
||||
return shaped
|
||||
|
||||
# -- Discord poster picker (#4488) ------------------------------------------
|
||||
|
||||
def _discord_page(self, url: str) -> tuple[str, str]:
|
||||
platform, raw_slug = self._derive(url)
|
||||
if platform != DISCORD:
|
||||
raise InvalidUrlError("not a Discord page")
|
||||
server_id, _, channel_id = raw_slug.partition("/")
|
||||
if not channel_id:
|
||||
raise PosterLookupError("Open a channel to see who posts in it.")
|
||||
return server_id, channel_id
|
||||
|
||||
async def _source_for_page(self, server_id: str, channel_id: str) -> Source | None:
|
||||
"""The source this channel's posts land on: the channel's own, else a
|
||||
whole-server source that walks it."""
|
||||
base = f"https://discord.com/channels/{server_id}"
|
||||
return (
|
||||
await self._existing_source(DISCORD, f"{base}/{channel_id}")
|
||||
or await self._existing_source(DISCORD, base)
|
||||
)
|
||||
|
||||
async def discord_posters(self, url: str) -> dict:
|
||||
"""Who posts in the channel `url` shows, newest `_POSTER_SCAN`
|
||||
messages deep, with the likely creator marked — and, when a source
|
||||
already follows it, the poster list that source has now."""
|
||||
server_id, channel_id = self._discord_page(url)
|
||||
import asyncio
|
||||
|
||||
from .credential_service import CredentialService
|
||||
from .discord_client import DiscordAPIError, DiscordClient
|
||||
token = None
|
||||
if self._crypto is not None:
|
||||
token = await CredentialService(self.session, self._crypto).get_token(DISCORD)
|
||||
if not token:
|
||||
raise PosterLookupError("No Discord token is saved in FabledCurator.")
|
||||
client = DiscordClient(token, max_retries=0)
|
||||
loop = asyncio.get_running_loop()
|
||||
try:
|
||||
found = await asyncio.wait_for(
|
||||
loop.run_in_executor(None, client.recent_posters, server_id, channel_id),
|
||||
timeout=_POSTER_LOOKUP_SECONDS,
|
||||
)
|
||||
except TimeoutError as exc:
|
||||
raise PosterLookupError("Discord took too long to answer; try again.") from exc
|
||||
except DiscordAPIError as exc:
|
||||
raise PosterLookupError(f"Couldn't read this channel: {exc}") from exc
|
||||
source = await self._source_for_page(server_id, channel_id)
|
||||
co = (source.config_overrides or {}) if source is not None else {}
|
||||
from .discord_ingester import AUTHORS_KEY
|
||||
return {
|
||||
**found,
|
||||
"source_id": source.id if source is not None else None,
|
||||
"selected": list(co.get(AUTHORS_KEY) or []),
|
||||
# So someone on the list who hasn't posted lately still shows by
|
||||
# name, and can be unticked.
|
||||
"selected_labels": dict(co.get(AUTHOR_LABELS_KEY) or {}),
|
||||
}
|
||||
|
||||
async def set_discord_posters(self, url: str, authors: list[dict]) -> dict:
|
||||
"""Store the picked posters on the source that follows `url`'s channel."""
|
||||
server_id, channel_id = self._discord_page(url)
|
||||
source = await self._source_for_page(server_id, channel_id)
|
||||
if source is None:
|
||||
raise UnknownSourceError("No source follows this channel yet.")
|
||||
return await self._apply_posters(source, authors)
|
||||
|
||||
async def _apply_posters(self, source: Source, authors: list[dict]) -> dict:
|
||||
"""Ids are what the list keys on — a username can change on a whim,
|
||||
an id can't — and the name each was picked under rides beside it for
|
||||
showing the list to a person. An empty pick clears the list, which
|
||||
takes everyone's posts again."""
|
||||
from .discord_ingester import AUTHORS_KEY
|
||||
picked = [a for a in authors if str(a.get("id") or "").strip()]
|
||||
co = dict(source.config_overrides or {})
|
||||
if picked:
|
||||
co[AUTHORS_KEY] = [str(a["id"]).strip() for a in picked]
|
||||
co[AUTHOR_LABELS_KEY] = {
|
||||
str(a["id"]).strip(): str(a.get("name") or a["id"]) for a in picked
|
||||
}
|
||||
else:
|
||||
co.pop(AUTHORS_KEY, None)
|
||||
co.pop(AUTHOR_LABELS_KEY, None)
|
||||
source.config_overrides = co or None
|
||||
await self.session.commit()
|
||||
return {"source_id": source.id, "discord_authors": co.get(AUTHORS_KEY, [])}
|
||||
|
||||
async def _adopt_patreon_name(self, artist, raw_slug: str, url: str) -> str | None:
|
||||
"""Rename `artist` to the Patreon display name; the old name when it
|
||||
changed, else None. Unreadable name → no rename, never the handle."""
|
||||
|
||||
@@ -479,6 +479,10 @@ class PostFeedService:
|
||||
# the UI uses this to explain why a post it linked to is not in the
|
||||
# stream.
|
||||
"absorbed_by_post_id": post.absorbed_by_post_id,
|
||||
# #4481. The Discord channel a message was posted in: both the native
|
||||
# post record and gallery-dl's sidecars carry it as `channel`. None
|
||||
# on every other platform, and on a Discord post that never said.
|
||||
"channel": _discord_channel(post, source),
|
||||
"artist": {"id": artist.id, "name": artist.name, "slug": artist.slug},
|
||||
"source": (
|
||||
{"id": source.id, "platform": source.platform}
|
||||
@@ -488,3 +492,13 @@ class PostFeedService:
|
||||
"thumbnails_more": thumbs_entry["more"],
|
||||
"attachments": atts_map.get(post.id, []),
|
||||
}
|
||||
|
||||
|
||||
def _discord_channel(post: Post, source: Source | None) -> str | None:
|
||||
if source is None or source.platform != "discord":
|
||||
return None
|
||||
raw = post.raw_metadata if isinstance(post.raw_metadata, dict) else {}
|
||||
channel = raw.get("channel")
|
||||
if not isinstance(channel, str):
|
||||
return None
|
||||
return channel.strip() or None
|
||||
|
||||
@@ -68,6 +68,9 @@ class SourceRecord:
|
||||
artist_slug: str
|
||||
platform: str
|
||||
url: str
|
||||
# alembic 0115: the name the platform gives it, where the URL is opaque
|
||||
# (a Discord link is two ids). None until a walk has read it.
|
||||
display_name: str | None
|
||||
enabled: bool
|
||||
config_overrides: dict | None
|
||||
last_checked_at: str | None
|
||||
@@ -110,6 +113,7 @@ class SourceRecord:
|
||||
"artist_slug": self.artist_slug,
|
||||
"platform": self.platform,
|
||||
"url": self.url,
|
||||
"display_name": self.display_name,
|
||||
"enabled": self.enabled,
|
||||
"config_overrides": self.config_overrides,
|
||||
"last_checked_at": self.last_checked_at,
|
||||
@@ -301,6 +305,7 @@ class SourceService:
|
||||
artist_slug=artist.slug,
|
||||
platform=source.platform,
|
||||
url=source.url,
|
||||
display_name=source.display_name,
|
||||
enabled=source.enabled,
|
||||
config_overrides=source.config_overrides,
|
||||
last_checked_at=source.last_checked_at.isoformat() if source.last_checked_at else None,
|
||||
|
||||
@@ -276,11 +276,26 @@ browser.runtime.onMessage.addListener(async (msg) => {
|
||||
artistId: msg.artistId ?? null,
|
||||
artistName: msg.artistName ?? null,
|
||||
usePlatformName: msg.usePlatformName === true,
|
||||
discordAuthors: Array.isArray(msg.discordAuthors) ? msg.discordAuthors : null,
|
||||
});
|
||||
} catch (e) {
|
||||
return { error: e.message };
|
||||
}
|
||||
|
||||
case 'DISCORD_POSTERS':
|
||||
try {
|
||||
return await api.getDiscordPosters(msg.url);
|
||||
} catch (e) {
|
||||
return { error: e.message };
|
||||
}
|
||||
|
||||
case 'SET_DISCORD_POSTERS':
|
||||
try {
|
||||
return await api.setDiscordPosters(msg.url, msg.authors || []);
|
||||
} catch (e) {
|
||||
return { error: e.message };
|
||||
}
|
||||
|
||||
case 'SEARCH_ARTISTS':
|
||||
try {
|
||||
return { artists: await api.searchArtists(msg.q || '') };
|
||||
|
||||
@@ -100,3 +100,7 @@
|
||||
/* The panel's own [hidden] — its rows are display:flex, which beats the UA's. */
|
||||
.fc-panel [hidden] { display: none !important; }
|
||||
.fc-panel__rename { margin-top: 8px; font-size: 13px; }
|
||||
/* Poster picker (#4488): who posts in the channel, the likely creator ticked. */
|
||||
.fc-panel__posters { max-height: 220px; overflow-y: auto; }
|
||||
.fc-panel__poster { flex-wrap: wrap; }
|
||||
.fc-panel__poster-meta { flex-basis: 100%; padding-left: 21px; font-size: 11px; color: rgb(170, 166, 156); }
|
||||
|
||||
@@ -81,6 +81,13 @@
|
||||
const probe = currentProbe;
|
||||
|
||||
if (probe?.state === 'source_match') {
|
||||
// A followed Discord channel opens its poster list (#4488); the artist
|
||||
// page is one click further, in that panel.
|
||||
if (probe.platform === 'discord' && probe.discord?.channel_id) {
|
||||
if (document.getElementById('fc-add-panel')) closePanel();
|
||||
else openPostersPanel(btn, probe);
|
||||
return;
|
||||
}
|
||||
await openArtist(btn, probe.artist?.slug);
|
||||
return;
|
||||
}
|
||||
@@ -158,6 +165,120 @@
|
||||
document.getElementById('fc-add-panel')?.remove();
|
||||
}
|
||||
|
||||
// ---- Poster picker (#4488) ----
|
||||
// Who posted in this channel lately, read by FabledCurator with its Discord
|
||||
// token, the creator the server is named for (or owned by) ticked. The
|
||||
// source keeps their ids: a username can change on a whim, an id can't, so
|
||||
// the list never quietly stops matching. Names are other people's text —
|
||||
// createElement only.
|
||||
|
||||
function posterSection() {
|
||||
const list = el('div', { class: 'fc-panel__posters' }, [
|
||||
el('div', { class: 'fc-panel__hint', text: 'Reading who posts here…' }),
|
||||
]);
|
||||
const node = el('div', {}, [
|
||||
el('div', { class: 'fc-panel__label', text: 'Only posts by' }),
|
||||
list,
|
||||
]);
|
||||
const state = { data: null, picked: new Set() };
|
||||
|
||||
function render() {
|
||||
const posters = withSelected(state.data);
|
||||
if (!posters.length) {
|
||||
list.replaceChildren(el('div', { class: 'fc-panel__hint', text: 'No one has posted here lately.' }));
|
||||
return;
|
||||
}
|
||||
const rows = posters.map((p) => {
|
||||
const id = String(p.id);
|
||||
const box = el('input', { type: 'checkbox', checked: state.picked.has(id) });
|
||||
box.addEventListener('change', () => {
|
||||
if (box.checked) state.picked.add(id);
|
||||
else state.picked.delete(id);
|
||||
hint.textContent = pickHint();
|
||||
});
|
||||
const bits = [`${p.images} image${p.images === 1 ? '' : 's'}`, `${p.messages} message${p.messages === 1 ? '' : 's'}`];
|
||||
if (p.reasons?.length) bits.unshift(p.reasons.join(', '));
|
||||
return el('label', { class: 'fc-panel__radio fc-panel__poster' }, [
|
||||
box,
|
||||
el('span', { text: posterLabel(p) }),
|
||||
el('span', { class: 'fc-panel__poster-meta', text: bits.join(' · ') }),
|
||||
]);
|
||||
});
|
||||
const hint = el('div', { class: 'fc-panel__hint', text: pickHint() });
|
||||
list.replaceChildren(...rows, hint);
|
||||
}
|
||||
|
||||
function pickHint() {
|
||||
return state.picked.size
|
||||
? 'Only the ticked people’s posts are taken.'
|
||||
: 'Nothing ticked: everyone’s posts are taken.';
|
||||
}
|
||||
|
||||
async function load() {
|
||||
let r;
|
||||
try {
|
||||
r = await browser.runtime.sendMessage({ type: 'DISCORD_POSTERS', url: window.location.href });
|
||||
} catch (e) {
|
||||
r = { error: e.message };
|
||||
}
|
||||
if (r?.error) {
|
||||
list.replaceChildren(el('div', { class: 'fc-panel__hint', text: `Couldn’t list posters — ${r.error}` }));
|
||||
return;
|
||||
}
|
||||
state.data = r;
|
||||
state.picked = initialPosterPick(r);
|
||||
render();
|
||||
}
|
||||
|
||||
// null until the list has loaded: an add made before then leaves the
|
||||
// source's list as it is instead of clearing it.
|
||||
const request = () => (state.data ? postersRequest(state.data.posters, state.picked) : null);
|
||||
return { node, load, request };
|
||||
}
|
||||
|
||||
function openPostersPanel(btn, probe) {
|
||||
closePanel();
|
||||
const d = probe.discord || {};
|
||||
const posters = posterSection();
|
||||
const saveBtn = el('button', { class: 'fc-panel__btn fc-panel__btn--primary', text: 'Save' });
|
||||
const artistBtn = el('button', { class: 'fc-panel__btn', text: `Open ${probe.artist?.name || 'artist'}` });
|
||||
const cancelBtn = el('button', { class: 'fc-panel__btn', text: 'Close' });
|
||||
const panel = el('div', { id: 'fc-add-panel', class: 'fc-panel' }, [
|
||||
el('div', { class: 'fc-panel__title', text: 'Discord source' }),
|
||||
el('div', { class: 'fc-panel__sub', text: d.channel_id ? channelLabel(d) : serverLabel(d) }),
|
||||
posters.node,
|
||||
el('div', { class: 'fc-panel__actions' }, [artistBtn, cancelBtn, saveBtn]),
|
||||
]);
|
||||
panel.addEventListener('keydown', (e) => {
|
||||
e.stopPropagation();
|
||||
if (e.key === 'Escape') closePanel();
|
||||
});
|
||||
cancelBtn.addEventListener('click', closePanel);
|
||||
artistBtn.addEventListener('click', () => { closePanel(); openArtist(btn, probe.artist?.slug); });
|
||||
saveBtn.addEventListener('click', async () => {
|
||||
const authors = posters.request();
|
||||
if (!authors) return;
|
||||
saveBtn.disabled = true;
|
||||
try {
|
||||
const r = await browser.runtime.sendMessage({
|
||||
type: 'SET_DISCORD_POSTERS', url: window.location.href, authors,
|
||||
});
|
||||
if (r?.error) {
|
||||
showToast(`Error: ${r.error}`, 'error');
|
||||
return;
|
||||
}
|
||||
showToast(authors.length ? `Only posts by ${authors.map((a) => a.name || a.id).join(', ')}` : 'Taking everyone’s posts', 'success');
|
||||
closePanel();
|
||||
} catch (e) {
|
||||
showToast(`Error: ${e.message}`, 'error');
|
||||
} finally {
|
||||
saveBtn.disabled = false;
|
||||
}
|
||||
});
|
||||
document.body.appendChild(panel);
|
||||
posters.load();
|
||||
}
|
||||
|
||||
async function openAddPanel(btn, probe) {
|
||||
closePanel();
|
||||
const platformName = PLATFORMS[probe.platform]?.name || probe.platform;
|
||||
@@ -184,6 +305,8 @@
|
||||
const d = probe.discord || {};
|
||||
const discord = probe.platform === 'discord';
|
||||
const choice = panelDefaults(probe, window.location.href);
|
||||
// #4488: on a channel page, pick whose posts the new source takes.
|
||||
const posters = discord && d.channel_id ? posterSection() : null;
|
||||
|
||||
const scopeRow = (value, label, disabled) => {
|
||||
const input = el('input', {
|
||||
@@ -234,6 +357,7 @@
|
||||
el('div', { class: 'fc-panel__label', text: 'Artist' }),
|
||||
el('div', { class: 'fc-panel__combo' }, [nameInput, results]),
|
||||
renameRow,
|
||||
...(posters ? [posters.node] : []),
|
||||
hint,
|
||||
el('div', { class: 'fc-panel__actions' }, [cancelBtn, addBtn]),
|
||||
]);
|
||||
@@ -392,6 +516,8 @@
|
||||
addBtn.addEventListener('click', async () => {
|
||||
const req = addRequest(choice);
|
||||
if (!req) return;
|
||||
const authors = posters?.request();
|
||||
if (authors) req.discordAuthors = authors;
|
||||
const btn = document.getElementById('fc-add-source-btn');
|
||||
addBtn.disabled = true;
|
||||
const ok = await add(btn || addBtn, req);
|
||||
@@ -402,6 +528,7 @@
|
||||
document.body.appendChild(panel);
|
||||
refresh();
|
||||
nameInput.focus();
|
||||
posters?.load();
|
||||
nameInput.select();
|
||||
// Search what the field opens with — the server's name, usually — so an
|
||||
// artist it already matches is picked before the operator types anything.
|
||||
|
||||
+13
-1
@@ -106,8 +106,11 @@ class FabledCuratorAPI {
|
||||
// artist from the URL. A Discord channel always sends one.
|
||||
// usePlatformName: a Patreon source joining an existing artist renames it
|
||||
// to the Patreon display name (Patreon is canon; name only, never the slug).
|
||||
quickAddSource(url, { artistId = null, artistName = null, usePlatformName = false } = {}) {
|
||||
// discordAuthors ([{id, name}], #4488): a Discord source's poster list, set
|
||||
// in the same step.
|
||||
quickAddSource(url, { artistId = null, artistName = null, usePlatformName = false, discordAuthors = null } = {}) {
|
||||
const body = { url };
|
||||
if (discordAuthors) body.discord_authors = discordAuthors;
|
||||
if (artistId != null) body.artist_id = artistId;
|
||||
else if (artistName) body.artist_name = artistName;
|
||||
if (usePlatformName) body.use_platform_name = true;
|
||||
@@ -123,6 +126,15 @@ class FabledCuratorAPI {
|
||||
const qs = new URLSearchParams(params).toString();
|
||||
return this.request('GET', `/extension/probe?${qs}`);
|
||||
}
|
||||
// #4488: who posts in the Discord channel `url` shows, lately, with the
|
||||
// likely creator marked; and setting the list on the source that follows it.
|
||||
getDiscordPosters(url) {
|
||||
const qs = new URLSearchParams({ url }).toString();
|
||||
return this.request('GET', `/extension/discord/posters?${qs}`);
|
||||
}
|
||||
setDiscordPosters(url, authors) {
|
||||
return this.request('POST', '/extension/discord/posters', { url, authors });
|
||||
}
|
||||
// Latest published extension version on this instance — drives the in-app
|
||||
// update prompt. Public endpoint (no key needed, but request() sends it
|
||||
// harmlessly). Returns {version, xpi_url, latest_url, sha256}.
|
||||
|
||||
@@ -144,3 +144,49 @@ function inlineCompletion(typed, results) {
|
||||
if (!t) return null;
|
||||
return (results || []).find((a) => a.name.length > t.length && a.name.toLowerCase().startsWith(t)) || null;
|
||||
}
|
||||
|
||||
// ---- Discord poster picker (#4488) ----
|
||||
// /api/extension/discord/posters lists who posted in the channel lately, with
|
||||
// the creator the server is named for (or owned by) marked `suggested`. The
|
||||
// picker ticks posters; the source keeps their ids, since a username can
|
||||
// change on a whim and an id can't.
|
||||
|
||||
/** "Todding (@todding)" — the display name, then the handle when it differs. */
|
||||
function posterLabel(p) {
|
||||
const shown = p?.global_name || p?.username || p?.id || '';
|
||||
const handle = p?.username && p.username !== shown ? ` (@${p.username})` : '';
|
||||
return `${shown}${handle}`;
|
||||
}
|
||||
|
||||
/** What the picker starts with: the source's current list when it has one,
|
||||
* else the suggested creator(s). A set of ids. */
|
||||
function initialPosterPick(data) {
|
||||
const current = (data?.selected || []).map(String);
|
||||
if (current.length) return new Set(current);
|
||||
return new Set((data?.posters || []).filter((p) => p.suggested).map((p) => String(p.id)));
|
||||
}
|
||||
|
||||
/** The checklist's rows: who posted lately, then anyone already on the
|
||||
* source's list who hasn't — by the name they were picked under — so they
|
||||
* can still be unticked. */
|
||||
function withSelected(data) {
|
||||
const posters = [...(data?.posters || [])];
|
||||
const shown = new Set(posters.map((p) => String(p.id)));
|
||||
const labels = data?.selected_labels || {};
|
||||
for (const id of (data?.selected || []).map(String)) {
|
||||
if (shown.has(id)) continue;
|
||||
posters.push({ id, global_name: labels[id] || null, username: null,
|
||||
messages: 0, images: 0, reasons: ['on the list, not posting lately'] });
|
||||
}
|
||||
return posters;
|
||||
}
|
||||
|
||||
/** The picked posters as the API takes them: [{id, name}], in list order.
|
||||
* An id already on the source but no longer posting lately is kept. */
|
||||
function postersRequest(posters, picked) {
|
||||
const byId = new Map((posters || []).map((p) => [String(p.id), p]));
|
||||
return [...picked].map((id) => ({
|
||||
id,
|
||||
name: byId.has(id) ? (byId.get(id).global_name || byId.get(id).username || id) : null,
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -198,3 +198,46 @@ describe('Add panel on Patreon and SubscribeStar', () => {
|
||||
expect(c.artistName).toBe('Tamada')
|
||||
})
|
||||
})
|
||||
|
||||
const { posterLabel, initialPosterPick, postersRequest, withSelected } = loadLib('chip.js', [
|
||||
'posterLabel', 'initialPosterPick', 'postersRequest', 'withSelected',
|
||||
])
|
||||
|
||||
describe('the poster checklist rows', () => {
|
||||
it('adds someone on the list who has not posted lately, by their saved name', () => {
|
||||
const rows = withSelected({
|
||||
posters: [{ id: '7', username: 'todding', global_name: 'Todding' }],
|
||||
selected: ['7', '42'],
|
||||
selected_labels: { 42: 'Old Name' },
|
||||
})
|
||||
expect(rows.map((r) => r.id)).toEqual(['7', '42'])
|
||||
expect(posterLabel(rows[1])).toBe('Old Name')
|
||||
expect(rows[1].reasons).toEqual(['on the list, not posting lately'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('the Discord poster picker (#4488)', () => {
|
||||
const posters = [
|
||||
{ id: '7', username: 'todding', global_name: 'Todding', suggested: true },
|
||||
{ id: '8', username: 'jakeboii', global_name: 'Jake Boii', suggested: false },
|
||||
{ id: '9', username: 'plain', global_name: null, suggested: false },
|
||||
]
|
||||
|
||||
it('labels a poster by display name, then handle', () => {
|
||||
expect(posterLabel(posters[0])).toBe('Todding (@todding)')
|
||||
expect(posterLabel(posters[2])).toBe('plain')
|
||||
})
|
||||
|
||||
it('starts from the source list when it has one, else the suggestion', () => {
|
||||
expect([...initialPosterPick({ posters, selected: [] })]).toEqual(['7'])
|
||||
expect([...initialPosterPick({ posters, selected: ['8'] })]).toEqual(['8'])
|
||||
expect([...initialPosterPick({ posters: [] })]).toEqual([])
|
||||
})
|
||||
|
||||
it('sends ids with the names they were picked under, keeping unknown ids', () => {
|
||||
expect(postersRequest(posters, new Set(['7', '42']))).toEqual([
|
||||
{ id: '7', name: 'Todding' },
|
||||
{ id: '42', name: null },
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
@@ -101,7 +101,7 @@ import MaintenanceTile from '../common/MaintenanceTile.vue'
|
||||
import { useCleanupStore } from '../../stores/cleanup.js'
|
||||
|
||||
const store = useCleanupStore()
|
||||
const threshold = ref(0.95)
|
||||
const threshold = ref(0.995)
|
||||
const tolerance = ref(30)
|
||||
const audit = ref(null)
|
||||
const busy = ref(false)
|
||||
|
||||
@@ -26,6 +26,7 @@
|
||||
:to="{ name: 'artist', params: { slug: post.artist.slug } }"
|
||||
class="fc-post-card__artist"
|
||||
>{{ post.artist.name }}</RouterLink>
|
||||
<span v-if="post.channel" class="fc-post-card__meta">#{{ post.channel }}</span>
|
||||
<span class="fc-post-card__date" :title="absoluteDate">{{ relativeDate }}</span>
|
||||
<span v-if="totalImages" class="fc-post-card__meta">
|
||||
· {{ totalImages }} image{{ totalImages === 1 ? '' : 's' }}
|
||||
|
||||
@@ -77,7 +77,7 @@
|
||||
<v-col cols="12" sm="6">
|
||||
<v-slider
|
||||
v-model="local.single_color_threshold" label="Single-color threshold"
|
||||
min="0.5" max="1" step="0.05" thumb-label hide-details color="accent"
|
||||
min="0.9" max="1" step="0.005" thumb-label hide-details color="accent"
|
||||
:disabled="!local.skip_single_color" @end="save"
|
||||
/>
|
||||
</v-col>
|
||||
@@ -142,7 +142,7 @@ const PHASH_TICKS = { 0: 'Exact', 12: 'Strict', 24: 'Default', 48: 'Loose' }
|
||||
const local = reactive({
|
||||
min_width: 0, min_height: 0,
|
||||
skip_transparent: false, transparency_threshold: 0.9,
|
||||
skip_single_color: false, single_color_threshold: 0.95,
|
||||
skip_single_color: false, single_color_threshold: 0.995,
|
||||
phash_threshold: 24,
|
||||
wip_title_tagging_enabled: true,
|
||||
})
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
<template>
|
||||
<!-- #4486. A Discord source's posts by people outside its "Only posts by"
|
||||
list, taken before the list existed. Preview first, always: nothing is
|
||||
deleted until the operator has seen the per-poster breakdown and typed
|
||||
the token that preview returned. -->
|
||||
<v-dialog :model-value="modelValue" max-width="560"
|
||||
@update:model-value="$emit('update:modelValue', $event)">
|
||||
<v-card>
|
||||
<v-card-title>Remove posts from other posters</v-card-title>
|
||||
<v-card-text>
|
||||
<div class="text-caption mb-2">{{ source.display_name || source.url }}</div>
|
||||
<v-progress-linear v-if="loading" indeterminate color="accent" class="mb-3" />
|
||||
<v-alert v-else-if="error" type="warning" variant="tonal" density="compact">
|
||||
{{ error }}
|
||||
</v-alert>
|
||||
<template v-else-if="result">
|
||||
<v-alert type="success" variant="tonal" density="compact">
|
||||
Removed {{ result.posts_deleted }} post(s), {{ result.images_deleted }} image(s),
|
||||
{{ result.attachments_deleted }} attachment(s) and {{ result.drops_deleted }}
|
||||
grouped post(s). Grouping rebuilds what remains on its next sweep.
|
||||
</v-alert>
|
||||
</template>
|
||||
<template v-else-if="preview">
|
||||
<p class="mb-2">
|
||||
Keeping posts by <strong>{{ preview.authors.join(', ') }}</strong>.
|
||||
</p>
|
||||
<p v-if="!preview.posts" class="fc-dim">
|
||||
Nothing to remove: every post with a known poster is by someone on the list.
|
||||
</p>
|
||||
<template v-else>
|
||||
<v-table density="compact" class="mb-3">
|
||||
<thead><tr><th>Poster</th><th class="text-right">Posts</th></tr></thead>
|
||||
<tbody>
|
||||
<tr v-for="p in preview.posters" :key="p.poster">
|
||||
<td>{{ p.poster }}</td>
|
||||
<td class="text-right">{{ p.posts }}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</v-table>
|
||||
<p class="mb-1">
|
||||
{{ preview.posts }} post(s), {{ preview.images }} image(s) found only on them,
|
||||
{{ preview.attachments }} attachment(s).
|
||||
</p>
|
||||
<p v-if="preview.drops" class="mb-1">
|
||||
{{ preview.drops }} grouped post(s) that include them are ungrouped; the
|
||||
rest regroup on the next sweep.
|
||||
</p>
|
||||
<p class="fc-dim">
|
||||
Images that also belong to a kept post stay. If a name you want to keep
|
||||
appears above, add it to the source's "Only posts by" list first.
|
||||
</p>
|
||||
</template>
|
||||
<p v-if="preview.unknown_posts" class="fc-dim mt-2">
|
||||
{{ preview.unknown_posts }} older post(s) don't record who posted them and
|
||||
are left alone.
|
||||
</p>
|
||||
</template>
|
||||
</v-card-text>
|
||||
<v-card-actions>
|
||||
<v-spacer />
|
||||
<v-btn variant="text" @click="$emit('update:modelValue', false)">
|
||||
{{ result ? 'Close' : 'Cancel' }}
|
||||
</v-btn>
|
||||
<v-btn
|
||||
v-if="preview && preview.posts && !result"
|
||||
color="error" variant="flat" :loading="busy"
|
||||
@click="confirmOpen = true"
|
||||
>Remove {{ preview.posts }} post(s)</v-btn>
|
||||
</v-card-actions>
|
||||
</v-card>
|
||||
<DestructiveConfirmModal
|
||||
v-if="preview"
|
||||
v-model="confirmOpen"
|
||||
action="delete" kind="other posters' posts" tier="C"
|
||||
:expected-token-override="preview.confirm_token"
|
||||
:projected-counts="{ posts: preview.posts, images: preview.images,
|
||||
attachments: preview.attachments, grouped: preview.drops }"
|
||||
@confirm="onConfirm"
|
||||
/>
|
||||
</v-dialog>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { ref, watch } from 'vue'
|
||||
import { useSourcesStore } from '../../stores/sources.js'
|
||||
import DestructiveConfirmModal from '../modal/DestructiveConfirmModal.vue'
|
||||
|
||||
const props = defineProps({
|
||||
modelValue: { type: Boolean, default: false },
|
||||
source: { type: Object, required: true },
|
||||
})
|
||||
defineEmits(['update:modelValue'])
|
||||
const store = useSourcesStore()
|
||||
|
||||
const loading = ref(false)
|
||||
const busy = ref(false)
|
||||
const error = ref('')
|
||||
const preview = ref(null)
|
||||
const result = ref(null)
|
||||
const confirmOpen = ref(false)
|
||||
|
||||
function message(e) {
|
||||
return e?.body?.detail || e?.body?.error || e?.message || String(e)
|
||||
}
|
||||
|
||||
watch(() => props.modelValue, async (open) => {
|
||||
if (!open) return
|
||||
preview.value = null
|
||||
result.value = null
|
||||
error.value = ''
|
||||
loading.value = true
|
||||
try {
|
||||
preview.value = await store.previewOtherPosters(props.source.id)
|
||||
} catch (e) {
|
||||
error.value = message(e)
|
||||
} finally {
|
||||
loading.value = false
|
||||
}
|
||||
})
|
||||
|
||||
async function onConfirm(token) {
|
||||
busy.value = true
|
||||
try {
|
||||
result.value = await store.removeOtherPosters(props.source.id, token)
|
||||
} catch (e) {
|
||||
error.value = message(e)
|
||||
} finally {
|
||||
busy.value = false
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-dim { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||
</style>
|
||||
@@ -50,6 +50,17 @@
|
||||
library — without re-downloading media
|
||||
</v-list-item-subtitle>
|
||||
</v-list-item>
|
||||
<v-list-item
|
||||
v-if="hasPosterList && !running"
|
||||
prepend-icon="mdi-account-remove-outline"
|
||||
@click="cleanupOpen = true"
|
||||
>
|
||||
<v-list-item-title>Remove posts from other posters</v-list-item-title>
|
||||
<v-list-item-subtitle>
|
||||
Preview, then delete the posts taken from people outside this
|
||||
source's "Only posts by" list
|
||||
</v-list-item-subtitle>
|
||||
</v-list-item>
|
||||
<v-divider v-if="isNative && !running" />
|
||||
<v-list-item
|
||||
base-color="error"
|
||||
@@ -59,12 +70,14 @@
|
||||
<v-list-item-title>Remove source</v-list-item-title>
|
||||
</v-list-item>
|
||||
</KebabMenu>
|
||||
<PosterCleanupDialog v-if="hasPosterList" v-model="cleanupOpen" :source="source" />
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed } from 'vue'
|
||||
import { computed, ref } from 'vue'
|
||||
import KebabMenu from '../common/KebabMenu.vue'
|
||||
import PosterCleanupDialog from './PosterCleanupDialog.vue'
|
||||
|
||||
const props = defineProps({
|
||||
source: { type: Object, required: true },
|
||||
@@ -80,6 +93,10 @@ const recapturing = computed(() => !!props.source.backfill_recapture)
|
||||
// which those are (`native_ingester`); a copied list here went stale when
|
||||
// Discord moved over.
|
||||
const isNative = computed(() => !!props.source.native_ingester)
|
||||
// #4486: only a Discord source with an "Only posts by" list has other posters.
|
||||
const hasPosterList = computed(() => props.source.platform === 'discord'
|
||||
&& (props.source.config_overrides?.discord_authors || []).length > 0)
|
||||
const cleanupOpen = ref(false)
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
<a
|
||||
:href="source.url" target="_blank" rel="noopener"
|
||||
class="fc-source-card__url" @click.stop
|
||||
>{{ source.url }}</a>
|
||||
>{{ source.display_name || source.url }}</a>
|
||||
<v-btn
|
||||
icon="mdi-pencil" size="x-small" variant="text"
|
||||
@click.stop="$emit('edit', source)"
|
||||
@@ -39,18 +39,18 @@
|
||||
</v-chip>
|
||||
<v-chip
|
||||
v-else-if="source.backfill_state === 'running'"
|
||||
size="x-small" color="info" variant="tonal" label
|
||||
size="x-small" color="info" variant="flat" label
|
||||
>{{ source.backfill_bypass_seen ? 'Recovering' : (source.backfill_recapture ? 'Recapturing' : 'Backfilling')
|
||||
}}{{ source.backfill_posts
|
||||
? ` · ${source.backfill_posts} posts`
|
||||
: (source.backfill_chunks ? ` (${source.backfill_chunks})` : '') }}</v-chip>
|
||||
<v-chip
|
||||
v-else-if="source.backfill_state === 'complete'"
|
||||
size="x-small" color="success" variant="tonal" label
|
||||
size="x-small" color="success" variant="flat" label
|
||||
>Backfilled</v-chip>
|
||||
<v-chip
|
||||
v-else-if="source.backfill_state === 'stalled'"
|
||||
size="x-small" color="warning" variant="tonal" label
|
||||
size="x-small" color="warning" variant="flat" label
|
||||
>Stalled</v-chip>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -44,6 +44,17 @@
|
||||
v-model="structuredSince" label="Skip posts older than (YYYY-MM-DD)"
|
||||
placeholder="2024-01-01" hide-details class="mt-2"
|
||||
/>
|
||||
<!-- A creator's server is full of other members posting their own
|
||||
pictures; the source is subscribed to the creator (#4481). -->
|
||||
<v-text-field
|
||||
v-if="platform === 'discord'"
|
||||
v-model="structuredAuthors" label="Only posts by (Discord names or ids)"
|
||||
placeholder="Todding" hint="Comma-separated. Empty takes everyone's."
|
||||
persistent-hint class="mt-2"
|
||||
/>
|
||||
<div v-if="platform === 'discord' && authorNames.length" class="text-caption mt-2">
|
||||
{{ authorNames.join(', ') }}
|
||||
</div>
|
||||
<p class="text-caption mt-2" style="opacity: 0.75">
|
||||
More per-platform fields land here over time. Use Advanced JSON for everything else.
|
||||
</p>
|
||||
@@ -98,6 +109,33 @@ const urlError = ref('')
|
||||
const configTab = ref('structured')
|
||||
const structuredVideos = ref(true)
|
||||
const structuredSince = ref('')
|
||||
const structuredAuthors = ref('')
|
||||
// Keys the structured view has no field for, carried through its saves so
|
||||
// switching tabs never drops what the JSON view set.
|
||||
const otherConfig = ref({})
|
||||
const STRUCTURED_KEYS = ['videos', 'since', 'discord_authors']
|
||||
|
||||
// The extension's poster picker stores ids (#4488) and the name each was
|
||||
// picked under, so a list of numbers still says who it means.
|
||||
const authorNames = computed(() => {
|
||||
const labels = otherConfig.value.discord_author_labels || {}
|
||||
return splitAuthors(structuredAuthors.value)
|
||||
.map(a => (labels[a] ? `${labels[a]} (${a})` : null))
|
||||
.filter(Boolean)
|
||||
})
|
||||
|
||||
function splitAuthors(txt) {
|
||||
return (txt || '').split(',').map(s => s.trim()).filter(Boolean)
|
||||
}
|
||||
|
||||
function takeConfig(co) {
|
||||
structuredVideos.value = co.videos !== false
|
||||
structuredSince.value = co.since ?? ''
|
||||
structuredAuthors.value = Array.isArray(co.discord_authors) ? co.discord_authors.join(', ') : ''
|
||||
otherConfig.value = Object.fromEntries(
|
||||
Object.entries(co).filter(([k]) => !STRUCTURED_KEYS.includes(k)),
|
||||
)
|
||||
}
|
||||
const jsonText = ref('{}')
|
||||
const jsonError = ref('')
|
||||
|
||||
@@ -105,13 +143,15 @@ const busy = ref(false)
|
||||
|
||||
// Sync config_overrides between the two views.
|
||||
const config = computed(() => {
|
||||
const out = {}
|
||||
const out = { ...otherConfig.value }
|
||||
if (!structuredVideos.value) out.videos = false
|
||||
if (structuredSince.value) out.since = structuredSince.value
|
||||
const authors = splitAuthors(structuredAuthors.value)
|
||||
if (authors.length) out.discord_authors = authors
|
||||
return out
|
||||
})
|
||||
|
||||
watch([structuredVideos, structuredSince], () => {
|
||||
watch([structuredVideos, structuredSince, structuredAuthors], () => {
|
||||
if (configTab.value === 'structured') {
|
||||
jsonText.value = JSON.stringify(config.value, null, 2)
|
||||
jsonError.value = ''
|
||||
@@ -128,8 +168,7 @@ watch(jsonText, (txt) => {
|
||||
}
|
||||
jsonError.value = ''
|
||||
// Reflect recognized keys into the structured view.
|
||||
structuredVideos.value = parsed.videos !== false
|
||||
structuredSince.value = parsed.since ?? ''
|
||||
takeConfig(parsed)
|
||||
} catch {
|
||||
jsonError.value = 'Invalid JSON'
|
||||
}
|
||||
@@ -146,14 +185,13 @@ watch(() => props.modelValue, async (open) => {
|
||||
url.value = props.source.url
|
||||
enabled.value = props.source.enabled
|
||||
const co = props.source.config_overrides || {}
|
||||
structuredVideos.value = co.videos !== false
|
||||
structuredSince.value = co.since ?? ''
|
||||
takeConfig(co)
|
||||
jsonText.value = JSON.stringify(co, null, 2)
|
||||
artistChoice.value = { id: props.source.artist_id, name: props.source.artist_name }
|
||||
} else {
|
||||
platform.value = platformsStore.list[0]?.key || 'patreon'
|
||||
url.value = ''; enabled.value = true
|
||||
structuredVideos.value = true; structuredSince.value = ''
|
||||
takeConfig({})
|
||||
jsonText.value = '{}'
|
||||
artistChoice.value = props.initialArtist
|
||||
? { id: props.initialArtist.id, name: props.initialArtist.name }
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
<div class="fc-source-row__url-wrap">
|
||||
<a :href="source.url" target="_blank" rel="noopener" class="fc-source-row__url"
|
||||
@click.stop>
|
||||
{{ source.url }}
|
||||
{{ source.display_name || source.url }}
|
||||
</a>
|
||||
<!-- Edit sits next to the source identity (operator-requested), not in
|
||||
the action cluster where it was easy to fat-finger Remove. -->
|
||||
@@ -72,7 +72,7 @@
|
||||
worked, the content just isn't ours. -->
|
||||
<v-chip
|
||||
v-else-if="source.error_type === 'tier_limited'"
|
||||
size="x-small" color="info" variant="tonal" label
|
||||
size="x-small" color="info" variant="flat" label
|
||||
prepend-icon="mdi-lock-outline"
|
||||
>{{ source.tier_gated_count ? `${source.tier_gated_count} gated` : 'No access' }}
|
||||
<v-tooltip activator="parent" location="top" max-width="480">
|
||||
@@ -81,18 +81,18 @@
|
||||
</v-chip>
|
||||
<v-chip
|
||||
v-else-if="source.backfill_state === 'running'"
|
||||
size="x-small" color="info" variant="tonal" label
|
||||
size="x-small" color="info" variant="flat" label
|
||||
>{{ source.backfill_bypass_seen ? 'Recovering' : (source.backfill_recapture ? 'Recapturing' : 'Backfilling')
|
||||
}}{{ source.backfill_posts
|
||||
? ` · ${source.backfill_posts} posts`
|
||||
: (source.backfill_chunks ? ` (${source.backfill_chunks})` : '') }}</v-chip>
|
||||
<v-chip
|
||||
v-else-if="source.backfill_state === 'complete'"
|
||||
size="x-small" color="success" variant="tonal" label
|
||||
size="x-small" color="success" variant="flat" label
|
||||
>Backfilled</v-chip>
|
||||
<v-chip
|
||||
v-else-if="source.backfill_state === 'stalled'"
|
||||
size="x-small" color="warning" variant="tonal" label
|
||||
size="x-small" color="warning" variant="flat" label
|
||||
>Stalled</v-chip>
|
||||
<span v-else class="fc-source-row__zero">0</span>
|
||||
</td>
|
||||
|
||||
@@ -108,7 +108,7 @@
|
||||
v-if="item.singleSource"
|
||||
:href="item.singleSource.url" target="_blank" rel="noopener"
|
||||
class="fc-subs__sub-url" @click.stop
|
||||
>{{ item.singleSource.url }}</a>
|
||||
>{{ item.singleSource.display_name || item.singleSource.url }}</a>
|
||||
</template>
|
||||
|
||||
<template #item.platforms="{ item }">
|
||||
@@ -517,6 +517,7 @@ const filteredGroups = computed(() => {
|
||||
g.sources.some(
|
||||
(s) =>
|
||||
(s.url || '').toLowerCase().includes(q) ||
|
||||
(s.display_name || '').toLowerCase().includes(q) ||
|
||||
(s.platform || '').toLowerCase().includes(q),
|
||||
),
|
||||
)
|
||||
|
||||
@@ -12,7 +12,7 @@ export const useCleanupStore = defineStore('cleanup', () => {
|
||||
min_width: 0,
|
||||
min_height: 0,
|
||||
transparency_threshold: 0.9,
|
||||
single_color_threshold: 0.95,
|
||||
single_color_threshold: 0.995,
|
||||
single_color_tolerance: 30,
|
||||
})
|
||||
|
||||
@@ -24,7 +24,7 @@ export const useCleanupStore = defineStore('cleanup', () => {
|
||||
min_width: s.min_width ?? 0,
|
||||
min_height: s.min_height ?? 0,
|
||||
transparency_threshold: s.transparency_threshold ?? 0.9,
|
||||
single_color_threshold: s.single_color_threshold ?? 0.95,
|
||||
single_color_threshold: s.single_color_threshold ?? 0.995,
|
||||
single_color_tolerance: s.single_color_tolerance ?? 30,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -149,6 +149,15 @@ export const useSourcesStore = defineStore('sources', () => {
|
||||
return body
|
||||
}
|
||||
|
||||
// #4486: a Discord source's posts by people outside its poster list.
|
||||
// The preview touches nothing and hands back the token the remove must echo.
|
||||
async function previewOtherPosters(id) {
|
||||
return api.get(`/api/sources/${id}/discord/other-posters`)
|
||||
}
|
||||
async function removeOtherPosters(id, confirm) {
|
||||
return api.post(`/api/sources/${id}/discord/other-posters/remove`, { body: { confirm } })
|
||||
}
|
||||
|
||||
function sourcesByArtistGrouped() {
|
||||
// returns [{artist: {id,name,slug}, sources: [...]}, ...]
|
||||
const arr = byArtist.value.get(null) ?? []
|
||||
@@ -179,6 +188,7 @@ export const useSourcesStore = defineStore('sources', () => {
|
||||
stopBackfill,
|
||||
recoverSource,
|
||||
recaptureSource,
|
||||
previewOtherPosters, removeOtherPosters,
|
||||
findOrCreateArtist, autocompleteArtist, reassign,
|
||||
loadScheduleStatus,
|
||||
sourcesByArtistGrouped,
|
||||
|
||||
@@ -40,3 +40,48 @@ def test_single_color_evaluate_handles_rgba_input():
|
||||
# Alpha channel should be ignored — only RGB matters for the rule.
|
||||
im = Image.new("RGBA", (50, 50), (100, 100, 100, 128))
|
||||
assert single_color.evaluate(im, threshold=0.9, tolerance=10) is True
|
||||
|
||||
|
||||
# -- line art is a drawing, not a blank (#4483) ---------------------------------
|
||||
|
||||
_T = 0.995 # the default since alembic 0116
|
||||
_TOL = 30
|
||||
|
||||
|
||||
def _doodle(size=(1500, 2000), lines=12):
|
||||
"""Pencil line art on white: a few thin dark strokes plus faint
|
||||
construction lines — a couple of percent ink, the rest paper."""
|
||||
from PIL import ImageDraw
|
||||
|
||||
im = Image.new("RGB", size, (255, 255, 255))
|
||||
draw = ImageDraw.Draw(im)
|
||||
w, h = size
|
||||
for i in range(lines):
|
||||
x = (i + 1) * w // (lines + 1)
|
||||
draw.line([(x, 0), (w - x, h)], fill=(70, 60, 60), width=3)
|
||||
draw.line([(0, (i + 1) * h // (lines + 1)), (w, h - x)], fill=(235, 235, 235), width=2)
|
||||
return im
|
||||
|
||||
|
||||
def test_line_art_on_white_is_not_single_color():
|
||||
"""Five of Todding's Discord doodles were skipped as blank: a smoothing
|
||||
64px thumbnail blended the strokes into the paper."""
|
||||
assert single_color.evaluate(_doodle(), threshold=_T, tolerance=_TOL) is False
|
||||
|
||||
|
||||
def test_a_blank_page_with_a_small_mark_is_still_single_color():
|
||||
from PIL import ImageDraw
|
||||
|
||||
im = Image.new("RGB", (1000, 1000), (250, 250, 250))
|
||||
ImageDraw.Draw(im).rectangle([480, 480, 520, 520], fill=(40, 40, 40))
|
||||
assert single_color.evaluate(im, threshold=_T, tolerance=_TOL) is True
|
||||
|
||||
|
||||
def test_a_noisy_solid_fill_is_still_single_color():
|
||||
"""JPEG noise on a placeholder stays inside the tolerance."""
|
||||
im = Image.new("RGB", (600, 600), (30, 90, 160))
|
||||
for i in range(0, 600 * 600, 7):
|
||||
x, y = i % 600, i // 600
|
||||
d = (i % 21) - 10
|
||||
im.putpixel((x, y), (30 + d, 90 + d, 160 + d))
|
||||
assert single_color.evaluate(im, threshold=_T, tolerance=_TOL) is True
|
||||
|
||||
@@ -153,9 +153,11 @@ def test_an_embeds_identity_ignores_its_signature():
|
||||
def embed(sig):
|
||||
return {"type": "image", "image": {"proxy_url": f"https://media/p/x.png?ex={sig}"}}
|
||||
|
||||
one = DiscordClient.extract_media(_msg(9, embeds=[embed("a")]))[0].media_id
|
||||
two = DiscordClient.extract_media(_msg(9, embeds=[embed("b")]))[0].media_id
|
||||
assert one == two and len(one) <= 33
|
||||
art = [{"url": "https://cdn/a/1.png"}]
|
||||
one = DiscordClient.extract_media(_msg(9, attachments=art, embeds=[embed("a")]))[1]
|
||||
two = DiscordClient.extract_media(_msg(9, attachments=art, embeds=[embed("b")]))[1]
|
||||
assert one.kind == two.kind == "embed"
|
||||
assert one.media_id == two.media_id and len(one.media_id) <= 33
|
||||
|
||||
|
||||
def test_post_seams():
|
||||
@@ -165,11 +167,52 @@ def test_post_seams():
|
||||
|
||||
|
||||
def test_a_text_only_message_is_not_a_post():
|
||||
"""gallery-dl never made one: chat lines would bury the drops."""
|
||||
"""Chat lines would bury the drops."""
|
||||
assert DiscordClient.post_record_key(_msg(6, content="brb")) is None
|
||||
assert DiscordClient.post_meta(_msg(1))["date"].startswith("2026-09-20")
|
||||
|
||||
|
||||
def test_a_message_without_an_attached_image_is_chat():
|
||||
"""Operator 2026-09-28: only content with an image attached. A lone
|
||||
archive, a link preview or a Tenor GIF takes nothing and records nothing."""
|
||||
chat = [
|
||||
_msg(1, content="the stash", attachments=[
|
||||
{"url": "https://cdn/a/Links_Stash.rar", "content_type": "application/x-rar"},
|
||||
]),
|
||||
_msg(2, content="look", embeds=[
|
||||
{"type": "image", "image": {"proxy_url": "https://media/p/x.png"}},
|
||||
]),
|
||||
_msg(3, content="lol", embeds=[
|
||||
{"type": "gifv", "video": {"proxy_url": "https://media/p/t.mp4"}},
|
||||
]),
|
||||
_msg(4, content="wip", attachments=[{"url": "https://cdn/a/wip.psd"}]),
|
||||
]
|
||||
for m in chat:
|
||||
assert DiscordClient.extract_media(m) == []
|
||||
assert DiscordClient.post_record_key(m) is None
|
||||
|
||||
|
||||
def test_an_attached_image_takes_every_file_numbered_as_gallery_dl_does():
|
||||
"""The gate decides WHETHER a message is taken, never which of its files:
|
||||
the rar beside the image keeps its number, so on-disk names still match."""
|
||||
m = _msg(7, attachments=[
|
||||
{"url": "https://cdn/a/pack.rar"},
|
||||
{"url": "https://cdn/a/noext", "content_type": "image/png"},
|
||||
])
|
||||
items = DiscordClient.extract_media(m)
|
||||
assert [(i.num, i.filename) for i in items] == [(1, "pack"), (2, "noext")]
|
||||
assert DiscordClient.post_record_key(m) == ("message:7", "7")
|
||||
video = _msg(8, attachments=[{"url": "https://cdn/a/clip.MP4?ex=1"}])
|
||||
assert len(DiscordClient.extract_media(video)) == 1
|
||||
|
||||
|
||||
def test_a_forwarded_image_counts_as_attached():
|
||||
m = _msg(9, message_snapshots=[
|
||||
{"message": {"type": 0, "attachments": [{"url": "https://cdn/a/fwd.png"}]}},
|
||||
])
|
||||
assert DiscordClient.post_record_key(m) == ("message:9", "9")
|
||||
|
||||
|
||||
# -- the walk --------------------------------------------------------------------
|
||||
|
||||
def test_a_channel_pages_newest_first_and_skips_system_messages():
|
||||
@@ -206,6 +249,50 @@ def test_messages_carry_server_and_channel_metadata():
|
||||
assert meta["parent"] == "Art"
|
||||
|
||||
|
||||
def _one_channel(messages):
|
||||
return {
|
||||
"/guilds/1": _ok({"id": "1", "name": "Todding's Server"}),
|
||||
"/guilds/1/channels": _ok([
|
||||
{"id": "4", "type": 4, "name": "Todding"},
|
||||
{"id": "2", "type": 0, "name": "banana-land", "parent_id": "4"},
|
||||
]),
|
||||
("/channels/2/messages", None): _ok(messages),
|
||||
("/channels/2/threads/search", 0): _ok({"threads": []}),
|
||||
}
|
||||
|
||||
|
||||
def test_only_from_takes_the_creators_messages_and_no_one_elses():
|
||||
"""Operator 2026-09-28: in the creator's server, another member posting a
|
||||
meme is not the creator's art. Matched by id, username or display name."""
|
||||
creator = {"id": "7", "username": "todding", "global_name": "Todding"}
|
||||
other = {"id": "8", "username": "jakeboii", "global_name": "Jake Boii"}
|
||||
msgs = [_msg(3, author=creator), _msg(2, author=other), _msg(1, author=creator)]
|
||||
url = "https://discord.com/channels/1/2"
|
||||
|
||||
everyone = _client(_one_channel([dict(m) for m in msgs]))
|
||||
assert [mid for mid, _ in _ids(everyone, url)] == ["3", "2", "1"]
|
||||
for who in (["Todding"], ["TODDING "], ["7"]):
|
||||
client = _client(_one_channel([dict(m) for m in msgs]))
|
||||
client.only_from(who)
|
||||
assert [mid for mid, _ in _ids(client, url)] == ["3", "1"]
|
||||
client = _client(_one_channel([dict(m) for m in msgs]))
|
||||
client.only_from([])
|
||||
assert len(_ids(client, url)) == 3
|
||||
|
||||
|
||||
def test_the_source_label_names_the_server_and_channel_the_walk_read():
|
||||
client = _client(_one_channel([]))
|
||||
url = "https://discord.com/channels/1/2"
|
||||
assert client.source_label(url) is None # nothing walked yet
|
||||
list(client.iter_posts(url))
|
||||
assert client.source_label(url) == "Todding's Server · #banana-land"
|
||||
assert client.source_label("https://discord.com/channels/1") == "Todding's Server"
|
||||
client._channels["9"] = {"channel": "wip", "is_thread": True, "parent": "banana-land"}
|
||||
assert client.source_label("https://discord.com/channels/1/9") == (
|
||||
"Todding's Server · #banana-land › wip"
|
||||
)
|
||||
|
||||
|
||||
def test_a_server_walks_text_then_threads_newest_created_first_and_skips_private():
|
||||
routes = {
|
||||
"/guilds/1": _ok({"id": "1", "name": "S"}),
|
||||
@@ -337,3 +424,60 @@ def test_the_adapter_authenticates_with_the_token_and_keys_by_identity(tmp_path)
|
||||
_msg(9, attachments=[{"id": "300", "url": "https://cdn/a/1.png"}])
|
||||
)
|
||||
assert _ledger_key(media) == "9:300"
|
||||
|
||||
|
||||
# -- the poster picker (#4488) -------------------------------------------------
|
||||
|
||||
def _person(pid, username, global_name=None, **extra):
|
||||
return {"id": pid, "username": username, "global_name": global_name,
|
||||
"messages": 1, "images": 0, **extra}
|
||||
|
||||
|
||||
def test_the_creator_a_server_is_named_for_is_suggested():
|
||||
ranked = dc.rank_posters("Todding's Server", "99", [
|
||||
_person("8", "jakeboii", "Jake Boii", images=3),
|
||||
_person("7", "todding", "Todding", images=1),
|
||||
])
|
||||
assert ranked[0]["id"] == "7" and ranked[0]["suggested"] is True
|
||||
assert ranked[0]["reasons"] == ["name matches the server"]
|
||||
assert ranked[1]["suggested"] is False
|
||||
|
||||
|
||||
def test_owning_the_server_and_matching_its_name_outranks_either_alone():
|
||||
ranked = dc.rank_posters("The Official Todding Discord", "7", [
|
||||
_person("5", "toddingfan"),
|
||||
_person("7", "t0dd", "Todding"),
|
||||
])
|
||||
assert [r["id"] for r in ranked if r["suggested"]] == ["7"]
|
||||
assert ranked[0]["reasons"] == ["owns the server", "name matches the server"]
|
||||
|
||||
|
||||
def test_no_signal_suggests_no_one_and_bots_never():
|
||||
assert not any(r["suggested"] for r in dc.rank_posters("Art Club", None, [
|
||||
_person("1", "alice"), _person("2", "bob"),
|
||||
]))
|
||||
ranked = dc.rank_posters("Todding's Server", None, [_person("3", "toddingbot", bot=True)])
|
||||
assert ranked[0]["suggested"] is False
|
||||
|
||||
|
||||
def test_recent_posters_tallies_a_shallow_window_by_id():
|
||||
creator = {"id": "7", "username": "todding", "global_name": "Todding"}
|
||||
other = {"id": "8", "username": "jakeboii", "global_name": "Jake Boii"}
|
||||
art = [{"url": "https://cdn/a/1.png"}]
|
||||
page = [_msg(i, author=creator, attachments=art) for i in range(300, 240, -1)]
|
||||
page += [_msg(i, author=other) for i in range(240, 200, -1)]
|
||||
routes = {
|
||||
"/guilds/1": _ok({"id": "1", "name": "Todding's Server", "owner_id": "7"}),
|
||||
("/channels/2/messages", None): _ok(page),
|
||||
("/channels/2/messages", "201"): _ok([_msg(150, author=other)]),
|
||||
}
|
||||
client = _client(routes)
|
||||
got = client.recent_posters("1", "2", max_messages=150)
|
||||
assert got["server"] == "Todding's Server" and got["owner_id"] == "7"
|
||||
assert got["scanned"] == 101
|
||||
first = got["posters"][0]
|
||||
assert (first["id"], first["messages"], first["images"], first["suggested"]) == ("7", 60, 60, True)
|
||||
assert got["posters"][1]["messages"] == 41
|
||||
# It asked for no more than the window: 100, then the 50 left of 150.
|
||||
limits = [p.get("limit") for e, p in client._session.calls if e.endswith("/messages")]
|
||||
assert limits == [100, 50]
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
"""Removing a Discord source's posts by people outside its poster list (#4486).
|
||||
|
||||
Real Postgres (db_sync): the predicates read `raw_metadata->>'author'`, which
|
||||
only the database can evaluate. The test that matters runs preview and apply
|
||||
on one fixture and asserts they agree AND that the rows went (snippet #3087).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from backend.app.models import (
|
||||
Artist,
|
||||
ImageProvenance,
|
||||
ImageRecord,
|
||||
Post,
|
||||
PostAttachment,
|
||||
Source,
|
||||
)
|
||||
from backend.app.services import discord_poster_cleanup as cleanup
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
def _image(db, artist, tmp_path, name, post):
|
||||
f = tmp_path / f"{name}.png"
|
||||
f.write_bytes(b"x")
|
||||
img = ImageRecord(
|
||||
artist_id=artist.id, path=str(f), sha256=name.ljust(64, "0"),
|
||||
size_bytes=1, mime="image/png", origin="downloaded", primary_post_id=post.id,
|
||||
)
|
||||
db.add(img)
|
||||
db.flush()
|
||||
return img
|
||||
|
||||
|
||||
def _post(db, source, mid, raw, **extra):
|
||||
p = Post(
|
||||
source_id=source.id, artist_id=source.artist_id, external_post_id=mid,
|
||||
raw_metadata=raw, **extra,
|
||||
)
|
||||
db.add(p)
|
||||
db.flush()
|
||||
return p
|
||||
|
||||
|
||||
def _fixture(db, tmp_path, authors=("Todding",)):
|
||||
"""Todding's post, two of Jake's (one sharing an image with Todding's), an
|
||||
old post with no poster, and a drop that absorbed Todding's and Jake's."""
|
||||
artist = Artist(name="Todding", slug="todding-cleanup")
|
||||
db.add(artist)
|
||||
db.flush()
|
||||
src = Source(
|
||||
artist_id=artist.id, platform="discord",
|
||||
url="https://discord.com/channels/1/2",
|
||||
config_overrides={"discord_authors": list(authors)} if authors else None,
|
||||
)
|
||||
db.add(src)
|
||||
db.flush()
|
||||
todd = _post(db, src, "10", {"author": "todding", "author_name": "Todding", "author_id": "1"})
|
||||
jake1 = _post(db, src, "11", {"author": "jakeboii", "author_id": "8"})
|
||||
jake2 = _post(db, src, "12", {"author": "jakeboii", "author_id": "8"})
|
||||
old = _post(db, src, "13", {"category": "discord"})
|
||||
drop = _post(db, src, "fc-drop:10", None, synthesized_by="discord_drop")
|
||||
todd.absorbed_by_post_id = drop.id
|
||||
jake1.absorbed_by_post_id = drop.id
|
||||
|
||||
art = _image(db, artist, tmp_path, "art", todd)
|
||||
meme = _image(db, artist, tmp_path, "meme", jake1)
|
||||
shared = _image(db, artist, tmp_path, "shared", jake2)
|
||||
legacy = _image(db, artist, tmp_path, "legacy", old)
|
||||
db.add_all([
|
||||
ImageProvenance(image_record_id=shared.id, post_id=todd.id, source_id=src.id),
|
||||
ImageProvenance(image_record_id=art.id, post_id=drop.id, source_id=src.id),
|
||||
ImageProvenance(image_record_id=meme.id, post_id=drop.id, source_id=src.id),
|
||||
PostAttachment(
|
||||
post_id=jake2.id, artist_id=artist.id, sha256="z" * 64,
|
||||
path="/store/z/f.zip", original_filename="f.zip", ext=".zip", size_bytes=1,
|
||||
),
|
||||
])
|
||||
db.commit()
|
||||
return src, {
|
||||
"todd": todd.id, "jake1": jake1.id, "jake2": jake2.id, "old": old.id,
|
||||
"drop": drop.id, "art": art.id, "meme": meme.id, "shared": shared.id,
|
||||
"legacy": legacy.id,
|
||||
}
|
||||
|
||||
|
||||
def test_preview_matches_apply_and_the_rows_go(db_sync, tmp_path):
|
||||
src, ids = _fixture(db_sync, tmp_path)
|
||||
|
||||
projected = cleanup.preview(db_sync, source_id=src.id)
|
||||
assert projected["authors"] == ["todding"]
|
||||
assert projected["posters"] == [{"poster": "jakeboii", "posts": 2}]
|
||||
assert projected["unknown_posts"] == 1
|
||||
|
||||
done = cleanup.apply(db_sync, source_id=src.id, images_root=tmp_path)
|
||||
assert projected["posts"] == done["posts_deleted"] == 2
|
||||
assert projected["images"] == done["images_deleted"] == 1 # the meme only
|
||||
assert projected["attachments"] == done["attachments_deleted"] == 1
|
||||
assert projected["drops"] == done["drops_deleted"] == 1
|
||||
|
||||
post_ids = set(db_sync.execute(select(Post.id).where(Post.source_id == src.id)).scalars())
|
||||
assert post_ids == {ids["todd"], ids["old"]}
|
||||
image_ids = set(db_sync.execute(
|
||||
select(ImageRecord.id).where(ImageRecord.id.in_(
|
||||
[ids["art"], ids["meme"], ids["shared"], ids["legacy"]]))
|
||||
).scalars())
|
||||
# The meme was only Jake's. The shared piece is also on Todding's post, and
|
||||
# the drop's link to the art was never what kept it.
|
||||
assert image_ids == {ids["art"], ids["shared"], ids["legacy"]}
|
||||
assert not (tmp_path / "meme.png").exists()
|
||||
# Todding's message is back in the feed, for the sweep to regroup.
|
||||
assert db_sync.get(Post, ids["todd"]).absorbed_by_post_id is None
|
||||
assert db_sync.execute(select(func.count(PostAttachment.id)).where(
|
||||
PostAttachment.sha256 == "z" * 64)).scalar_one() == 0
|
||||
|
||||
|
||||
def test_a_poster_matches_by_display_name_or_id_too(db_sync, tmp_path):
|
||||
src, _ = _fixture(db_sync, tmp_path, authors=("8",))
|
||||
projected = cleanup.preview(db_sync, source_id=src.id)
|
||||
assert projected["posters"] == [{"poster": "Todding", "posts": 1}]
|
||||
|
||||
|
||||
def test_no_list_means_nothing_to_remove(db_sync, tmp_path):
|
||||
src, _ = _fixture(db_sync, tmp_path, authors=())
|
||||
with pytest.raises(cleanup.PosterCleanupError):
|
||||
cleanup.preview(db_sync, source_id=src.id)
|
||||
|
||||
|
||||
def test_the_token_names_what_the_preview_showed():
|
||||
assert cleanup.confirm_token({"posts": 2, "images": 1}) == "remove-2-posts-1-images"
|
||||
@@ -283,6 +283,27 @@ async def test_scroll_item_shape_minimal(db):
|
||||
assert "description_full" not in item
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_discord_post_names_its_channel(db):
|
||||
"""#4481: the card says which channel a message came from. Read from the
|
||||
record's `channel`; never on another platform, even with the same key."""
|
||||
artist = await _seed_artist(db, "todding-ch")
|
||||
dsrc = await _seed_source(db, artist.id, "discord", "https://discord.com/channels/1/2")
|
||||
psrc = await _seed_source(db, artist.id, "patreon", "https://p/todding-ch")
|
||||
now = datetime.now(UTC)
|
||||
d = await _seed_post(db, dsrc.id, external_id="D1", post_date=now)
|
||||
d.raw_metadata = {"category": "discord", "channel": "banana-land"}
|
||||
blank = await _seed_post(db, dsrc.id, external_id="D2", post_date=now)
|
||||
blank.raw_metadata = {"category": "discord", "channel": " "}
|
||||
p = await _seed_post(db, psrc.id, external_id="P1", post_date=now)
|
||||
p.raw_metadata = {"channel": "not-discord"}
|
||||
await db.commit()
|
||||
|
||||
page = await PostFeedService(db).scroll(cursor=None, limit=10, artist_id=artist.id)
|
||||
by_id = {it["external_post_id"]: it["channel"] for it in page["items"]}
|
||||
assert by_id == {"D1": "banana-land", "D2": None, "P1": None}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scroll_surfaces_translation_fields(db):
|
||||
# #143: a translated post exposes the translated title/description + source
|
||||
|
||||
Reference in New Issue
Block a user