diff --git a/backend/app/api/sources.py b/backend/app/api/sources.py index 6500aa6..a9d3d1e 100644 --- a/backend/app/api/sources.py +++ b/backend/app/api/sources.py @@ -1,10 +1,13 @@ """FC-3a: CRUD over Source rows. FC-3c adds POST //check.""" +from pathlib import Path + from quart import Blueprint, jsonify, request from sqlalchemy import func, select from ..extensions import get_session from ..models import DownloadEvent, MembershipSync, PlatformMembership, Source +from ..services import discord_poster_cleanup as poster_cleanup from ..services.artist_membership_service import ArtistMembershipService from ..services.artist_membership_service import rescan as membership_rescan from ..services.artist_service import ArtistService @@ -463,3 +466,49 @@ async def adopt_membership(): return jsonify({"already_tracked": exc.existing_id}) artist_id = artist.id return jsonify({"source_id": record.id, "artist_id": artist_id}), 201 + + +# -- Discord: remove posts by people outside the poster list (#4486) -------------- + +_IMAGES_ROOT = Path("/images") + + +async def _poster_cleanup_preview(source_id: int): + async with get_session() as session: + return await session.run_sync( + lambda s: poster_cleanup.preview(s, source_id=source_id) + ) + + +@sources_bp.route("//discord/other-posters", methods=["GET"]) +async def other_posters_preview(source_id: int): + """What removing other posters' posts would delete. Nothing is touched.""" + try: + projection = await _poster_cleanup_preview(source_id) + except LookupError: + return _bad("not_found", status=404) + except poster_cleanup.PosterCleanupError as exc: + return _bad("not_applicable", detail=str(exc), status=409) + projection["confirm_token"] = poster_cleanup.confirm_token(projection) + return jsonify(projection) + + +@sources_bp.route("//discord/other-posters/remove", methods=["POST"]) +async def other_posters_remove(source_id: int): + """Delete them. `confirm` must be the token of the preview the operator saw: + a poster list edited since then changes the set, and the token with it.""" + body = await request.get_json(silent=True) or {} + try: + projection = await _poster_cleanup_preview(source_id) + except LookupError: + return _bad("not_found", status=404) + except poster_cleanup.PosterCleanupError as exc: + return _bad("not_applicable", detail=str(exc), status=409) + expected = poster_cleanup.confirm_token(projection) + if body.get("confirm") != expected: + return _bad("confirm_mismatch", expected=expected) + async with get_session() as session: + result = await session.run_sync( + lambda s: poster_cleanup.apply(s, source_id=source_id, images_root=_IMAGES_ROOT) + ) + return jsonify(result) diff --git a/backend/app/services/discord_downloader.py b/backend/app/services/discord_downloader.py index d83e354..c9f542a 100644 --- a/backend/app/services/discord_downloader.py +++ b/backend/app/services/discord_downloader.py @@ -195,6 +195,7 @@ class DiscordDownloader(BaseNativeDownloader): "parent": meta.get("parent"), "is_thread": meta.get("is_thread"), "author": author.get("username"), + "author_name": author.get("global_name"), "author_id": author.get("id"), "message": body, "date": post.get("timestamp"), diff --git a/backend/app/services/discord_poster_cleanup.py b/backend/app/services/discord_poster_cleanup.py new file mode 100644 index 0000000..507f00c --- /dev/null +++ b/backend/app/services/discord_poster_cleanup.py @@ -0,0 +1,243 @@ +"""Remove a Discord source's posts by people outside its poster list (#4486). + +A creator's server is full of other members posting their own pictures. The +poster list (`discord_authors`, #4481) stops a walk taking them; this removes +the ones taken before the list existed. The operator sets the list first, then +previews what would go, then applies. + +## What goes + +A post of this source whose record names its poster (`author_id` / `author`, +written by `DiscordDownloader.write_post_record`) and names someone NOT on the +list. Two kinds of post are never touched: + +* a post whose record names no poster — a message recorded before the native + ingester, whose poster FC cannot tell. Counted as `unknown`, never guessed; +* a synthetic drop post (`synthesized_by`) — FC authored it, and it is handled + below as a consequence, not matched as a poster's post. + +An image goes when every REAL post it belongs to is going. A link to a +synthetic drop does not keep an image: the drop only references its members' +images. An image also on a kept post — the creator re-posting a piece someone +else shared — stays. + +A drop that absorbed a removed message is deleted outright. Deleting a drop is +its undo (discord_grouping's honesty rule): its remaining members return to +the feed and the grouping sweep regroups them on its next run, so no half- +rebuilt drop keeps text or thumbnails from someone it no longer contains. + +Preview and apply spread the same predicates (rule 93, snippet #3087). +""" + +from __future__ import annotations + +import logging +from pathlib import Path + +from sqlalchemy import and_, delete, exists, func, or_, select +from sqlalchemy.orm import Session, aliased + +from ..models import ImageProvenance, ImageRecord, Post, PostAttachment, Source +from .cleanup_service import delete_images +from .discord_ingester import AUTHORS_KEY + +log = logging.getLogger(__name__) + +PLATFORM = "discord" +# The record keys that name a message's poster. `author_name` (the display +# name) is only on records written after 2026-09-28. +_POSTER_KEYS = ("author_id", "author", "author_name") + + +class PosterCleanupError(ValueError): + """The source can't be cleaned this way (not Discord, or no list).""" + + +def source_authors(source: Source) -> list[str]: + """The source's poster list, lowercased; empty means everyone's wanted.""" + authors = (source.config_overrides or {}).get(AUTHORS_KEY) or [] + if not isinstance(authors, list): + return [] + return sorted({str(a).strip().lower() for a in authors if str(a).strip()}) + + +def _poster(key: str): + return func.lower(Post.raw_metadata[key].as_string()) + + +def _has_poster(): + return or_(*(Post.raw_metadata[k].as_string().isnot(None) for k in ("author_id", "author"))) + + +def _real_post_conditions(source_id: int) -> list: + return [Post.source_id == source_id, Post.synthesized_by.is_(None)] + + +def _named_poster_conditions(source_id: int) -> list: + """A real post of this source whose record says who posted it.""" + return [*_real_post_conditions(source_id), _has_poster()] + + +def _other_poster_post_conditions(source_id: int, authors: list[str]) -> list: + """The posts that go: a named poster none of whose names is on the list.""" + return [ + *_named_poster_conditions(source_id), + *(func.coalesce(_poster(k), "").notin_(authors) for k in _POSTER_KEYS), + ] + + +def _doomed_post_ids(source_id: int, authors: list[str]): + # correlate(None): used inside queries that are themselves over `post` (the + # drops, the delete), where auto-correlation would bind this to the outer + # row instead of scanning the table. + return ( + select(Post.id) + .where(*_other_poster_post_conditions(source_id, authors)) + .correlate(None) + ) + + +def _image_conditions(source_id: int, authors: list[str]) -> list: + """Images that belong to a removed post and to no kept real post.""" + doomed = _doomed_post_ids(source_id, authors) + keeper = aliased(Post) + on_doomed = or_( + ImageRecord.primary_post_id.in_(doomed), + exists().where( + ImageProvenance.image_record_id == ImageRecord.id, + ImageProvenance.post_id.in_(doomed), + ), + ) + on_kept = or_( + and_( + ImageRecord.primary_post_id.isnot(None), + ImageRecord.primary_post_id.notin_(doomed), + ), + exists().where( + ImageProvenance.image_record_id == ImageRecord.id, + ImageProvenance.post_id == keeper.id, + keeper.synthesized_by.is_(None), + keeper.id.notin_(doomed), + ), + ) + return [on_doomed, ~on_kept] + + +def _drop_conditions(source_id: int, authors: list[str]) -> list: + """The synthetic drops that absorbed a removed message.""" + member = aliased(Post) + return [ + Post.source_id == source_id, + Post.synthesized_by.isnot(None), + exists().where( + member.absorbed_by_post_id == Post.id, + member.id.in_(_doomed_post_ids(source_id, authors)), + ), + ] + + +def _attachment_conditions(source_id: int, authors: list[str]) -> list: + return [PostAttachment.post_id.in_(_doomed_post_ids(source_id, authors))] + + +def _load(session: Session, source_id: int) -> tuple[Source, list[str]]: + source = session.get(Source, source_id) + if source is None: + raise LookupError(source_id) + if source.platform != PLATFORM: + raise PosterCleanupError("Only a Discord source has posters to filter by.") + authors = source_authors(source) + if not authors: + raise PosterCleanupError( + "Set this source's 'Only posts by' list first — with no list, " + "every poster is wanted and nothing would be removed." + ) + return source, authors + + +def _count(session: Session, stmt) -> int: + return session.execute(stmt).scalar_one() + + +def preview(session: Session, *, source_id: int) -> dict: + """What `apply` would remove, per poster, without touching anything.""" + _, authors = _load(session, source_id) + who = func.coalesce( + Post.raw_metadata["author_name"].as_string(), + Post.raw_metadata["author"].as_string(), + Post.raw_metadata["author_id"].as_string(), + ) + rows = session.execute( + select(who, func.count(Post.id)) + .where(*_other_poster_post_conditions(source_id, authors)) + .group_by(who) + .order_by(func.count(Post.id).desc()) + ).all() + posters = [{"poster": name, "posts": n} for name, n in rows] + unknown = _count(session, select(func.count(Post.id)).where( + *_real_post_conditions(source_id), ~_has_poster(), + )) + return { + "authors": authors, + "posters": posters, + "posts": sum(p["posts"] for p in posters), + "images": _count(session, select(func.count(ImageRecord.id)).where( + *_image_conditions(source_id, authors))), + "attachments": _count(session, select(func.count(PostAttachment.id)).where( + *_attachment_conditions(source_id, authors))), + "drops": _count(session, select(func.count(Post.id)).where( + *_drop_conditions(source_id, authors))), + "unknown_posts": unknown, + } + + +def confirm_token(projection: dict) -> str: + """What the operator's apply must echo: the preview it was shown, so a + list edited between preview and apply can't delete an unseen set.""" + return f"remove-{projection['posts']}-posts-{projection['images']}-images" + + +def apply(session: Session, *, source_id: int, images_root: Path) -> dict: + """Remove them. Counts are taken before the deletes they describe — once + the rows are gone there is nothing left to count (snippet #3087 note 2).""" + projection = preview(session, source_id=source_id) + _, authors = _load(session, source_id) + + image_ids = session.execute( + select(ImageRecord.id).where(*_image_conditions(source_id, authors)) + ).scalars().all() + # Drops first-class: found BEFORE their members go, since the link that + # finds them is the member's absorbed_by_post_id. + drop_ids = session.execute( + select(Post.id).where(*_drop_conditions(source_id, authors)) + ).scalars().all() + + deleted = delete_images(session, image_ids=list(image_ids), images_root=images_root) + # Attachments before posts: post_attachment.post_id is SET NULL, and a + # NULL-post attachment collides on its partial unique index (see + # cleanup_service.delete_artist_cascade). + attachments = session.execute( + delete(PostAttachment).where(*_attachment_conditions(source_id, authors)) + ).rowcount or 0 + posts = session.execute( + Post.__table__.delete().where(Post.id.in_(_doomed_post_ids(source_id, authors))) + ).rowcount or 0 + drops = 0 + if drop_ids: + drops = session.execute( + Post.__table__.delete().where(Post.id.in_(drop_ids)) + ).rowcount or 0 + session.commit() + log.info( + "discord poster cleanup (source %s, keeping %s): %d post(s), %d image(s), " + "%d attachment(s), %d drop(s) removed", + source_id, authors, posts, deleted["images_deleted"], attachments, drops, + ) + return { + **projection, + "posts_deleted": posts, + "images_deleted": deleted["images_deleted"], + "files_deleted": deleted["files_deleted"], + "attachments_deleted": attachments, + "drops_deleted": drops, + } diff --git a/frontend/src/components/subscriptions/PosterCleanupDialog.vue b/frontend/src/components/subscriptions/PosterCleanupDialog.vue new file mode 100644 index 0000000..cc2873b --- /dev/null +++ b/frontend/src/components/subscriptions/PosterCleanupDialog.vue @@ -0,0 +1,135 @@ + + + + + diff --git a/frontend/src/components/subscriptions/SourceActions.vue b/frontend/src/components/subscriptions/SourceActions.vue index 3512b70..03e4ddc 100644 --- a/frontend/src/components/subscriptions/SourceActions.vue +++ b/frontend/src/components/subscriptions/SourceActions.vue @@ -50,6 +50,17 @@ library — without re-downloading media + + Remove posts from other posters + + Preview, then delete the posts taken from people outside this + source's "Only posts by" list + + Remove source +