feat: a Discord source can remove the posts it took from people outside its poster list (#4486)
CI and images / extension-version (push) Successful in 3s
CI and images / lint (push) Successful in 3s
CI and images / extension-test (push) Successful in 20s
CI and images / frontend-build (push) Successful in 21s
CI and images / backend-lint-and-test (push) Successful in 33s
CI and images / integration (push) Successful in 2m39s
CI and images / sign-extension (push) Successful in 3s
CI and images / build-agent (push) Successful in 6s
CI and images / build-web (push) Successful in 1m43s
CI and images / smoke-web (push) Successful in 54s
CI and images / promote (push) Successful in 2s
CI and images / extension-version (push) Successful in 3s
CI and images / lint (push) Successful in 3s
CI and images / extension-test (push) Successful in 20s
CI and images / frontend-build (push) Successful in 21s
CI and images / backend-lint-and-test (push) Successful in 33s
CI and images / integration (push) Successful in 2m39s
CI and images / sign-extension (push) Successful in 3s
CI and images / build-agent (push) Successful in 6s
CI and images / build-web (push) Successful in 1m43s
CI and images / smoke-web (push) Successful in 54s
CI and images / promote (push) Successful in 2s
"Remove posts from other posters" on a Discord source with an "Only posts by" list previews what goes, per poster, then deletes it behind a typed token: - posts whose record names a poster not on the list (by id, username or display name); posts that record no poster are left alone and counted; - images found only on those posts; one also on a kept post stays, and a synthetic drop's link never keeps one; - their attachments, and every drop that absorbed one of them, so the grouping sweep regroups what remains. Preview and apply share one predicate, and a parity test holds them to it. The post record now also saves the poster's display name. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,10 +1,13 @@
|
|||||||
"""FC-3a: CRUD over Source rows. FC-3c adds POST /<id>/check."""
|
"""FC-3a: CRUD over Source rows. FC-3c adds POST /<id>/check."""
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
from quart import Blueprint, jsonify, request
|
from quart import Blueprint, jsonify, request
|
||||||
from sqlalchemy import func, select
|
from sqlalchemy import func, select
|
||||||
|
|
||||||
from ..extensions import get_session
|
from ..extensions import get_session
|
||||||
from ..models import DownloadEvent, MembershipSync, PlatformMembership, Source
|
from ..models import DownloadEvent, MembershipSync, PlatformMembership, Source
|
||||||
|
from ..services import discord_poster_cleanup as poster_cleanup
|
||||||
from ..services.artist_membership_service import ArtistMembershipService
|
from ..services.artist_membership_service import ArtistMembershipService
|
||||||
from ..services.artist_membership_service import rescan as membership_rescan
|
from ..services.artist_membership_service import rescan as membership_rescan
|
||||||
from ..services.artist_service import ArtistService
|
from ..services.artist_service import ArtistService
|
||||||
@@ -463,3 +466,49 @@ async def adopt_membership():
|
|||||||
return jsonify({"already_tracked": exc.existing_id})
|
return jsonify({"already_tracked": exc.existing_id})
|
||||||
artist_id = artist.id
|
artist_id = artist.id
|
||||||
return jsonify({"source_id": record.id, "artist_id": artist_id}), 201
|
return jsonify({"source_id": record.id, "artist_id": artist_id}), 201
|
||||||
|
|
||||||
|
|
||||||
|
# -- Discord: remove posts by people outside the poster list (#4486) --------------
|
||||||
|
|
||||||
|
_IMAGES_ROOT = Path("/images")
|
||||||
|
|
||||||
|
|
||||||
|
async def _poster_cleanup_preview(source_id: int):
|
||||||
|
async with get_session() as session:
|
||||||
|
return await session.run_sync(
|
||||||
|
lambda s: poster_cleanup.preview(s, source_id=source_id)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@sources_bp.route("/<int:source_id>/discord/other-posters", methods=["GET"])
|
||||||
|
async def other_posters_preview(source_id: int):
|
||||||
|
"""What removing other posters' posts would delete. Nothing is touched."""
|
||||||
|
try:
|
||||||
|
projection = await _poster_cleanup_preview(source_id)
|
||||||
|
except LookupError:
|
||||||
|
return _bad("not_found", status=404)
|
||||||
|
except poster_cleanup.PosterCleanupError as exc:
|
||||||
|
return _bad("not_applicable", detail=str(exc), status=409)
|
||||||
|
projection["confirm_token"] = poster_cleanup.confirm_token(projection)
|
||||||
|
return jsonify(projection)
|
||||||
|
|
||||||
|
|
||||||
|
@sources_bp.route("/<int:source_id>/discord/other-posters/remove", methods=["POST"])
|
||||||
|
async def other_posters_remove(source_id: int):
|
||||||
|
"""Delete them. `confirm` must be the token of the preview the operator saw:
|
||||||
|
a poster list edited since then changes the set, and the token with it."""
|
||||||
|
body = await request.get_json(silent=True) or {}
|
||||||
|
try:
|
||||||
|
projection = await _poster_cleanup_preview(source_id)
|
||||||
|
except LookupError:
|
||||||
|
return _bad("not_found", status=404)
|
||||||
|
except poster_cleanup.PosterCleanupError as exc:
|
||||||
|
return _bad("not_applicable", detail=str(exc), status=409)
|
||||||
|
expected = poster_cleanup.confirm_token(projection)
|
||||||
|
if body.get("confirm") != expected:
|
||||||
|
return _bad("confirm_mismatch", expected=expected)
|
||||||
|
async with get_session() as session:
|
||||||
|
result = await session.run_sync(
|
||||||
|
lambda s: poster_cleanup.apply(s, source_id=source_id, images_root=_IMAGES_ROOT)
|
||||||
|
)
|
||||||
|
return jsonify(result)
|
||||||
|
|||||||
@@ -195,6 +195,7 @@ class DiscordDownloader(BaseNativeDownloader):
|
|||||||
"parent": meta.get("parent"),
|
"parent": meta.get("parent"),
|
||||||
"is_thread": meta.get("is_thread"),
|
"is_thread": meta.get("is_thread"),
|
||||||
"author": author.get("username"),
|
"author": author.get("username"),
|
||||||
|
"author_name": author.get("global_name"),
|
||||||
"author_id": author.get("id"),
|
"author_id": author.get("id"),
|
||||||
"message": body,
|
"message": body,
|
||||||
"date": post.get("timestamp"),
|
"date": post.get("timestamp"),
|
||||||
|
|||||||
@@ -0,0 +1,243 @@
|
|||||||
|
"""Remove a Discord source's posts by people outside its poster list (#4486).
|
||||||
|
|
||||||
|
A creator's server is full of other members posting their own pictures. The
|
||||||
|
poster list (`discord_authors`, #4481) stops a walk taking them; this removes
|
||||||
|
the ones taken before the list existed. The operator sets the list first, then
|
||||||
|
previews what would go, then applies.
|
||||||
|
|
||||||
|
## What goes
|
||||||
|
|
||||||
|
A post of this source whose record names its poster (`author_id` / `author`,
|
||||||
|
written by `DiscordDownloader.write_post_record`) and names someone NOT on the
|
||||||
|
list. Two kinds of post are never touched:
|
||||||
|
|
||||||
|
* a post whose record names no poster — a message recorded before the native
|
||||||
|
ingester, whose poster FC cannot tell. Counted as `unknown`, never guessed;
|
||||||
|
* a synthetic drop post (`synthesized_by`) — FC authored it, and it is handled
|
||||||
|
below as a consequence, not matched as a poster's post.
|
||||||
|
|
||||||
|
An image goes when every REAL post it belongs to is going. A link to a
|
||||||
|
synthetic drop does not keep an image: the drop only references its members'
|
||||||
|
images. An image also on a kept post — the creator re-posting a piece someone
|
||||||
|
else shared — stays.
|
||||||
|
|
||||||
|
A drop that absorbed a removed message is deleted outright. Deleting a drop is
|
||||||
|
its undo (discord_grouping's honesty rule): its remaining members return to
|
||||||
|
the feed and the grouping sweep regroups them on its next run, so no half-
|
||||||
|
rebuilt drop keeps text or thumbnails from someone it no longer contains.
|
||||||
|
|
||||||
|
Preview and apply spread the same predicates (rule 93, snippet #3087).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from sqlalchemy import and_, delete, exists, func, or_, select
|
||||||
|
from sqlalchemy.orm import Session, aliased
|
||||||
|
|
||||||
|
from ..models import ImageProvenance, ImageRecord, Post, PostAttachment, Source
|
||||||
|
from .cleanup_service import delete_images
|
||||||
|
from .discord_ingester import AUTHORS_KEY
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
PLATFORM = "discord"
|
||||||
|
# The record keys that name a message's poster. `author_name` (the display
|
||||||
|
# name) is only on records written after 2026-09-28.
|
||||||
|
_POSTER_KEYS = ("author_id", "author", "author_name")
|
||||||
|
|
||||||
|
|
||||||
|
class PosterCleanupError(ValueError):
|
||||||
|
"""The source can't be cleaned this way (not Discord, or no list)."""
|
||||||
|
|
||||||
|
|
||||||
|
def source_authors(source: Source) -> list[str]:
|
||||||
|
"""The source's poster list, lowercased; empty means everyone's wanted."""
|
||||||
|
authors = (source.config_overrides or {}).get(AUTHORS_KEY) or []
|
||||||
|
if not isinstance(authors, list):
|
||||||
|
return []
|
||||||
|
return sorted({str(a).strip().lower() for a in authors if str(a).strip()})
|
||||||
|
|
||||||
|
|
||||||
|
def _poster(key: str):
|
||||||
|
return func.lower(Post.raw_metadata[key].as_string())
|
||||||
|
|
||||||
|
|
||||||
|
def _has_poster():
|
||||||
|
return or_(*(Post.raw_metadata[k].as_string().isnot(None) for k in ("author_id", "author")))
|
||||||
|
|
||||||
|
|
||||||
|
def _real_post_conditions(source_id: int) -> list:
|
||||||
|
return [Post.source_id == source_id, Post.synthesized_by.is_(None)]
|
||||||
|
|
||||||
|
|
||||||
|
def _named_poster_conditions(source_id: int) -> list:
|
||||||
|
"""A real post of this source whose record says who posted it."""
|
||||||
|
return [*_real_post_conditions(source_id), _has_poster()]
|
||||||
|
|
||||||
|
|
||||||
|
def _other_poster_post_conditions(source_id: int, authors: list[str]) -> list:
|
||||||
|
"""The posts that go: a named poster none of whose names is on the list."""
|
||||||
|
return [
|
||||||
|
*_named_poster_conditions(source_id),
|
||||||
|
*(func.coalesce(_poster(k), "").notin_(authors) for k in _POSTER_KEYS),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _doomed_post_ids(source_id: int, authors: list[str]):
|
||||||
|
# correlate(None): used inside queries that are themselves over `post` (the
|
||||||
|
# drops, the delete), where auto-correlation would bind this to the outer
|
||||||
|
# row instead of scanning the table.
|
||||||
|
return (
|
||||||
|
select(Post.id)
|
||||||
|
.where(*_other_poster_post_conditions(source_id, authors))
|
||||||
|
.correlate(None)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _image_conditions(source_id: int, authors: list[str]) -> list:
|
||||||
|
"""Images that belong to a removed post and to no kept real post."""
|
||||||
|
doomed = _doomed_post_ids(source_id, authors)
|
||||||
|
keeper = aliased(Post)
|
||||||
|
on_doomed = or_(
|
||||||
|
ImageRecord.primary_post_id.in_(doomed),
|
||||||
|
exists().where(
|
||||||
|
ImageProvenance.image_record_id == ImageRecord.id,
|
||||||
|
ImageProvenance.post_id.in_(doomed),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
on_kept = or_(
|
||||||
|
and_(
|
||||||
|
ImageRecord.primary_post_id.isnot(None),
|
||||||
|
ImageRecord.primary_post_id.notin_(doomed),
|
||||||
|
),
|
||||||
|
exists().where(
|
||||||
|
ImageProvenance.image_record_id == ImageRecord.id,
|
||||||
|
ImageProvenance.post_id == keeper.id,
|
||||||
|
keeper.synthesized_by.is_(None),
|
||||||
|
keeper.id.notin_(doomed),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return [on_doomed, ~on_kept]
|
||||||
|
|
||||||
|
|
||||||
|
def _drop_conditions(source_id: int, authors: list[str]) -> list:
|
||||||
|
"""The synthetic drops that absorbed a removed message."""
|
||||||
|
member = aliased(Post)
|
||||||
|
return [
|
||||||
|
Post.source_id == source_id,
|
||||||
|
Post.synthesized_by.isnot(None),
|
||||||
|
exists().where(
|
||||||
|
member.absorbed_by_post_id == Post.id,
|
||||||
|
member.id.in_(_doomed_post_ids(source_id, authors)),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _attachment_conditions(source_id: int, authors: list[str]) -> list:
|
||||||
|
return [PostAttachment.post_id.in_(_doomed_post_ids(source_id, authors))]
|
||||||
|
|
||||||
|
|
||||||
|
def _load(session: Session, source_id: int) -> tuple[Source, list[str]]:
|
||||||
|
source = session.get(Source, source_id)
|
||||||
|
if source is None:
|
||||||
|
raise LookupError(source_id)
|
||||||
|
if source.platform != PLATFORM:
|
||||||
|
raise PosterCleanupError("Only a Discord source has posters to filter by.")
|
||||||
|
authors = source_authors(source)
|
||||||
|
if not authors:
|
||||||
|
raise PosterCleanupError(
|
||||||
|
"Set this source's 'Only posts by' list first — with no list, "
|
||||||
|
"every poster is wanted and nothing would be removed."
|
||||||
|
)
|
||||||
|
return source, authors
|
||||||
|
|
||||||
|
|
||||||
|
def _count(session: Session, stmt) -> int:
|
||||||
|
return session.execute(stmt).scalar_one()
|
||||||
|
|
||||||
|
|
||||||
|
def preview(session: Session, *, source_id: int) -> dict:
|
||||||
|
"""What `apply` would remove, per poster, without touching anything."""
|
||||||
|
_, authors = _load(session, source_id)
|
||||||
|
who = func.coalesce(
|
||||||
|
Post.raw_metadata["author_name"].as_string(),
|
||||||
|
Post.raw_metadata["author"].as_string(),
|
||||||
|
Post.raw_metadata["author_id"].as_string(),
|
||||||
|
)
|
||||||
|
rows = session.execute(
|
||||||
|
select(who, func.count(Post.id))
|
||||||
|
.where(*_other_poster_post_conditions(source_id, authors))
|
||||||
|
.group_by(who)
|
||||||
|
.order_by(func.count(Post.id).desc())
|
||||||
|
).all()
|
||||||
|
posters = [{"poster": name, "posts": n} for name, n in rows]
|
||||||
|
unknown = _count(session, select(func.count(Post.id)).where(
|
||||||
|
*_real_post_conditions(source_id), ~_has_poster(),
|
||||||
|
))
|
||||||
|
return {
|
||||||
|
"authors": authors,
|
||||||
|
"posters": posters,
|
||||||
|
"posts": sum(p["posts"] for p in posters),
|
||||||
|
"images": _count(session, select(func.count(ImageRecord.id)).where(
|
||||||
|
*_image_conditions(source_id, authors))),
|
||||||
|
"attachments": _count(session, select(func.count(PostAttachment.id)).where(
|
||||||
|
*_attachment_conditions(source_id, authors))),
|
||||||
|
"drops": _count(session, select(func.count(Post.id)).where(
|
||||||
|
*_drop_conditions(source_id, authors))),
|
||||||
|
"unknown_posts": unknown,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def confirm_token(projection: dict) -> str:
|
||||||
|
"""What the operator's apply must echo: the preview it was shown, so a
|
||||||
|
list edited between preview and apply can't delete an unseen set."""
|
||||||
|
return f"remove-{projection['posts']}-posts-{projection['images']}-images"
|
||||||
|
|
||||||
|
|
||||||
|
def apply(session: Session, *, source_id: int, images_root: Path) -> dict:
|
||||||
|
"""Remove them. Counts are taken before the deletes they describe — once
|
||||||
|
the rows are gone there is nothing left to count (snippet #3087 note 2)."""
|
||||||
|
projection = preview(session, source_id=source_id)
|
||||||
|
_, authors = _load(session, source_id)
|
||||||
|
|
||||||
|
image_ids = session.execute(
|
||||||
|
select(ImageRecord.id).where(*_image_conditions(source_id, authors))
|
||||||
|
).scalars().all()
|
||||||
|
# Drops first-class: found BEFORE their members go, since the link that
|
||||||
|
# finds them is the member's absorbed_by_post_id.
|
||||||
|
drop_ids = session.execute(
|
||||||
|
select(Post.id).where(*_drop_conditions(source_id, authors))
|
||||||
|
).scalars().all()
|
||||||
|
|
||||||
|
deleted = delete_images(session, image_ids=list(image_ids), images_root=images_root)
|
||||||
|
# Attachments before posts: post_attachment.post_id is SET NULL, and a
|
||||||
|
# NULL-post attachment collides on its partial unique index (see
|
||||||
|
# cleanup_service.delete_artist_cascade).
|
||||||
|
attachments = session.execute(
|
||||||
|
delete(PostAttachment).where(*_attachment_conditions(source_id, authors))
|
||||||
|
).rowcount or 0
|
||||||
|
posts = session.execute(
|
||||||
|
Post.__table__.delete().where(Post.id.in_(_doomed_post_ids(source_id, authors)))
|
||||||
|
).rowcount or 0
|
||||||
|
drops = 0
|
||||||
|
if drop_ids:
|
||||||
|
drops = session.execute(
|
||||||
|
Post.__table__.delete().where(Post.id.in_(drop_ids))
|
||||||
|
).rowcount or 0
|
||||||
|
session.commit()
|
||||||
|
log.info(
|
||||||
|
"discord poster cleanup (source %s, keeping %s): %d post(s), %d image(s), "
|
||||||
|
"%d attachment(s), %d drop(s) removed",
|
||||||
|
source_id, authors, posts, deleted["images_deleted"], attachments, drops,
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
**projection,
|
||||||
|
"posts_deleted": posts,
|
||||||
|
"images_deleted": deleted["images_deleted"],
|
||||||
|
"files_deleted": deleted["files_deleted"],
|
||||||
|
"attachments_deleted": attachments,
|
||||||
|
"drops_deleted": drops,
|
||||||
|
}
|
||||||
@@ -0,0 +1,135 @@
|
|||||||
|
<template>
|
||||||
|
<!-- #4486. A Discord source's posts by people outside its "Only posts by"
|
||||||
|
list, taken before the list existed. Preview first, always: nothing is
|
||||||
|
deleted until the operator has seen the per-poster breakdown and typed
|
||||||
|
the token that preview returned. -->
|
||||||
|
<v-dialog :model-value="modelValue" max-width="560"
|
||||||
|
@update:model-value="$emit('update:modelValue', $event)">
|
||||||
|
<v-card>
|
||||||
|
<v-card-title>Remove posts from other posters</v-card-title>
|
||||||
|
<v-card-text>
|
||||||
|
<div class="text-caption mb-2">{{ source.display_name || source.url }}</div>
|
||||||
|
<v-progress-linear v-if="loading" indeterminate color="accent" class="mb-3" />
|
||||||
|
<v-alert v-else-if="error" type="warning" variant="tonal" density="compact">
|
||||||
|
{{ error }}
|
||||||
|
</v-alert>
|
||||||
|
<template v-else-if="result">
|
||||||
|
<v-alert type="success" variant="tonal" density="compact">
|
||||||
|
Removed {{ result.posts_deleted }} post(s), {{ result.images_deleted }} image(s),
|
||||||
|
{{ result.attachments_deleted }} attachment(s) and {{ result.drops_deleted }}
|
||||||
|
grouped post(s). Grouping rebuilds what remains on its next sweep.
|
||||||
|
</v-alert>
|
||||||
|
</template>
|
||||||
|
<template v-else-if="preview">
|
||||||
|
<p class="mb-2">
|
||||||
|
Keeping posts by <strong>{{ preview.authors.join(', ') }}</strong>.
|
||||||
|
</p>
|
||||||
|
<p v-if="!preview.posts" class="fc-dim">
|
||||||
|
Nothing to remove: every post with a known poster is by someone on the list.
|
||||||
|
</p>
|
||||||
|
<template v-else>
|
||||||
|
<v-table density="compact" class="mb-3">
|
||||||
|
<thead><tr><th>Poster</th><th class="text-right">Posts</th></tr></thead>
|
||||||
|
<tbody>
|
||||||
|
<tr v-for="p in preview.posters" :key="p.poster">
|
||||||
|
<td>{{ p.poster }}</td>
|
||||||
|
<td class="text-right">{{ p.posts }}</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</v-table>
|
||||||
|
<p class="mb-1">
|
||||||
|
{{ preview.posts }} post(s), {{ preview.images }} image(s) found only on them,
|
||||||
|
{{ preview.attachments }} attachment(s).
|
||||||
|
</p>
|
||||||
|
<p v-if="preview.drops" class="mb-1">
|
||||||
|
{{ preview.drops }} grouped post(s) that include them are ungrouped; the
|
||||||
|
rest regroup on the next sweep.
|
||||||
|
</p>
|
||||||
|
<p class="fc-dim">
|
||||||
|
Images that also belong to a kept post stay. If a name you want to keep
|
||||||
|
appears above, add it to the source's "Only posts by" list first.
|
||||||
|
</p>
|
||||||
|
</template>
|
||||||
|
<p v-if="preview.unknown_posts" class="fc-dim mt-2">
|
||||||
|
{{ preview.unknown_posts }} older post(s) don't record who posted them and
|
||||||
|
are left alone.
|
||||||
|
</p>
|
||||||
|
</template>
|
||||||
|
</v-card-text>
|
||||||
|
<v-card-actions>
|
||||||
|
<v-spacer />
|
||||||
|
<v-btn variant="text" @click="$emit('update:modelValue', false)">
|
||||||
|
{{ result ? 'Close' : 'Cancel' }}
|
||||||
|
</v-btn>
|
||||||
|
<v-btn
|
||||||
|
v-if="preview && preview.posts && !result"
|
||||||
|
color="error" variant="flat" :loading="busy"
|
||||||
|
@click="confirmOpen = true"
|
||||||
|
>Remove {{ preview.posts }} post(s)</v-btn>
|
||||||
|
</v-card-actions>
|
||||||
|
</v-card>
|
||||||
|
<DestructiveConfirmModal
|
||||||
|
v-if="preview"
|
||||||
|
v-model="confirmOpen"
|
||||||
|
action="delete" kind="other posters' posts" tier="C"
|
||||||
|
:expected-token-override="preview.confirm_token"
|
||||||
|
:projected-counts="{ posts: preview.posts, images: preview.images,
|
||||||
|
attachments: preview.attachments, grouped: preview.drops }"
|
||||||
|
@confirm="onConfirm"
|
||||||
|
/>
|
||||||
|
</v-dialog>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { ref, watch } from 'vue'
|
||||||
|
import { useSourcesStore } from '../../stores/sources.js'
|
||||||
|
import DestructiveConfirmModal from '../modal/DestructiveConfirmModal.vue'
|
||||||
|
|
||||||
|
const props = defineProps({
|
||||||
|
modelValue: { type: Boolean, default: false },
|
||||||
|
source: { type: Object, required: true },
|
||||||
|
})
|
||||||
|
defineEmits(['update:modelValue'])
|
||||||
|
const store = useSourcesStore()
|
||||||
|
|
||||||
|
const loading = ref(false)
|
||||||
|
const busy = ref(false)
|
||||||
|
const error = ref('')
|
||||||
|
const preview = ref(null)
|
||||||
|
const result = ref(null)
|
||||||
|
const confirmOpen = ref(false)
|
||||||
|
|
||||||
|
function message(e) {
|
||||||
|
return e?.body?.detail || e?.body?.error || e?.message || String(e)
|
||||||
|
}
|
||||||
|
|
||||||
|
watch(() => props.modelValue, async (open) => {
|
||||||
|
if (!open) return
|
||||||
|
preview.value = null
|
||||||
|
result.value = null
|
||||||
|
error.value = ''
|
||||||
|
loading.value = true
|
||||||
|
try {
|
||||||
|
preview.value = await store.previewOtherPosters(props.source.id)
|
||||||
|
} catch (e) {
|
||||||
|
error.value = message(e)
|
||||||
|
} finally {
|
||||||
|
loading.value = false
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
async function onConfirm(token) {
|
||||||
|
busy.value = true
|
||||||
|
try {
|
||||||
|
result.value = await store.removeOtherPosters(props.source.id, token)
|
||||||
|
} catch (e) {
|
||||||
|
error.value = message(e)
|
||||||
|
} finally {
|
||||||
|
busy.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-dim { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||||
|
</style>
|
||||||
@@ -50,6 +50,17 @@
|
|||||||
library — without re-downloading media
|
library — without re-downloading media
|
||||||
</v-list-item-subtitle>
|
</v-list-item-subtitle>
|
||||||
</v-list-item>
|
</v-list-item>
|
||||||
|
<v-list-item
|
||||||
|
v-if="hasPosterList && !running"
|
||||||
|
prepend-icon="mdi-account-remove-outline"
|
||||||
|
@click="cleanupOpen = true"
|
||||||
|
>
|
||||||
|
<v-list-item-title>Remove posts from other posters</v-list-item-title>
|
||||||
|
<v-list-item-subtitle>
|
||||||
|
Preview, then delete the posts taken from people outside this
|
||||||
|
source's "Only posts by" list
|
||||||
|
</v-list-item-subtitle>
|
||||||
|
</v-list-item>
|
||||||
<v-divider v-if="isNative && !running" />
|
<v-divider v-if="isNative && !running" />
|
||||||
<v-list-item
|
<v-list-item
|
||||||
base-color="error"
|
base-color="error"
|
||||||
@@ -59,12 +70,14 @@
|
|||||||
<v-list-item-title>Remove source</v-list-item-title>
|
<v-list-item-title>Remove source</v-list-item-title>
|
||||||
</v-list-item>
|
</v-list-item>
|
||||||
</KebabMenu>
|
</KebabMenu>
|
||||||
|
<PosterCleanupDialog v-if="hasPosterList" v-model="cleanupOpen" :source="source" />
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { computed } from 'vue'
|
import { computed, ref } from 'vue'
|
||||||
import KebabMenu from '../common/KebabMenu.vue'
|
import KebabMenu from '../common/KebabMenu.vue'
|
||||||
|
import PosterCleanupDialog from './PosterCleanupDialog.vue'
|
||||||
|
|
||||||
const props = defineProps({
|
const props = defineProps({
|
||||||
source: { type: Object, required: true },
|
source: { type: Object, required: true },
|
||||||
@@ -80,6 +93,10 @@ const recapturing = computed(() => !!props.source.backfill_recapture)
|
|||||||
// which those are (`native_ingester`); a copied list here went stale when
|
// which those are (`native_ingester`); a copied list here went stale when
|
||||||
// Discord moved over.
|
// Discord moved over.
|
||||||
const isNative = computed(() => !!props.source.native_ingester)
|
const isNative = computed(() => !!props.source.native_ingester)
|
||||||
|
// #4486: only a Discord source with an "Only posts by" list has other posters.
|
||||||
|
const hasPosterList = computed(() => props.source.platform === 'discord'
|
||||||
|
&& (props.source.config_overrides?.discord_authors || []).length > 0)
|
||||||
|
const cleanupOpen = ref(false)
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
|
|||||||
@@ -149,6 +149,15 @@ export const useSourcesStore = defineStore('sources', () => {
|
|||||||
return body
|
return body
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// #4486: a Discord source's posts by people outside its poster list.
|
||||||
|
// The preview touches nothing and hands back the token the remove must echo.
|
||||||
|
async function previewOtherPosters(id) {
|
||||||
|
return api.get(`/api/sources/${id}/discord/other-posters`)
|
||||||
|
}
|
||||||
|
async function removeOtherPosters(id, confirm) {
|
||||||
|
return api.post(`/api/sources/${id}/discord/other-posters/remove`, { body: { confirm } })
|
||||||
|
}
|
||||||
|
|
||||||
function sourcesByArtistGrouped() {
|
function sourcesByArtistGrouped() {
|
||||||
// returns [{artist: {id,name,slug}, sources: [...]}, ...]
|
// returns [{artist: {id,name,slug}, sources: [...]}, ...]
|
||||||
const arr = byArtist.value.get(null) ?? []
|
const arr = byArtist.value.get(null) ?? []
|
||||||
@@ -179,6 +188,7 @@ export const useSourcesStore = defineStore('sources', () => {
|
|||||||
stopBackfill,
|
stopBackfill,
|
||||||
recoverSource,
|
recoverSource,
|
||||||
recaptureSource,
|
recaptureSource,
|
||||||
|
previewOtherPosters, removeOtherPosters,
|
||||||
findOrCreateArtist, autocompleteArtist, reassign,
|
findOrCreateArtist, autocompleteArtist, reassign,
|
||||||
loadScheduleStatus,
|
loadScheduleStatus,
|
||||||
sourcesByArtistGrouped,
|
sourcesByArtistGrouped,
|
||||||
|
|||||||
@@ -0,0 +1,131 @@
|
|||||||
|
"""Removing a Discord source's posts by people outside its poster list (#4486).
|
||||||
|
|
||||||
|
Real Postgres (db_sync): the predicates read `raw_metadata->>'author'`, which
|
||||||
|
only the database can evaluate. The test that matters runs preview and apply
|
||||||
|
on one fixture and asserts they agree AND that the rows went (snippet #3087).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
|
||||||
|
from backend.app.models import (
|
||||||
|
Artist,
|
||||||
|
ImageProvenance,
|
||||||
|
ImageRecord,
|
||||||
|
Post,
|
||||||
|
PostAttachment,
|
||||||
|
Source,
|
||||||
|
)
|
||||||
|
from backend.app.services import discord_poster_cleanup as cleanup
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
|
|
||||||
|
def _image(db, artist, tmp_path, name, post):
|
||||||
|
f = tmp_path / f"{name}.png"
|
||||||
|
f.write_bytes(b"x")
|
||||||
|
img = ImageRecord(
|
||||||
|
artist_id=artist.id, path=str(f), sha256=name.ljust(64, "0"),
|
||||||
|
size_bytes=1, mime="image/png", origin="downloaded", primary_post_id=post.id,
|
||||||
|
)
|
||||||
|
db.add(img)
|
||||||
|
db.flush()
|
||||||
|
return img
|
||||||
|
|
||||||
|
|
||||||
|
def _post(db, source, mid, raw, **extra):
|
||||||
|
p = Post(
|
||||||
|
source_id=source.id, artist_id=source.artist_id, external_post_id=mid,
|
||||||
|
raw_metadata=raw, **extra,
|
||||||
|
)
|
||||||
|
db.add(p)
|
||||||
|
db.flush()
|
||||||
|
return p
|
||||||
|
|
||||||
|
|
||||||
|
def _fixture(db, tmp_path, authors=("Todding",)):
|
||||||
|
"""Todding's post, two of Jake's (one sharing an image with Todding's), an
|
||||||
|
old post with no poster, and a drop that absorbed Todding's and Jake's."""
|
||||||
|
artist = Artist(name="Todding", slug="todding-cleanup")
|
||||||
|
db.add(artist)
|
||||||
|
db.flush()
|
||||||
|
src = Source(
|
||||||
|
artist_id=artist.id, platform="discord",
|
||||||
|
url="https://discord.com/channels/1/2",
|
||||||
|
config_overrides={"discord_authors": list(authors)} if authors else None,
|
||||||
|
)
|
||||||
|
db.add(src)
|
||||||
|
db.flush()
|
||||||
|
todd = _post(db, src, "10", {"author": "todding", "author_name": "Todding", "author_id": "1"})
|
||||||
|
jake1 = _post(db, src, "11", {"author": "jakeboii", "author_id": "8"})
|
||||||
|
jake2 = _post(db, src, "12", {"author": "jakeboii", "author_id": "8"})
|
||||||
|
old = _post(db, src, "13", {"category": "discord"})
|
||||||
|
drop = _post(db, src, "fc-drop:10", None, synthesized_by="discord_drop")
|
||||||
|
todd.absorbed_by_post_id = drop.id
|
||||||
|
jake1.absorbed_by_post_id = drop.id
|
||||||
|
|
||||||
|
art = _image(db, artist, tmp_path, "art", todd)
|
||||||
|
meme = _image(db, artist, tmp_path, "meme", jake1)
|
||||||
|
shared = _image(db, artist, tmp_path, "shared", jake2)
|
||||||
|
legacy = _image(db, artist, tmp_path, "legacy", old)
|
||||||
|
db.add_all([
|
||||||
|
ImageProvenance(image_record_id=shared.id, post_id=todd.id, source_id=src.id),
|
||||||
|
ImageProvenance(image_record_id=art.id, post_id=drop.id, source_id=src.id),
|
||||||
|
ImageProvenance(image_record_id=meme.id, post_id=drop.id, source_id=src.id),
|
||||||
|
PostAttachment(
|
||||||
|
post_id=jake2.id, artist_id=artist.id, sha256="z" * 64,
|
||||||
|
path="/store/z/f.zip", original_filename="f.zip", ext=".zip", size_bytes=1,
|
||||||
|
),
|
||||||
|
])
|
||||||
|
db.commit()
|
||||||
|
return src, {
|
||||||
|
"todd": todd.id, "jake1": jake1.id, "jake2": jake2.id, "old": old.id,
|
||||||
|
"drop": drop.id, "art": art.id, "meme": meme.id, "shared": shared.id,
|
||||||
|
"legacy": legacy.id,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_preview_matches_apply_and_the_rows_go(db_sync, tmp_path):
|
||||||
|
src, ids = _fixture(db_sync, tmp_path)
|
||||||
|
|
||||||
|
projected = cleanup.preview(db_sync, source_id=src.id)
|
||||||
|
assert projected["authors"] == ["todding"]
|
||||||
|
assert projected["posters"] == [{"poster": "jakeboii", "posts": 2}]
|
||||||
|
assert projected["unknown_posts"] == 1
|
||||||
|
|
||||||
|
done = cleanup.apply(db_sync, source_id=src.id, images_root=tmp_path)
|
||||||
|
assert projected["posts"] == done["posts_deleted"] == 2
|
||||||
|
assert projected["images"] == done["images_deleted"] == 1 # the meme only
|
||||||
|
assert projected["attachments"] == done["attachments_deleted"] == 1
|
||||||
|
assert projected["drops"] == done["drops_deleted"] == 1
|
||||||
|
|
||||||
|
post_ids = set(db_sync.execute(select(Post.id).where(Post.source_id == src.id)).scalars())
|
||||||
|
assert post_ids == {ids["todd"], ids["old"]}
|
||||||
|
image_ids = set(db_sync.execute(
|
||||||
|
select(ImageRecord.id).where(ImageRecord.id.in_(
|
||||||
|
[ids["art"], ids["meme"], ids["shared"], ids["legacy"]]))
|
||||||
|
).scalars())
|
||||||
|
# The meme was only Jake's. The shared piece is also on Todding's post, and
|
||||||
|
# the drop's link to the art was never what kept it.
|
||||||
|
assert image_ids == {ids["art"], ids["shared"], ids["legacy"]}
|
||||||
|
assert not (tmp_path / "meme.png").exists()
|
||||||
|
# Todding's message is back in the feed, for the sweep to regroup.
|
||||||
|
assert db_sync.get(Post, ids["todd"]).absorbed_by_post_id is None
|
||||||
|
assert db_sync.execute(select(func.count(PostAttachment.id)).where(
|
||||||
|
PostAttachment.sha256 == "z" * 64)).scalar_one() == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_poster_matches_by_display_name_or_id_too(db_sync, tmp_path):
|
||||||
|
src, _ = _fixture(db_sync, tmp_path, authors=("8",))
|
||||||
|
projected = cleanup.preview(db_sync, source_id=src.id)
|
||||||
|
assert projected["posters"] == [{"poster": "Todding", "posts": 1}]
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_list_means_nothing_to_remove(db_sync, tmp_path):
|
||||||
|
src, _ = _fixture(db_sync, tmp_path, authors=())
|
||||||
|
with pytest.raises(cleanup.PosterCleanupError):
|
||||||
|
cleanup.preview(db_sync, source_id=src.id)
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_token_names_what_the_preview_showed():
|
||||||
|
assert cleanup.confirm_token({"posts": 2, "images": 1}) == "remove-2-posts-1-images"
|
||||||
Reference in New Issue
Block a user