feat: a Discord source can remove the posts it took from people outside its poster list (#4486)
CI and images / extension-version (push) Successful in 3s
CI and images / lint (push) Successful in 3s
CI and images / extension-test (push) Successful in 20s
CI and images / frontend-build (push) Successful in 21s
CI and images / backend-lint-and-test (push) Successful in 33s
CI and images / integration (push) Successful in 2m39s
CI and images / sign-extension (push) Successful in 3s
CI and images / build-agent (push) Successful in 6s
CI and images / build-web (push) Successful in 1m43s
CI and images / smoke-web (push) Successful in 54s
CI and images / promote (push) Successful in 2s
CI and images / extension-version (push) Successful in 3s
CI and images / lint (push) Successful in 3s
CI and images / extension-test (push) Successful in 20s
CI and images / frontend-build (push) Successful in 21s
CI and images / backend-lint-and-test (push) Successful in 33s
CI and images / integration (push) Successful in 2m39s
CI and images / sign-extension (push) Successful in 3s
CI and images / build-agent (push) Successful in 6s
CI and images / build-web (push) Successful in 1m43s
CI and images / smoke-web (push) Successful in 54s
CI and images / promote (push) Successful in 2s
"Remove posts from other posters" on a Discord source with an "Only posts by" list previews what goes, per poster, then deletes it behind a typed token: - posts whose record names a poster not on the list (by id, username or display name); posts that record no poster are left alone and counted; - images found only on those posts; one also on a kept post stays, and a synthetic drop's link never keeps one; - their attachments, and every drop that absorbed one of them, so the grouping sweep regroups what remains. Preview and apply share one predicate, and a parity test holds them to it. The post record now also saves the poster's display name. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,10 +1,13 @@
|
||||
"""FC-3a: CRUD over Source rows. FC-3c adds POST /<id>/check."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import DownloadEvent, MembershipSync, PlatformMembership, Source
|
||||
from ..services import discord_poster_cleanup as poster_cleanup
|
||||
from ..services.artist_membership_service import ArtistMembershipService
|
||||
from ..services.artist_membership_service import rescan as membership_rescan
|
||||
from ..services.artist_service import ArtistService
|
||||
@@ -463,3 +466,49 @@ async def adopt_membership():
|
||||
return jsonify({"already_tracked": exc.existing_id})
|
||||
artist_id = artist.id
|
||||
return jsonify({"source_id": record.id, "artist_id": artist_id}), 201
|
||||
|
||||
|
||||
# -- Discord: remove posts by people outside the poster list (#4486) --------------
|
||||
|
||||
_IMAGES_ROOT = Path("/images")
|
||||
|
||||
|
||||
async def _poster_cleanup_preview(source_id: int):
|
||||
async with get_session() as session:
|
||||
return await session.run_sync(
|
||||
lambda s: poster_cleanup.preview(s, source_id=source_id)
|
||||
)
|
||||
|
||||
|
||||
@sources_bp.route("/<int:source_id>/discord/other-posters", methods=["GET"])
|
||||
async def other_posters_preview(source_id: int):
|
||||
"""What removing other posters' posts would delete. Nothing is touched."""
|
||||
try:
|
||||
projection = await _poster_cleanup_preview(source_id)
|
||||
except LookupError:
|
||||
return _bad("not_found", status=404)
|
||||
except poster_cleanup.PosterCleanupError as exc:
|
||||
return _bad("not_applicable", detail=str(exc), status=409)
|
||||
projection["confirm_token"] = poster_cleanup.confirm_token(projection)
|
||||
return jsonify(projection)
|
||||
|
||||
|
||||
@sources_bp.route("/<int:source_id>/discord/other-posters/remove", methods=["POST"])
|
||||
async def other_posters_remove(source_id: int):
|
||||
"""Delete them. `confirm` must be the token of the preview the operator saw:
|
||||
a poster list edited since then changes the set, and the token with it."""
|
||||
body = await request.get_json(silent=True) or {}
|
||||
try:
|
||||
projection = await _poster_cleanup_preview(source_id)
|
||||
except LookupError:
|
||||
return _bad("not_found", status=404)
|
||||
except poster_cleanup.PosterCleanupError as exc:
|
||||
return _bad("not_applicable", detail=str(exc), status=409)
|
||||
expected = poster_cleanup.confirm_token(projection)
|
||||
if body.get("confirm") != expected:
|
||||
return _bad("confirm_mismatch", expected=expected)
|
||||
async with get_session() as session:
|
||||
result = await session.run_sync(
|
||||
lambda s: poster_cleanup.apply(s, source_id=source_id, images_root=_IMAGES_ROOT)
|
||||
)
|
||||
return jsonify(result)
|
||||
|
||||
@@ -195,6 +195,7 @@ class DiscordDownloader(BaseNativeDownloader):
|
||||
"parent": meta.get("parent"),
|
||||
"is_thread": meta.get("is_thread"),
|
||||
"author": author.get("username"),
|
||||
"author_name": author.get("global_name"),
|
||||
"author_id": author.get("id"),
|
||||
"message": body,
|
||||
"date": post.get("timestamp"),
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
"""Remove a Discord source's posts by people outside its poster list (#4486).
|
||||
|
||||
A creator's server is full of other members posting their own pictures. The
|
||||
poster list (`discord_authors`, #4481) stops a walk taking them; this removes
|
||||
the ones taken before the list existed. The operator sets the list first, then
|
||||
previews what would go, then applies.
|
||||
|
||||
## What goes
|
||||
|
||||
A post of this source whose record names its poster (`author_id` / `author`,
|
||||
written by `DiscordDownloader.write_post_record`) and names someone NOT on the
|
||||
list. Two kinds of post are never touched:
|
||||
|
||||
* a post whose record names no poster — a message recorded before the native
|
||||
ingester, whose poster FC cannot tell. Counted as `unknown`, never guessed;
|
||||
* a synthetic drop post (`synthesized_by`) — FC authored it, and it is handled
|
||||
below as a consequence, not matched as a poster's post.
|
||||
|
||||
An image goes when every REAL post it belongs to is going. A link to a
|
||||
synthetic drop does not keep an image: the drop only references its members'
|
||||
images. An image also on a kept post — the creator re-posting a piece someone
|
||||
else shared — stays.
|
||||
|
||||
A drop that absorbed a removed message is deleted outright. Deleting a drop is
|
||||
its undo (discord_grouping's honesty rule): its remaining members return to
|
||||
the feed and the grouping sweep regroups them on its next run, so no half-
|
||||
rebuilt drop keeps text or thumbnails from someone it no longer contains.
|
||||
|
||||
Preview and apply spread the same predicates (rule 93, snippet #3087).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import and_, delete, exists, func, or_, select
|
||||
from sqlalchemy.orm import Session, aliased
|
||||
|
||||
from ..models import ImageProvenance, ImageRecord, Post, PostAttachment, Source
|
||||
from .cleanup_service import delete_images
|
||||
from .discord_ingester import AUTHORS_KEY
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
PLATFORM = "discord"
|
||||
# The record keys that name a message's poster. `author_name` (the display
|
||||
# name) is only on records written after 2026-09-28.
|
||||
_POSTER_KEYS = ("author_id", "author", "author_name")
|
||||
|
||||
|
||||
class PosterCleanupError(ValueError):
|
||||
"""The source can't be cleaned this way (not Discord, or no list)."""
|
||||
|
||||
|
||||
def source_authors(source: Source) -> list[str]:
|
||||
"""The source's poster list, lowercased; empty means everyone's wanted."""
|
||||
authors = (source.config_overrides or {}).get(AUTHORS_KEY) or []
|
||||
if not isinstance(authors, list):
|
||||
return []
|
||||
return sorted({str(a).strip().lower() for a in authors if str(a).strip()})
|
||||
|
||||
|
||||
def _poster(key: str):
|
||||
return func.lower(Post.raw_metadata[key].as_string())
|
||||
|
||||
|
||||
def _has_poster():
|
||||
return or_(*(Post.raw_metadata[k].as_string().isnot(None) for k in ("author_id", "author")))
|
||||
|
||||
|
||||
def _real_post_conditions(source_id: int) -> list:
|
||||
return [Post.source_id == source_id, Post.synthesized_by.is_(None)]
|
||||
|
||||
|
||||
def _named_poster_conditions(source_id: int) -> list:
|
||||
"""A real post of this source whose record says who posted it."""
|
||||
return [*_real_post_conditions(source_id), _has_poster()]
|
||||
|
||||
|
||||
def _other_poster_post_conditions(source_id: int, authors: list[str]) -> list:
|
||||
"""The posts that go: a named poster none of whose names is on the list."""
|
||||
return [
|
||||
*_named_poster_conditions(source_id),
|
||||
*(func.coalesce(_poster(k), "").notin_(authors) for k in _POSTER_KEYS),
|
||||
]
|
||||
|
||||
|
||||
def _doomed_post_ids(source_id: int, authors: list[str]):
|
||||
# correlate(None): used inside queries that are themselves over `post` (the
|
||||
# drops, the delete), where auto-correlation would bind this to the outer
|
||||
# row instead of scanning the table.
|
||||
return (
|
||||
select(Post.id)
|
||||
.where(*_other_poster_post_conditions(source_id, authors))
|
||||
.correlate(None)
|
||||
)
|
||||
|
||||
|
||||
def _image_conditions(source_id: int, authors: list[str]) -> list:
|
||||
"""Images that belong to a removed post and to no kept real post."""
|
||||
doomed = _doomed_post_ids(source_id, authors)
|
||||
keeper = aliased(Post)
|
||||
on_doomed = or_(
|
||||
ImageRecord.primary_post_id.in_(doomed),
|
||||
exists().where(
|
||||
ImageProvenance.image_record_id == ImageRecord.id,
|
||||
ImageProvenance.post_id.in_(doomed),
|
||||
),
|
||||
)
|
||||
on_kept = or_(
|
||||
and_(
|
||||
ImageRecord.primary_post_id.isnot(None),
|
||||
ImageRecord.primary_post_id.notin_(doomed),
|
||||
),
|
||||
exists().where(
|
||||
ImageProvenance.image_record_id == ImageRecord.id,
|
||||
ImageProvenance.post_id == keeper.id,
|
||||
keeper.synthesized_by.is_(None),
|
||||
keeper.id.notin_(doomed),
|
||||
),
|
||||
)
|
||||
return [on_doomed, ~on_kept]
|
||||
|
||||
|
||||
def _drop_conditions(source_id: int, authors: list[str]) -> list:
|
||||
"""The synthetic drops that absorbed a removed message."""
|
||||
member = aliased(Post)
|
||||
return [
|
||||
Post.source_id == source_id,
|
||||
Post.synthesized_by.isnot(None),
|
||||
exists().where(
|
||||
member.absorbed_by_post_id == Post.id,
|
||||
member.id.in_(_doomed_post_ids(source_id, authors)),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _attachment_conditions(source_id: int, authors: list[str]) -> list:
|
||||
return [PostAttachment.post_id.in_(_doomed_post_ids(source_id, authors))]
|
||||
|
||||
|
||||
def _load(session: Session, source_id: int) -> tuple[Source, list[str]]:
|
||||
source = session.get(Source, source_id)
|
||||
if source is None:
|
||||
raise LookupError(source_id)
|
||||
if source.platform != PLATFORM:
|
||||
raise PosterCleanupError("Only a Discord source has posters to filter by.")
|
||||
authors = source_authors(source)
|
||||
if not authors:
|
||||
raise PosterCleanupError(
|
||||
"Set this source's 'Only posts by' list first — with no list, "
|
||||
"every poster is wanted and nothing would be removed."
|
||||
)
|
||||
return source, authors
|
||||
|
||||
|
||||
def _count(session: Session, stmt) -> int:
|
||||
return session.execute(stmt).scalar_one()
|
||||
|
||||
|
||||
def preview(session: Session, *, source_id: int) -> dict:
|
||||
"""What `apply` would remove, per poster, without touching anything."""
|
||||
_, authors = _load(session, source_id)
|
||||
who = func.coalesce(
|
||||
Post.raw_metadata["author_name"].as_string(),
|
||||
Post.raw_metadata["author"].as_string(),
|
||||
Post.raw_metadata["author_id"].as_string(),
|
||||
)
|
||||
rows = session.execute(
|
||||
select(who, func.count(Post.id))
|
||||
.where(*_other_poster_post_conditions(source_id, authors))
|
||||
.group_by(who)
|
||||
.order_by(func.count(Post.id).desc())
|
||||
).all()
|
||||
posters = [{"poster": name, "posts": n} for name, n in rows]
|
||||
unknown = _count(session, select(func.count(Post.id)).where(
|
||||
*_real_post_conditions(source_id), ~_has_poster(),
|
||||
))
|
||||
return {
|
||||
"authors": authors,
|
||||
"posters": posters,
|
||||
"posts": sum(p["posts"] for p in posters),
|
||||
"images": _count(session, select(func.count(ImageRecord.id)).where(
|
||||
*_image_conditions(source_id, authors))),
|
||||
"attachments": _count(session, select(func.count(PostAttachment.id)).where(
|
||||
*_attachment_conditions(source_id, authors))),
|
||||
"drops": _count(session, select(func.count(Post.id)).where(
|
||||
*_drop_conditions(source_id, authors))),
|
||||
"unknown_posts": unknown,
|
||||
}
|
||||
|
||||
|
||||
def confirm_token(projection: dict) -> str:
|
||||
"""What the operator's apply must echo: the preview it was shown, so a
|
||||
list edited between preview and apply can't delete an unseen set."""
|
||||
return f"remove-{projection['posts']}-posts-{projection['images']}-images"
|
||||
|
||||
|
||||
def apply(session: Session, *, source_id: int, images_root: Path) -> dict:
|
||||
"""Remove them. Counts are taken before the deletes they describe — once
|
||||
the rows are gone there is nothing left to count (snippet #3087 note 2)."""
|
||||
projection = preview(session, source_id=source_id)
|
||||
_, authors = _load(session, source_id)
|
||||
|
||||
image_ids = session.execute(
|
||||
select(ImageRecord.id).where(*_image_conditions(source_id, authors))
|
||||
).scalars().all()
|
||||
# Drops first-class: found BEFORE their members go, since the link that
|
||||
# finds them is the member's absorbed_by_post_id.
|
||||
drop_ids = session.execute(
|
||||
select(Post.id).where(*_drop_conditions(source_id, authors))
|
||||
).scalars().all()
|
||||
|
||||
deleted = delete_images(session, image_ids=list(image_ids), images_root=images_root)
|
||||
# Attachments before posts: post_attachment.post_id is SET NULL, and a
|
||||
# NULL-post attachment collides on its partial unique index (see
|
||||
# cleanup_service.delete_artist_cascade).
|
||||
attachments = session.execute(
|
||||
delete(PostAttachment).where(*_attachment_conditions(source_id, authors))
|
||||
).rowcount or 0
|
||||
posts = session.execute(
|
||||
Post.__table__.delete().where(Post.id.in_(_doomed_post_ids(source_id, authors)))
|
||||
).rowcount or 0
|
||||
drops = 0
|
||||
if drop_ids:
|
||||
drops = session.execute(
|
||||
Post.__table__.delete().where(Post.id.in_(drop_ids))
|
||||
).rowcount or 0
|
||||
session.commit()
|
||||
log.info(
|
||||
"discord poster cleanup (source %s, keeping %s): %d post(s), %d image(s), "
|
||||
"%d attachment(s), %d drop(s) removed",
|
||||
source_id, authors, posts, deleted["images_deleted"], attachments, drops,
|
||||
)
|
||||
return {
|
||||
**projection,
|
||||
"posts_deleted": posts,
|
||||
"images_deleted": deleted["images_deleted"],
|
||||
"files_deleted": deleted["files_deleted"],
|
||||
"attachments_deleted": attachments,
|
||||
"drops_deleted": drops,
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
<template>
|
||||
<!-- #4486. A Discord source's posts by people outside its "Only posts by"
|
||||
list, taken before the list existed. Preview first, always: nothing is
|
||||
deleted until the operator has seen the per-poster breakdown and typed
|
||||
the token that preview returned. -->
|
||||
<v-dialog :model-value="modelValue" max-width="560"
|
||||
@update:model-value="$emit('update:modelValue', $event)">
|
||||
<v-card>
|
||||
<v-card-title>Remove posts from other posters</v-card-title>
|
||||
<v-card-text>
|
||||
<div class="text-caption mb-2">{{ source.display_name || source.url }}</div>
|
||||
<v-progress-linear v-if="loading" indeterminate color="accent" class="mb-3" />
|
||||
<v-alert v-else-if="error" type="warning" variant="tonal" density="compact">
|
||||
{{ error }}
|
||||
</v-alert>
|
||||
<template v-else-if="result">
|
||||
<v-alert type="success" variant="tonal" density="compact">
|
||||
Removed {{ result.posts_deleted }} post(s), {{ result.images_deleted }} image(s),
|
||||
{{ result.attachments_deleted }} attachment(s) and {{ result.drops_deleted }}
|
||||
grouped post(s). Grouping rebuilds what remains on its next sweep.
|
||||
</v-alert>
|
||||
</template>
|
||||
<template v-else-if="preview">
|
||||
<p class="mb-2">
|
||||
Keeping posts by <strong>{{ preview.authors.join(', ') }}</strong>.
|
||||
</p>
|
||||
<p v-if="!preview.posts" class="fc-dim">
|
||||
Nothing to remove: every post with a known poster is by someone on the list.
|
||||
</p>
|
||||
<template v-else>
|
||||
<v-table density="compact" class="mb-3">
|
||||
<thead><tr><th>Poster</th><th class="text-right">Posts</th></tr></thead>
|
||||
<tbody>
|
||||
<tr v-for="p in preview.posters" :key="p.poster">
|
||||
<td>{{ p.poster }}</td>
|
||||
<td class="text-right">{{ p.posts }}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</v-table>
|
||||
<p class="mb-1">
|
||||
{{ preview.posts }} post(s), {{ preview.images }} image(s) found only on them,
|
||||
{{ preview.attachments }} attachment(s).
|
||||
</p>
|
||||
<p v-if="preview.drops" class="mb-1">
|
||||
{{ preview.drops }} grouped post(s) that include them are ungrouped; the
|
||||
rest regroup on the next sweep.
|
||||
</p>
|
||||
<p class="fc-dim">
|
||||
Images that also belong to a kept post stay. If a name you want to keep
|
||||
appears above, add it to the source's "Only posts by" list first.
|
||||
</p>
|
||||
</template>
|
||||
<p v-if="preview.unknown_posts" class="fc-dim mt-2">
|
||||
{{ preview.unknown_posts }} older post(s) don't record who posted them and
|
||||
are left alone.
|
||||
</p>
|
||||
</template>
|
||||
</v-card-text>
|
||||
<v-card-actions>
|
||||
<v-spacer />
|
||||
<v-btn variant="text" @click="$emit('update:modelValue', false)">
|
||||
{{ result ? 'Close' : 'Cancel' }}
|
||||
</v-btn>
|
||||
<v-btn
|
||||
v-if="preview && preview.posts && !result"
|
||||
color="error" variant="flat" :loading="busy"
|
||||
@click="confirmOpen = true"
|
||||
>Remove {{ preview.posts }} post(s)</v-btn>
|
||||
</v-card-actions>
|
||||
</v-card>
|
||||
<DestructiveConfirmModal
|
||||
v-if="preview"
|
||||
v-model="confirmOpen"
|
||||
action="delete" kind="other posters' posts" tier="C"
|
||||
:expected-token-override="preview.confirm_token"
|
||||
:projected-counts="{ posts: preview.posts, images: preview.images,
|
||||
attachments: preview.attachments, grouped: preview.drops }"
|
||||
@confirm="onConfirm"
|
||||
/>
|
||||
</v-dialog>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { ref, watch } from 'vue'
|
||||
import { useSourcesStore } from '../../stores/sources.js'
|
||||
import DestructiveConfirmModal from '../modal/DestructiveConfirmModal.vue'
|
||||
|
||||
const props = defineProps({
|
||||
modelValue: { type: Boolean, default: false },
|
||||
source: { type: Object, required: true },
|
||||
})
|
||||
defineEmits(['update:modelValue'])
|
||||
const store = useSourcesStore()
|
||||
|
||||
const loading = ref(false)
|
||||
const busy = ref(false)
|
||||
const error = ref('')
|
||||
const preview = ref(null)
|
||||
const result = ref(null)
|
||||
const confirmOpen = ref(false)
|
||||
|
||||
function message(e) {
|
||||
return e?.body?.detail || e?.body?.error || e?.message || String(e)
|
||||
}
|
||||
|
||||
watch(() => props.modelValue, async (open) => {
|
||||
if (!open) return
|
||||
preview.value = null
|
||||
result.value = null
|
||||
error.value = ''
|
||||
loading.value = true
|
||||
try {
|
||||
preview.value = await store.previewOtherPosters(props.source.id)
|
||||
} catch (e) {
|
||||
error.value = message(e)
|
||||
} finally {
|
||||
loading.value = false
|
||||
}
|
||||
})
|
||||
|
||||
async function onConfirm(token) {
|
||||
busy.value = true
|
||||
try {
|
||||
result.value = await store.removeOtherPosters(props.source.id, token)
|
||||
} catch (e) {
|
||||
error.value = message(e)
|
||||
} finally {
|
||||
busy.value = false
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-dim { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||
</style>
|
||||
@@ -50,6 +50,17 @@
|
||||
library — without re-downloading media
|
||||
</v-list-item-subtitle>
|
||||
</v-list-item>
|
||||
<v-list-item
|
||||
v-if="hasPosterList && !running"
|
||||
prepend-icon="mdi-account-remove-outline"
|
||||
@click="cleanupOpen = true"
|
||||
>
|
||||
<v-list-item-title>Remove posts from other posters</v-list-item-title>
|
||||
<v-list-item-subtitle>
|
||||
Preview, then delete the posts taken from people outside this
|
||||
source's "Only posts by" list
|
||||
</v-list-item-subtitle>
|
||||
</v-list-item>
|
||||
<v-divider v-if="isNative && !running" />
|
||||
<v-list-item
|
||||
base-color="error"
|
||||
@@ -59,12 +70,14 @@
|
||||
<v-list-item-title>Remove source</v-list-item-title>
|
||||
</v-list-item>
|
||||
</KebabMenu>
|
||||
<PosterCleanupDialog v-if="hasPosterList" v-model="cleanupOpen" :source="source" />
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed } from 'vue'
|
||||
import { computed, ref } from 'vue'
|
||||
import KebabMenu from '../common/KebabMenu.vue'
|
||||
import PosterCleanupDialog from './PosterCleanupDialog.vue'
|
||||
|
||||
const props = defineProps({
|
||||
source: { type: Object, required: true },
|
||||
@@ -80,6 +93,10 @@ const recapturing = computed(() => !!props.source.backfill_recapture)
|
||||
// which those are (`native_ingester`); a copied list here went stale when
|
||||
// Discord moved over.
|
||||
const isNative = computed(() => !!props.source.native_ingester)
|
||||
// #4486: only a Discord source with an "Only posts by" list has other posters.
|
||||
const hasPosterList = computed(() => props.source.platform === 'discord'
|
||||
&& (props.source.config_overrides?.discord_authors || []).length > 0)
|
||||
const cleanupOpen = ref(false)
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
|
||||
@@ -149,6 +149,15 @@ export const useSourcesStore = defineStore('sources', () => {
|
||||
return body
|
||||
}
|
||||
|
||||
// #4486: a Discord source's posts by people outside its poster list.
|
||||
// The preview touches nothing and hands back the token the remove must echo.
|
||||
async function previewOtherPosters(id) {
|
||||
return api.get(`/api/sources/${id}/discord/other-posters`)
|
||||
}
|
||||
async function removeOtherPosters(id, confirm) {
|
||||
return api.post(`/api/sources/${id}/discord/other-posters/remove`, { body: { confirm } })
|
||||
}
|
||||
|
||||
function sourcesByArtistGrouped() {
|
||||
// returns [{artist: {id,name,slug}, sources: [...]}, ...]
|
||||
const arr = byArtist.value.get(null) ?? []
|
||||
@@ -179,6 +188,7 @@ export const useSourcesStore = defineStore('sources', () => {
|
||||
stopBackfill,
|
||||
recoverSource,
|
||||
recaptureSource,
|
||||
previewOtherPosters, removeOtherPosters,
|
||||
findOrCreateArtist, autocompleteArtist, reassign,
|
||||
loadScheduleStatus,
|
||||
sourcesByArtistGrouped,
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
"""Removing a Discord source's posts by people outside its poster list (#4486).
|
||||
|
||||
Real Postgres (db_sync): the predicates read `raw_metadata->>'author'`, which
|
||||
only the database can evaluate. The test that matters runs preview and apply
|
||||
on one fixture and asserts they agree AND that the rows went (snippet #3087).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from backend.app.models import (
|
||||
Artist,
|
||||
ImageProvenance,
|
||||
ImageRecord,
|
||||
Post,
|
||||
PostAttachment,
|
||||
Source,
|
||||
)
|
||||
from backend.app.services import discord_poster_cleanup as cleanup
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
def _image(db, artist, tmp_path, name, post):
|
||||
f = tmp_path / f"{name}.png"
|
||||
f.write_bytes(b"x")
|
||||
img = ImageRecord(
|
||||
artist_id=artist.id, path=str(f), sha256=name.ljust(64, "0"),
|
||||
size_bytes=1, mime="image/png", origin="downloaded", primary_post_id=post.id,
|
||||
)
|
||||
db.add(img)
|
||||
db.flush()
|
||||
return img
|
||||
|
||||
|
||||
def _post(db, source, mid, raw, **extra):
|
||||
p = Post(
|
||||
source_id=source.id, artist_id=source.artist_id, external_post_id=mid,
|
||||
raw_metadata=raw, **extra,
|
||||
)
|
||||
db.add(p)
|
||||
db.flush()
|
||||
return p
|
||||
|
||||
|
||||
def _fixture(db, tmp_path, authors=("Todding",)):
|
||||
"""Todding's post, two of Jake's (one sharing an image with Todding's), an
|
||||
old post with no poster, and a drop that absorbed Todding's and Jake's."""
|
||||
artist = Artist(name="Todding", slug="todding-cleanup")
|
||||
db.add(artist)
|
||||
db.flush()
|
||||
src = Source(
|
||||
artist_id=artist.id, platform="discord",
|
||||
url="https://discord.com/channels/1/2",
|
||||
config_overrides={"discord_authors": list(authors)} if authors else None,
|
||||
)
|
||||
db.add(src)
|
||||
db.flush()
|
||||
todd = _post(db, src, "10", {"author": "todding", "author_name": "Todding", "author_id": "1"})
|
||||
jake1 = _post(db, src, "11", {"author": "jakeboii", "author_id": "8"})
|
||||
jake2 = _post(db, src, "12", {"author": "jakeboii", "author_id": "8"})
|
||||
old = _post(db, src, "13", {"category": "discord"})
|
||||
drop = _post(db, src, "fc-drop:10", None, synthesized_by="discord_drop")
|
||||
todd.absorbed_by_post_id = drop.id
|
||||
jake1.absorbed_by_post_id = drop.id
|
||||
|
||||
art = _image(db, artist, tmp_path, "art", todd)
|
||||
meme = _image(db, artist, tmp_path, "meme", jake1)
|
||||
shared = _image(db, artist, tmp_path, "shared", jake2)
|
||||
legacy = _image(db, artist, tmp_path, "legacy", old)
|
||||
db.add_all([
|
||||
ImageProvenance(image_record_id=shared.id, post_id=todd.id, source_id=src.id),
|
||||
ImageProvenance(image_record_id=art.id, post_id=drop.id, source_id=src.id),
|
||||
ImageProvenance(image_record_id=meme.id, post_id=drop.id, source_id=src.id),
|
||||
PostAttachment(
|
||||
post_id=jake2.id, artist_id=artist.id, sha256="z" * 64,
|
||||
path="/store/z/f.zip", original_filename="f.zip", ext=".zip", size_bytes=1,
|
||||
),
|
||||
])
|
||||
db.commit()
|
||||
return src, {
|
||||
"todd": todd.id, "jake1": jake1.id, "jake2": jake2.id, "old": old.id,
|
||||
"drop": drop.id, "art": art.id, "meme": meme.id, "shared": shared.id,
|
||||
"legacy": legacy.id,
|
||||
}
|
||||
|
||||
|
||||
def test_preview_matches_apply_and_the_rows_go(db_sync, tmp_path):
|
||||
src, ids = _fixture(db_sync, tmp_path)
|
||||
|
||||
projected = cleanup.preview(db_sync, source_id=src.id)
|
||||
assert projected["authors"] == ["todding"]
|
||||
assert projected["posters"] == [{"poster": "jakeboii", "posts": 2}]
|
||||
assert projected["unknown_posts"] == 1
|
||||
|
||||
done = cleanup.apply(db_sync, source_id=src.id, images_root=tmp_path)
|
||||
assert projected["posts"] == done["posts_deleted"] == 2
|
||||
assert projected["images"] == done["images_deleted"] == 1 # the meme only
|
||||
assert projected["attachments"] == done["attachments_deleted"] == 1
|
||||
assert projected["drops"] == done["drops_deleted"] == 1
|
||||
|
||||
post_ids = set(db_sync.execute(select(Post.id).where(Post.source_id == src.id)).scalars())
|
||||
assert post_ids == {ids["todd"], ids["old"]}
|
||||
image_ids = set(db_sync.execute(
|
||||
select(ImageRecord.id).where(ImageRecord.id.in_(
|
||||
[ids["art"], ids["meme"], ids["shared"], ids["legacy"]]))
|
||||
).scalars())
|
||||
# The meme was only Jake's. The shared piece is also on Todding's post, and
|
||||
# the drop's link to the art was never what kept it.
|
||||
assert image_ids == {ids["art"], ids["shared"], ids["legacy"]}
|
||||
assert not (tmp_path / "meme.png").exists()
|
||||
# Todding's message is back in the feed, for the sweep to regroup.
|
||||
assert db_sync.get(Post, ids["todd"]).absorbed_by_post_id is None
|
||||
assert db_sync.execute(select(func.count(PostAttachment.id)).where(
|
||||
PostAttachment.sha256 == "z" * 64)).scalar_one() == 0
|
||||
|
||||
|
||||
def test_a_poster_matches_by_display_name_or_id_too(db_sync, tmp_path):
|
||||
src, _ = _fixture(db_sync, tmp_path, authors=("8",))
|
||||
projected = cleanup.preview(db_sync, source_id=src.id)
|
||||
assert projected["posters"] == [{"poster": "Todding", "posts": 1}]
|
||||
|
||||
|
||||
def test_no_list_means_nothing_to_remove(db_sync, tmp_path):
|
||||
src, _ = _fixture(db_sync, tmp_path, authors=())
|
||||
with pytest.raises(cleanup.PosterCleanupError):
|
||||
cleanup.preview(db_sync, source_id=src.id)
|
||||
|
||||
|
||||
def test_the_token_names_what_the_preview_showed():
|
||||
assert cleanup.confirm_token({"posts": 2, "images": 1}) == "remove-2-posts-1-images"
|
||||
Reference in New Issue
Block a user