feat: a Discord source can remove the posts it took from people outside its poster list (#4486)
CI and images / extension-version (push) Successful in 3s
CI and images / lint (push) Successful in 3s
CI and images / extension-test (push) Successful in 20s
CI and images / frontend-build (push) Successful in 21s
CI and images / backend-lint-and-test (push) Successful in 33s
CI and images / integration (push) Successful in 2m39s
CI and images / sign-extension (push) Successful in 3s
CI and images / build-agent (push) Successful in 6s
CI and images / build-web (push) Successful in 1m43s
CI and images / smoke-web (push) Successful in 54s
CI and images / promote (push) Successful in 2s

"Remove posts from other posters" on a Discord source with an "Only posts by"
list previews what goes, per poster, then deletes it behind a typed token:

- posts whose record names a poster not on the list (by id, username or
  display name); posts that record no poster are left alone and counted;
- images found only on those posts; one also on a kept post stays, and a
  synthetic drop's link never keeps one;
- their attachments, and every drop that absorbed one of them, so the grouping
  sweep regroups what remains.

Preview and apply share one predicate, and a parity test holds them to it.
The post record now also saves the poster's display name.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-09-28 08:34:06 -04:00
co-authored by Claude Opus 5.5
parent 20aeac81df
commit 1efece0f21
7 changed files with 587 additions and 1 deletions
+49
View File
@@ -1,10 +1,13 @@
"""FC-3a: CRUD over Source rows. FC-3c adds POST /<id>/check."""
from pathlib import Path
from quart import Blueprint, jsonify, request
from sqlalchemy import func, select
from ..extensions import get_session
from ..models import DownloadEvent, MembershipSync, PlatformMembership, Source
from ..services import discord_poster_cleanup as poster_cleanup
from ..services.artist_membership_service import ArtistMembershipService
from ..services.artist_membership_service import rescan as membership_rescan
from ..services.artist_service import ArtistService
@@ -463,3 +466,49 @@ async def adopt_membership():
return jsonify({"already_tracked": exc.existing_id})
artist_id = artist.id
return jsonify({"source_id": record.id, "artist_id": artist_id}), 201
# -- Discord: remove posts by people outside the poster list (#4486) --------------
_IMAGES_ROOT = Path("/images")
async def _poster_cleanup_preview(source_id: int):
async with get_session() as session:
return await session.run_sync(
lambda s: poster_cleanup.preview(s, source_id=source_id)
)
@sources_bp.route("/<int:source_id>/discord/other-posters", methods=["GET"])
async def other_posters_preview(source_id: int):
"""What removing other posters' posts would delete. Nothing is touched."""
try:
projection = await _poster_cleanup_preview(source_id)
except LookupError:
return _bad("not_found", status=404)
except poster_cleanup.PosterCleanupError as exc:
return _bad("not_applicable", detail=str(exc), status=409)
projection["confirm_token"] = poster_cleanup.confirm_token(projection)
return jsonify(projection)
@sources_bp.route("/<int:source_id>/discord/other-posters/remove", methods=["POST"])
async def other_posters_remove(source_id: int):
"""Delete them. `confirm` must be the token of the preview the operator saw:
a poster list edited since then changes the set, and the token with it."""
body = await request.get_json(silent=True) or {}
try:
projection = await _poster_cleanup_preview(source_id)
except LookupError:
return _bad("not_found", status=404)
except poster_cleanup.PosterCleanupError as exc:
return _bad("not_applicable", detail=str(exc), status=409)
expected = poster_cleanup.confirm_token(projection)
if body.get("confirm") != expected:
return _bad("confirm_mismatch", expected=expected)
async with get_session() as session:
result = await session.run_sync(
lambda s: poster_cleanup.apply(s, source_id=source_id, images_root=_IMAGES_ROOT)
)
return jsonify(result)
@@ -195,6 +195,7 @@ class DiscordDownloader(BaseNativeDownloader):
"parent": meta.get("parent"),
"is_thread": meta.get("is_thread"),
"author": author.get("username"),
"author_name": author.get("global_name"),
"author_id": author.get("id"),
"message": body,
"date": post.get("timestamp"),
@@ -0,0 +1,243 @@
"""Remove a Discord source's posts by people outside its poster list (#4486).
A creator's server is full of other members posting their own pictures. The
poster list (`discord_authors`, #4481) stops a walk taking them; this removes
the ones taken before the list existed. The operator sets the list first, then
previews what would go, then applies.
## What goes
A post of this source whose record names its poster (`author_id` / `author`,
written by `DiscordDownloader.write_post_record`) and names someone NOT on the
list. Two kinds of post are never touched:
* a post whose record names no poster — a message recorded before the native
ingester, whose poster FC cannot tell. Counted as `unknown`, never guessed;
* a synthetic drop post (`synthesized_by`) — FC authored it, and it is handled
below as a consequence, not matched as a poster's post.
An image goes when every REAL post it belongs to is going. A link to a
synthetic drop does not keep an image: the drop only references its members'
images. An image also on a kept post — the creator re-posting a piece someone
else shared — stays.
A drop that absorbed a removed message is deleted outright. Deleting a drop is
its undo (discord_grouping's honesty rule): its remaining members return to
the feed and the grouping sweep regroups them on its next run, so no half-
rebuilt drop keeps text or thumbnails from someone it no longer contains.
Preview and apply spread the same predicates (rule 93, snippet #3087).
"""
from __future__ import annotations
import logging
from pathlib import Path
from sqlalchemy import and_, delete, exists, func, or_, select
from sqlalchemy.orm import Session, aliased
from ..models import ImageProvenance, ImageRecord, Post, PostAttachment, Source
from .cleanup_service import delete_images
from .discord_ingester import AUTHORS_KEY
log = logging.getLogger(__name__)
PLATFORM = "discord"
# The record keys that name a message's poster. `author_name` (the display
# name) is only on records written after 2026-09-28.
_POSTER_KEYS = ("author_id", "author", "author_name")
class PosterCleanupError(ValueError):
"""The source can't be cleaned this way (not Discord, or no list)."""
def source_authors(source: Source) -> list[str]:
"""The source's poster list, lowercased; empty means everyone's wanted."""
authors = (source.config_overrides or {}).get(AUTHORS_KEY) or []
if not isinstance(authors, list):
return []
return sorted({str(a).strip().lower() for a in authors if str(a).strip()})
def _poster(key: str):
return func.lower(Post.raw_metadata[key].as_string())
def _has_poster():
return or_(*(Post.raw_metadata[k].as_string().isnot(None) for k in ("author_id", "author")))
def _real_post_conditions(source_id: int) -> list:
return [Post.source_id == source_id, Post.synthesized_by.is_(None)]
def _named_poster_conditions(source_id: int) -> list:
"""A real post of this source whose record says who posted it."""
return [*_real_post_conditions(source_id), _has_poster()]
def _other_poster_post_conditions(source_id: int, authors: list[str]) -> list:
"""The posts that go: a named poster none of whose names is on the list."""
return [
*_named_poster_conditions(source_id),
*(func.coalesce(_poster(k), "").notin_(authors) for k in _POSTER_KEYS),
]
def _doomed_post_ids(source_id: int, authors: list[str]):
# correlate(None): used inside queries that are themselves over `post` (the
# drops, the delete), where auto-correlation would bind this to the outer
# row instead of scanning the table.
return (
select(Post.id)
.where(*_other_poster_post_conditions(source_id, authors))
.correlate(None)
)
def _image_conditions(source_id: int, authors: list[str]) -> list:
"""Images that belong to a removed post and to no kept real post."""
doomed = _doomed_post_ids(source_id, authors)
keeper = aliased(Post)
on_doomed = or_(
ImageRecord.primary_post_id.in_(doomed),
exists().where(
ImageProvenance.image_record_id == ImageRecord.id,
ImageProvenance.post_id.in_(doomed),
),
)
on_kept = or_(
and_(
ImageRecord.primary_post_id.isnot(None),
ImageRecord.primary_post_id.notin_(doomed),
),
exists().where(
ImageProvenance.image_record_id == ImageRecord.id,
ImageProvenance.post_id == keeper.id,
keeper.synthesized_by.is_(None),
keeper.id.notin_(doomed),
),
)
return [on_doomed, ~on_kept]
def _drop_conditions(source_id: int, authors: list[str]) -> list:
"""The synthetic drops that absorbed a removed message."""
member = aliased(Post)
return [
Post.source_id == source_id,
Post.synthesized_by.isnot(None),
exists().where(
member.absorbed_by_post_id == Post.id,
member.id.in_(_doomed_post_ids(source_id, authors)),
),
]
def _attachment_conditions(source_id: int, authors: list[str]) -> list:
return [PostAttachment.post_id.in_(_doomed_post_ids(source_id, authors))]
def _load(session: Session, source_id: int) -> tuple[Source, list[str]]:
source = session.get(Source, source_id)
if source is None:
raise LookupError(source_id)
if source.platform != PLATFORM:
raise PosterCleanupError("Only a Discord source has posters to filter by.")
authors = source_authors(source)
if not authors:
raise PosterCleanupError(
"Set this source's 'Only posts by' list first — with no list, "
"every poster is wanted and nothing would be removed."
)
return source, authors
def _count(session: Session, stmt) -> int:
return session.execute(stmt).scalar_one()
def preview(session: Session, *, source_id: int) -> dict:
"""What `apply` would remove, per poster, without touching anything."""
_, authors = _load(session, source_id)
who = func.coalesce(
Post.raw_metadata["author_name"].as_string(),
Post.raw_metadata["author"].as_string(),
Post.raw_metadata["author_id"].as_string(),
)
rows = session.execute(
select(who, func.count(Post.id))
.where(*_other_poster_post_conditions(source_id, authors))
.group_by(who)
.order_by(func.count(Post.id).desc())
).all()
posters = [{"poster": name, "posts": n} for name, n in rows]
unknown = _count(session, select(func.count(Post.id)).where(
*_real_post_conditions(source_id), ~_has_poster(),
))
return {
"authors": authors,
"posters": posters,
"posts": sum(p["posts"] for p in posters),
"images": _count(session, select(func.count(ImageRecord.id)).where(
*_image_conditions(source_id, authors))),
"attachments": _count(session, select(func.count(PostAttachment.id)).where(
*_attachment_conditions(source_id, authors))),
"drops": _count(session, select(func.count(Post.id)).where(
*_drop_conditions(source_id, authors))),
"unknown_posts": unknown,
}
def confirm_token(projection: dict) -> str:
"""What the operator's apply must echo: the preview it was shown, so a
list edited between preview and apply can't delete an unseen set."""
return f"remove-{projection['posts']}-posts-{projection['images']}-images"
def apply(session: Session, *, source_id: int, images_root: Path) -> dict:
"""Remove them. Counts are taken before the deletes they describe — once
the rows are gone there is nothing left to count (snippet #3087 note 2)."""
projection = preview(session, source_id=source_id)
_, authors = _load(session, source_id)
image_ids = session.execute(
select(ImageRecord.id).where(*_image_conditions(source_id, authors))
).scalars().all()
# Drops first-class: found BEFORE their members go, since the link that
# finds them is the member's absorbed_by_post_id.
drop_ids = session.execute(
select(Post.id).where(*_drop_conditions(source_id, authors))
).scalars().all()
deleted = delete_images(session, image_ids=list(image_ids), images_root=images_root)
# Attachments before posts: post_attachment.post_id is SET NULL, and a
# NULL-post attachment collides on its partial unique index (see
# cleanup_service.delete_artist_cascade).
attachments = session.execute(
delete(PostAttachment).where(*_attachment_conditions(source_id, authors))
).rowcount or 0
posts = session.execute(
Post.__table__.delete().where(Post.id.in_(_doomed_post_ids(source_id, authors)))
).rowcount or 0
drops = 0
if drop_ids:
drops = session.execute(
Post.__table__.delete().where(Post.id.in_(drop_ids))
).rowcount or 0
session.commit()
log.info(
"discord poster cleanup (source %s, keeping %s): %d post(s), %d image(s), "
"%d attachment(s), %d drop(s) removed",
source_id, authors, posts, deleted["images_deleted"], attachments, drops,
)
return {
**projection,
"posts_deleted": posts,
"images_deleted": deleted["images_deleted"],
"files_deleted": deleted["files_deleted"],
"attachments_deleted": attachments,
"drops_deleted": drops,
}
@@ -0,0 +1,135 @@
<template>
<!-- #4486. A Discord source's posts by people outside its "Only posts by"
list, taken before the list existed. Preview first, always: nothing is
deleted until the operator has seen the per-poster breakdown and typed
the token that preview returned. -->
<v-dialog :model-value="modelValue" max-width="560"
@update:model-value="$emit('update:modelValue', $event)">
<v-card>
<v-card-title>Remove posts from other posters</v-card-title>
<v-card-text>
<div class="text-caption mb-2">{{ source.display_name || source.url }}</div>
<v-progress-linear v-if="loading" indeterminate color="accent" class="mb-3" />
<v-alert v-else-if="error" type="warning" variant="tonal" density="compact">
{{ error }}
</v-alert>
<template v-else-if="result">
<v-alert type="success" variant="tonal" density="compact">
Removed {{ result.posts_deleted }} post(s), {{ result.images_deleted }} image(s),
{{ result.attachments_deleted }} attachment(s) and {{ result.drops_deleted }}
grouped post(s). Grouping rebuilds what remains on its next sweep.
</v-alert>
</template>
<template v-else-if="preview">
<p class="mb-2">
Keeping posts by <strong>{{ preview.authors.join(', ') }}</strong>.
</p>
<p v-if="!preview.posts" class="fc-dim">
Nothing to remove: every post with a known poster is by someone on the list.
</p>
<template v-else>
<v-table density="compact" class="mb-3">
<thead><tr><th>Poster</th><th class="text-right">Posts</th></tr></thead>
<tbody>
<tr v-for="p in preview.posters" :key="p.poster">
<td>{{ p.poster }}</td>
<td class="text-right">{{ p.posts }}</td>
</tr>
</tbody>
</v-table>
<p class="mb-1">
{{ preview.posts }} post(s), {{ preview.images }} image(s) found only on them,
{{ preview.attachments }} attachment(s).
</p>
<p v-if="preview.drops" class="mb-1">
{{ preview.drops }} grouped post(s) that include them are ungrouped; the
rest regroup on the next sweep.
</p>
<p class="fc-dim">
Images that also belong to a kept post stay. If a name you want to keep
appears above, add it to the source's "Only posts by" list first.
</p>
</template>
<p v-if="preview.unknown_posts" class="fc-dim mt-2">
{{ preview.unknown_posts }} older post(s) don't record who posted them and
are left alone.
</p>
</template>
</v-card-text>
<v-card-actions>
<v-spacer />
<v-btn variant="text" @click="$emit('update:modelValue', false)">
{{ result ? 'Close' : 'Cancel' }}
</v-btn>
<v-btn
v-if="preview && preview.posts && !result"
color="error" variant="flat" :loading="busy"
@click="confirmOpen = true"
>Remove {{ preview.posts }} post(s)</v-btn>
</v-card-actions>
</v-card>
<DestructiveConfirmModal
v-if="preview"
v-model="confirmOpen"
action="delete" kind="other posters' posts" tier="C"
:expected-token-override="preview.confirm_token"
:projected-counts="{ posts: preview.posts, images: preview.images,
attachments: preview.attachments, grouped: preview.drops }"
@confirm="onConfirm"
/>
</v-dialog>
</template>
<script setup>
import { ref, watch } from 'vue'
import { useSourcesStore } from '../../stores/sources.js'
import DestructiveConfirmModal from '../modal/DestructiveConfirmModal.vue'
const props = defineProps({
modelValue: { type: Boolean, default: false },
source: { type: Object, required: true },
})
defineEmits(['update:modelValue'])
const store = useSourcesStore()
const loading = ref(false)
const busy = ref(false)
const error = ref('')
const preview = ref(null)
const result = ref(null)
const confirmOpen = ref(false)
function message(e) {
return e?.body?.detail || e?.body?.error || e?.message || String(e)
}
watch(() => props.modelValue, async (open) => {
if (!open) return
preview.value = null
result.value = null
error.value = ''
loading.value = true
try {
preview.value = await store.previewOtherPosters(props.source.id)
} catch (e) {
error.value = message(e)
} finally {
loading.value = false
}
})
async function onConfirm(token) {
busy.value = true
try {
result.value = await store.removeOtherPosters(props.source.id, token)
} catch (e) {
error.value = message(e)
} finally {
busy.value = false
}
}
</script>
<style scoped>
.fc-dim { color: rgb(var(--v-theme-on-surface-variant)); }
</style>
@@ -50,6 +50,17 @@
library — without re-downloading media
</v-list-item-subtitle>
</v-list-item>
<v-list-item
v-if="hasPosterList && !running"
prepend-icon="mdi-account-remove-outline"
@click="cleanupOpen = true"
>
<v-list-item-title>Remove posts from other posters</v-list-item-title>
<v-list-item-subtitle>
Preview, then delete the posts taken from people outside this
source's "Only posts by" list
</v-list-item-subtitle>
</v-list-item>
<v-divider v-if="isNative && !running" />
<v-list-item
base-color="error"
@@ -59,12 +70,14 @@
<v-list-item-title>Remove source</v-list-item-title>
</v-list-item>
</KebabMenu>
<PosterCleanupDialog v-if="hasPosterList" v-model="cleanupOpen" :source="source" />
</div>
</template>
<script setup>
import { computed } from 'vue'
import { computed, ref } from 'vue'
import KebabMenu from '../common/KebabMenu.vue'
import PosterCleanupDialog from './PosterCleanupDialog.vue'
const props = defineProps({
source: { type: Object, required: true },
@@ -80,6 +93,10 @@ const recapturing = computed(() => !!props.source.backfill_recapture)
// which those are (`native_ingester`); a copied list here went stale when
// Discord moved over.
const isNative = computed(() => !!props.source.native_ingester)
// #4486: only a Discord source with an "Only posts by" list has other posters.
const hasPosterList = computed(() => props.source.platform === 'discord'
&& (props.source.config_overrides?.discord_authors || []).length > 0)
const cleanupOpen = ref(false)
</script>
<style scoped>
+10
View File
@@ -149,6 +149,15 @@ export const useSourcesStore = defineStore('sources', () => {
return body
}
// #4486: a Discord source's posts by people outside its poster list.
// The preview touches nothing and hands back the token the remove must echo.
async function previewOtherPosters(id) {
return api.get(`/api/sources/${id}/discord/other-posters`)
}
async function removeOtherPosters(id, confirm) {
return api.post(`/api/sources/${id}/discord/other-posters/remove`, { body: { confirm } })
}
function sourcesByArtistGrouped() {
// returns [{artist: {id,name,slug}, sources: [...]}, ...]
const arr = byArtist.value.get(null) ?? []
@@ -179,6 +188,7 @@ export const useSourcesStore = defineStore('sources', () => {
stopBackfill,
recoverSource,
recaptureSource,
previewOtherPosters, removeOtherPosters,
findOrCreateArtist, autocompleteArtist, reassign,
loadScheduleStatus,
sourcesByArtistGrouped,
+131
View File
@@ -0,0 +1,131 @@
"""Removing a Discord source's posts by people outside its poster list (#4486).
Real Postgres (db_sync): the predicates read `raw_metadata->>'author'`, which
only the database can evaluate. The test that matters runs preview and apply
on one fixture and asserts they agree AND that the rows went (snippet #3087).
"""
import pytest
from sqlalchemy import func, select
from backend.app.models import (
Artist,
ImageProvenance,
ImageRecord,
Post,
PostAttachment,
Source,
)
from backend.app.services import discord_poster_cleanup as cleanup
pytestmark = pytest.mark.integration
def _image(db, artist, tmp_path, name, post):
f = tmp_path / f"{name}.png"
f.write_bytes(b"x")
img = ImageRecord(
artist_id=artist.id, path=str(f), sha256=name.ljust(64, "0"),
size_bytes=1, mime="image/png", origin="downloaded", primary_post_id=post.id,
)
db.add(img)
db.flush()
return img
def _post(db, source, mid, raw, **extra):
p = Post(
source_id=source.id, artist_id=source.artist_id, external_post_id=mid,
raw_metadata=raw, **extra,
)
db.add(p)
db.flush()
return p
def _fixture(db, tmp_path, authors=("Todding",)):
"""Todding's post, two of Jake's (one sharing an image with Todding's), an
old post with no poster, and a drop that absorbed Todding's and Jake's."""
artist = Artist(name="Todding", slug="todding-cleanup")
db.add(artist)
db.flush()
src = Source(
artist_id=artist.id, platform="discord",
url="https://discord.com/channels/1/2",
config_overrides={"discord_authors": list(authors)} if authors else None,
)
db.add(src)
db.flush()
todd = _post(db, src, "10", {"author": "todding", "author_name": "Todding", "author_id": "1"})
jake1 = _post(db, src, "11", {"author": "jakeboii", "author_id": "8"})
jake2 = _post(db, src, "12", {"author": "jakeboii", "author_id": "8"})
old = _post(db, src, "13", {"category": "discord"})
drop = _post(db, src, "fc-drop:10", None, synthesized_by="discord_drop")
todd.absorbed_by_post_id = drop.id
jake1.absorbed_by_post_id = drop.id
art = _image(db, artist, tmp_path, "art", todd)
meme = _image(db, artist, tmp_path, "meme", jake1)
shared = _image(db, artist, tmp_path, "shared", jake2)
legacy = _image(db, artist, tmp_path, "legacy", old)
db.add_all([
ImageProvenance(image_record_id=shared.id, post_id=todd.id, source_id=src.id),
ImageProvenance(image_record_id=art.id, post_id=drop.id, source_id=src.id),
ImageProvenance(image_record_id=meme.id, post_id=drop.id, source_id=src.id),
PostAttachment(
post_id=jake2.id, artist_id=artist.id, sha256="z" * 64,
path="/store/z/f.zip", original_filename="f.zip", ext=".zip", size_bytes=1,
),
])
db.commit()
return src, {
"todd": todd.id, "jake1": jake1.id, "jake2": jake2.id, "old": old.id,
"drop": drop.id, "art": art.id, "meme": meme.id, "shared": shared.id,
"legacy": legacy.id,
}
def test_preview_matches_apply_and_the_rows_go(db_sync, tmp_path):
src, ids = _fixture(db_sync, tmp_path)
projected = cleanup.preview(db_sync, source_id=src.id)
assert projected["authors"] == ["todding"]
assert projected["posters"] == [{"poster": "jakeboii", "posts": 2}]
assert projected["unknown_posts"] == 1
done = cleanup.apply(db_sync, source_id=src.id, images_root=tmp_path)
assert projected["posts"] == done["posts_deleted"] == 2
assert projected["images"] == done["images_deleted"] == 1 # the meme only
assert projected["attachments"] == done["attachments_deleted"] == 1
assert projected["drops"] == done["drops_deleted"] == 1
post_ids = set(db_sync.execute(select(Post.id).where(Post.source_id == src.id)).scalars())
assert post_ids == {ids["todd"], ids["old"]}
image_ids = set(db_sync.execute(
select(ImageRecord.id).where(ImageRecord.id.in_(
[ids["art"], ids["meme"], ids["shared"], ids["legacy"]]))
).scalars())
# The meme was only Jake's. The shared piece is also on Todding's post, and
# the drop's link to the art was never what kept it.
assert image_ids == {ids["art"], ids["shared"], ids["legacy"]}
assert not (tmp_path / "meme.png").exists()
# Todding's message is back in the feed, for the sweep to regroup.
assert db_sync.get(Post, ids["todd"]).absorbed_by_post_id is None
assert db_sync.execute(select(func.count(PostAttachment.id)).where(
PostAttachment.sha256 == "z" * 64)).scalar_one() == 0
def test_a_poster_matches_by_display_name_or_id_too(db_sync, tmp_path):
src, _ = _fixture(db_sync, tmp_path, authors=("8",))
projected = cleanup.preview(db_sync, source_id=src.id)
assert projected["posters"] == [{"poster": "Todding", "posts": 1}]
def test_no_list_means_nothing_to_remove(db_sync, tmp_path):
src, _ = _fixture(db_sync, tmp_path, authors=())
with pytest.raises(cleanup.PosterCleanupError):
cleanup.preview(db_sync, source_id=src.id)
def test_the_token_names_what_the_preview_showed():
assert cleanup.confirm_token({"posts": 2, "images": 1}) == "remove-2-posts-1-images"