feat(cleanup): purge misgrabbed gated-post blurred previews (#874 follow-up)
CI / lint (push) Successful in 2s
CI / frontend-build (push) Successful in 19s
CI / backend-lint-and-test (push) Successful in 31s
CI / integration (push) Successful in 3m18s

A one-shot Maintenance action to remove the blurred locked-preview images
the ingester downloaded from tier-gated Patreon posts before #874.

current_user_can_view was never persisted, so the cleanup re-walks each
enabled Patreon source (read-only) to re-derive which posts are gated now
and the blurred filehashes Patreon serves for them, then matches by
CONTENT HASH against stored source_filehash. Because the hash is
content-addressed, a real file downloaded when access existed has a
different hash and can never match — regained-then-lost-access content is
provably spared (operator's hard requirement). NULL source_filehash =>
unverifiable, kept + reported.

On apply: delete matched ImageRecords + files (provenance cascades),
clear seen/dead-letter ledger rows for those hashes so the real media
re-ingests if access returns, and delete gated posts left bare. Shares
one match predicate between preview and apply (rule 93).

- cleanup_service: collect_gated_previews + purge_gated_previews
- tasks.admin: purge_gated_previews_task (async re-walk bridge, timeboxed)
- api.admin: POST /maintenance/purge-gated-previews
- GatedPurgeCard.vue in Settings > Maintenance (preview -> confirm -> apply)
- tests: collect predicate, hash-match delete/spare/unverifiable, ledger
  clear, bare-post removal, no-op

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-06-16 15:20:35 -04:00
parent 9422eadabe
commit 540151290b
6 changed files with 650 additions and 1 deletions
+160
View File
@@ -12,8 +12,10 @@ from backend.app.models import (
Artist,
ImageProvenance,
ImageRecord,
PatreonSeenMedia,
Post,
PostAttachment,
Source,
Tag,
TagKind,
)
@@ -607,3 +609,161 @@ def test_dedup_videos_distinct_durations_not_grouped(db_sync, tmp_path):
assert db_sync.execute(
select(func.count(ImageRecord.id))
).scalar_one() == 2
# ---- Gated-post blurred-preview cleanup (#874 follow-up) ------------------
class _GatedFakeClient:
"""Stub PatreonClient for collect_gated_previews: `posts` is a list of
(post_id, gated_bool, [filehashes])."""
def __init__(self, posts):
self._posts = posts
@staticmethod
def post_is_gated(post):
return (post.get("attributes") or {}).get("current_user_can_view") is False
def iter_posts(self, campaign_id, cursor=None):
for pid, gated, hashes in self._posts:
yield (
{
"id": pid,
"attributes": {"current_user_can_view": not gated},
"_h": hashes,
},
{},
None,
)
def extract_media(self, post, included):
from backend.app.services.patreon_client import MediaItem
return [
MediaItem(
url=f"https://cdn.patreon.com/{h}/x.jpg", filename="x.jpg",
kind="images", filehash=h, post_id=str(post["id"]),
)
for h in post["_h"]
]
def test_collect_gated_previews_only_gated_posts():
client = _GatedFakeClient([
("g1", True, ["aa", "bb"]),
("open1", False, ["cc"]),
("g2", True, []), # gated but feed served no media
])
out = cleanup_service.collect_gated_previews(client, "camp")
assert out == {"g1": ["aa", "bb"], "g2": []} # accessible post excluded
def _make_src(db_sync, artist):
s = Source(
artist_id=artist.id, platform="patreon",
url="https://patreon.com/x", enabled=True, config_overrides={},
)
db_sync.add(s)
db_sync.flush()
return s
def _make_dl_image(db_sync, *, artist, path, sha256, fh, size=500):
img = ImageRecord(
artist_id=artist.id, path=path, sha256=sha256, size_bytes=size,
mime="image/jpeg", origin="downloaded", source_filehash=fh,
)
db_sync.add(img)
db_sync.flush()
return img
def test_purge_gated_previews_deletes_only_hash_matched(db_sync, tmp_path):
"""#874: hash-matched blurred previews are deleted; a real file with a
different source_filehash (downloaded when access existed) and a
null-source_filehash image are both spared. A post left bare is removed; a post
that still has real content is kept. Ledger rows for the blurred hash are
cleared (so real media can re-ingest) while the real hash stays."""
a = _make_artist(db_sync, slug="gp", name="GP")
s = _make_src(db_sync, a)
blurred = _make_dl_image(
db_sync, artist=a, path=str(tmp_path / "blur.jpg"), sha256="1" * 64,
fh="b" * 32, size=500,
)
real = _make_dl_image(
db_sync, artist=a, path=str(tmp_path / "real.jpg"), sha256="2" * 64,
fh="r" * 32, size=999,
)
nohash = _make_dl_image(
db_sync, artist=a, path=str(tmp_path / "nh.jpg"), sha256="3" * 64,
fh=None, size=100,
)
blurred2 = _make_dl_image(
db_sync, artist=a, path=str(tmp_path / "blur2.jpg"), sha256="4" * 64,
fh="c" * 32, size=700,
)
for name in ("blur.jpg", "real.jpg", "nh.jpg", "blur2.jpg"):
(tmp_path / name).write_bytes(b"x")
p1 = Post(artist_id=a.id, source_id=s.id, external_post_id="g1")
p2 = Post(artist_id=a.id, source_id=s.id, external_post_id="g2")
db_sync.add_all([p1, p2])
db_sync.flush()
# g1: blurred + real + nohash; g2: only blurred2.
for img in (blurred, real, nohash):
db_sync.add(ImageProvenance(image_record_id=img.id, post_id=p1.id, source_id=s.id))
db_sync.add(ImageProvenance(image_record_id=blurred2.id, post_id=p2.id, source_id=s.id))
db_sync.add_all([
PatreonSeenMedia(source_id=s.id, filehash="b" * 32, post_id="g1"),
PatreonSeenMedia(source_id=s.id, filehash="r" * 32, post_id="g1"),
PatreonSeenMedia(source_id=s.id, filehash="c" * 32, post_id="g2"),
])
db_sync.commit()
ids = {
"blurred": blurred.id, "real": real.id, "nohash": nohash.id,
"blurred2": blurred2.id, "p1": p1.id, "p2": p2.id,
}
# The gated feed now serves the blurred hashes for g1 + g2.
gated_map = {s.id: {"g1": ["b" * 32], "g2": ["c" * 32]}}
# Preview (rule 93): reports matches without deleting.
prev = cleanup_service.purge_gated_previews(
db_sync, gated_map=gated_map, images_root=tmp_path, dry_run=True,
)
assert prev["matched"] == 2
assert prev["gated_posts"] == 2
assert prev["unverifiable"] == 1 # the null-source_filehash image on g1
assert prev["reclaim_bytes"] == 500 + 700
assert db_sync.get(ImageRecord, ids["blurred"]) is not None # dry-run kept
# Apply.
out = cleanup_service.purge_gated_previews(
db_sync, gated_map=gated_map, images_root=tmp_path, dry_run=False,
)
assert out["deleted"] == 2
assert out["posts_deleted"] == 1 # g2 went bare; g1 still has real content
assert out["ledger_cleared"] == 2 # b* and c* cleared; r* retained
db_sync.expire_all()
assert db_sync.get(ImageRecord, ids["blurred"]) is None # blurred deleted
assert db_sync.get(ImageRecord, ids["blurred2"]) is None
assert db_sync.get(ImageRecord, ids["real"]) is not None # real spared
assert db_sync.get(ImageRecord, ids["nohash"]) is not None # unverifiable kept
assert not (tmp_path / "blur.jpg").exists()
assert (tmp_path / "real.jpg").exists()
# Ledger: blurred hashes cleared so the real media can re-ingest; real kept.
remaining = db_sync.execute(
select(PatreonSeenMedia.filehash).where(PatreonSeenMedia.source_id == s.id)
).scalars().all()
assert set(remaining) == {"r" * 32}
assert db_sync.get(Post, ids["p1"]) is not None # still has real + nohash
assert db_sync.get(Post, ids["p2"]) is None # bare → removed
def test_purge_gated_previews_no_gated_posts_is_noop(db_sync, tmp_path):
out = cleanup_service.purge_gated_previews(
db_sync, gated_map={}, images_root=tmp_path, dry_run=False,
)
assert out["matched"] == 0
assert out["deleted"] == 0
assert out["gated_posts"] == 0