feat(cleanup): purge misgrabbed gated-post blurred previews (#874 follow-up)
A one-shot Maintenance action to remove the blurred locked-preview images the ingester downloaded from tier-gated Patreon posts before #874. current_user_can_view was never persisted, so the cleanup re-walks each enabled Patreon source (read-only) to re-derive which posts are gated now and the blurred filehashes Patreon serves for them, then matches by CONTENT HASH against stored source_filehash. Because the hash is content-addressed, a real file downloaded when access existed has a different hash and can never match — regained-then-lost-access content is provably spared (operator's hard requirement). NULL source_filehash => unverifiable, kept + reported. On apply: delete matched ImageRecords + files (provenance cascades), clear seen/dead-letter ledger rows for those hashes so the real media re-ingests if access returns, and delete gated posts left bare. Shares one match predicate between preview and apply (rule 93). - cleanup_service: collect_gated_previews + purge_gated_previews - tasks.admin: purge_gated_previews_task (async re-walk bridge, timeboxed) - api.admin: POST /maintenance/purge-gated-previews - GatedPurgeCard.vue in Settings > Maintenance (preview -> confirm -> apply) - tests: collect predicate, hash-match delete/spare/unverifiable, ledger clear, bare-post removal, no-op Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -12,8 +12,10 @@ from backend.app.models import (
|
||||
Artist,
|
||||
ImageProvenance,
|
||||
ImageRecord,
|
||||
PatreonSeenMedia,
|
||||
Post,
|
||||
PostAttachment,
|
||||
Source,
|
||||
Tag,
|
||||
TagKind,
|
||||
)
|
||||
@@ -607,3 +609,161 @@ def test_dedup_videos_distinct_durations_not_grouped(db_sync, tmp_path):
|
||||
assert db_sync.execute(
|
||||
select(func.count(ImageRecord.id))
|
||||
).scalar_one() == 2
|
||||
|
||||
|
||||
# ---- Gated-post blurred-preview cleanup (#874 follow-up) ------------------
|
||||
|
||||
|
||||
class _GatedFakeClient:
|
||||
"""Stub PatreonClient for collect_gated_previews: `posts` is a list of
|
||||
(post_id, gated_bool, [filehashes])."""
|
||||
|
||||
def __init__(self, posts):
|
||||
self._posts = posts
|
||||
|
||||
@staticmethod
|
||||
def post_is_gated(post):
|
||||
return (post.get("attributes") or {}).get("current_user_can_view") is False
|
||||
|
||||
def iter_posts(self, campaign_id, cursor=None):
|
||||
for pid, gated, hashes in self._posts:
|
||||
yield (
|
||||
{
|
||||
"id": pid,
|
||||
"attributes": {"current_user_can_view": not gated},
|
||||
"_h": hashes,
|
||||
},
|
||||
{},
|
||||
None,
|
||||
)
|
||||
|
||||
def extract_media(self, post, included):
|
||||
from backend.app.services.patreon_client import MediaItem
|
||||
return [
|
||||
MediaItem(
|
||||
url=f"https://cdn.patreon.com/{h}/x.jpg", filename="x.jpg",
|
||||
kind="images", filehash=h, post_id=str(post["id"]),
|
||||
)
|
||||
for h in post["_h"]
|
||||
]
|
||||
|
||||
|
||||
def test_collect_gated_previews_only_gated_posts():
|
||||
client = _GatedFakeClient([
|
||||
("g1", True, ["aa", "bb"]),
|
||||
("open1", False, ["cc"]),
|
||||
("g2", True, []), # gated but feed served no media
|
||||
])
|
||||
out = cleanup_service.collect_gated_previews(client, "camp")
|
||||
assert out == {"g1": ["aa", "bb"], "g2": []} # accessible post excluded
|
||||
|
||||
|
||||
def _make_src(db_sync, artist):
|
||||
s = Source(
|
||||
artist_id=artist.id, platform="patreon",
|
||||
url="https://patreon.com/x", enabled=True, config_overrides={},
|
||||
)
|
||||
db_sync.add(s)
|
||||
db_sync.flush()
|
||||
return s
|
||||
|
||||
|
||||
def _make_dl_image(db_sync, *, artist, path, sha256, fh, size=500):
|
||||
img = ImageRecord(
|
||||
artist_id=artist.id, path=path, sha256=sha256, size_bytes=size,
|
||||
mime="image/jpeg", origin="downloaded", source_filehash=fh,
|
||||
)
|
||||
db_sync.add(img)
|
||||
db_sync.flush()
|
||||
return img
|
||||
|
||||
|
||||
def test_purge_gated_previews_deletes_only_hash_matched(db_sync, tmp_path):
|
||||
"""#874: hash-matched blurred previews are deleted; a real file with a
|
||||
different source_filehash (downloaded when access existed) and a
|
||||
null-source_filehash image are both spared. A post left bare is removed; a post
|
||||
that still has real content is kept. Ledger rows for the blurred hash are
|
||||
cleared (so real media can re-ingest) while the real hash stays."""
|
||||
a = _make_artist(db_sync, slug="gp", name="GP")
|
||||
s = _make_src(db_sync, a)
|
||||
blurred = _make_dl_image(
|
||||
db_sync, artist=a, path=str(tmp_path / "blur.jpg"), sha256="1" * 64,
|
||||
fh="b" * 32, size=500,
|
||||
)
|
||||
real = _make_dl_image(
|
||||
db_sync, artist=a, path=str(tmp_path / "real.jpg"), sha256="2" * 64,
|
||||
fh="r" * 32, size=999,
|
||||
)
|
||||
nohash = _make_dl_image(
|
||||
db_sync, artist=a, path=str(tmp_path / "nh.jpg"), sha256="3" * 64,
|
||||
fh=None, size=100,
|
||||
)
|
||||
blurred2 = _make_dl_image(
|
||||
db_sync, artist=a, path=str(tmp_path / "blur2.jpg"), sha256="4" * 64,
|
||||
fh="c" * 32, size=700,
|
||||
)
|
||||
for name in ("blur.jpg", "real.jpg", "nh.jpg", "blur2.jpg"):
|
||||
(tmp_path / name).write_bytes(b"x")
|
||||
p1 = Post(artist_id=a.id, source_id=s.id, external_post_id="g1")
|
||||
p2 = Post(artist_id=a.id, source_id=s.id, external_post_id="g2")
|
||||
db_sync.add_all([p1, p2])
|
||||
db_sync.flush()
|
||||
# g1: blurred + real + nohash; g2: only blurred2.
|
||||
for img in (blurred, real, nohash):
|
||||
db_sync.add(ImageProvenance(image_record_id=img.id, post_id=p1.id, source_id=s.id))
|
||||
db_sync.add(ImageProvenance(image_record_id=blurred2.id, post_id=p2.id, source_id=s.id))
|
||||
db_sync.add_all([
|
||||
PatreonSeenMedia(source_id=s.id, filehash="b" * 32, post_id="g1"),
|
||||
PatreonSeenMedia(source_id=s.id, filehash="r" * 32, post_id="g1"),
|
||||
PatreonSeenMedia(source_id=s.id, filehash="c" * 32, post_id="g2"),
|
||||
])
|
||||
db_sync.commit()
|
||||
ids = {
|
||||
"blurred": blurred.id, "real": real.id, "nohash": nohash.id,
|
||||
"blurred2": blurred2.id, "p1": p1.id, "p2": p2.id,
|
||||
}
|
||||
|
||||
# The gated feed now serves the blurred hashes for g1 + g2.
|
||||
gated_map = {s.id: {"g1": ["b" * 32], "g2": ["c" * 32]}}
|
||||
|
||||
# Preview (rule 93): reports matches without deleting.
|
||||
prev = cleanup_service.purge_gated_previews(
|
||||
db_sync, gated_map=gated_map, images_root=tmp_path, dry_run=True,
|
||||
)
|
||||
assert prev["matched"] == 2
|
||||
assert prev["gated_posts"] == 2
|
||||
assert prev["unverifiable"] == 1 # the null-source_filehash image on g1
|
||||
assert prev["reclaim_bytes"] == 500 + 700
|
||||
assert db_sync.get(ImageRecord, ids["blurred"]) is not None # dry-run kept
|
||||
|
||||
# Apply.
|
||||
out = cleanup_service.purge_gated_previews(
|
||||
db_sync, gated_map=gated_map, images_root=tmp_path, dry_run=False,
|
||||
)
|
||||
assert out["deleted"] == 2
|
||||
assert out["posts_deleted"] == 1 # g2 went bare; g1 still has real content
|
||||
assert out["ledger_cleared"] == 2 # b* and c* cleared; r* retained
|
||||
|
||||
db_sync.expire_all()
|
||||
assert db_sync.get(ImageRecord, ids["blurred"]) is None # blurred deleted
|
||||
assert db_sync.get(ImageRecord, ids["blurred2"]) is None
|
||||
assert db_sync.get(ImageRecord, ids["real"]) is not None # real spared
|
||||
assert db_sync.get(ImageRecord, ids["nohash"]) is not None # unverifiable kept
|
||||
assert not (tmp_path / "blur.jpg").exists()
|
||||
assert (tmp_path / "real.jpg").exists()
|
||||
# Ledger: blurred hashes cleared so the real media can re-ingest; real kept.
|
||||
remaining = db_sync.execute(
|
||||
select(PatreonSeenMedia.filehash).where(PatreonSeenMedia.source_id == s.id)
|
||||
).scalars().all()
|
||||
assert set(remaining) == {"r" * 32}
|
||||
assert db_sync.get(Post, ids["p1"]) is not None # still has real + nohash
|
||||
assert db_sync.get(Post, ids["p2"]) is None # bare → removed
|
||||
|
||||
|
||||
def test_purge_gated_previews_no_gated_posts_is_noop(db_sync, tmp_path):
|
||||
out = cleanup_service.purge_gated_previews(
|
||||
db_sync, gated_map={}, images_root=tmp_path, dry_run=False,
|
||||
)
|
||||
assert out["matched"] == 0
|
||||
assert out["deleted"] == 0
|
||||
assert out["gated_posts"] == 0
|
||||
|
||||
Reference in New Issue
Block a user