From 540151290b29d38c8806b658dcc007457c0df0c9 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 16 Jun 2026 15:20:35 -0400 Subject: [PATCH] feat(cleanup): purge misgrabbed gated-post blurred previews (#874 follow-up) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A one-shot Maintenance action to remove the blurred locked-preview images the ingester downloaded from tier-gated Patreon posts before #874. current_user_can_view was never persisted, so the cleanup re-walks each enabled Patreon source (read-only) to re-derive which posts are gated now and the blurred filehashes Patreon serves for them, then matches by CONTENT HASH against stored source_filehash. Because the hash is content-addressed, a real file downloaded when access existed has a different hash and can never match — regained-then-lost-access content is provably spared (operator's hard requirement). NULL source_filehash => unverifiable, kept + reported. On apply: delete matched ImageRecords + files (provenance cascades), clear seen/dead-letter ledger rows for those hashes so the real media re-ingests if access returns, and delete gated posts left bare. Shares one match predicate between preview and apply (rule 93). - cleanup_service: collect_gated_previews + purge_gated_previews - tasks.admin: purge_gated_previews_task (async re-walk bridge, timeboxed) - api.admin: POST /maintenance/purge-gated-previews - GatedPurgeCard.vue in Settings > Maintenance (preview -> confirm -> apply) - tests: collect predicate, hash-match delete/spare/unverifiable, ledger clear, bare-post removal, no-op Co-Authored-By: Claude Opus 4.8 --- backend/app/api/admin.py | 16 ++ backend/app/services/cleanup_service.py | 200 +++++++++++++++++- backend/app/tasks/admin.py | 100 +++++++++ .../components/settings/GatedPurgeCard.vue | 173 +++++++++++++++ .../components/settings/MaintenancePanel.vue | 2 + tests/test_cleanup_service.py | 160 ++++++++++++++ 6 files changed, 650 insertions(+), 1 deletion(-) create mode 100644 frontend/src/components/settings/GatedPurgeCard.vue diff --git a/backend/app/api/admin.py b/backend/app/api/admin.py index 917a9cd..efd6468 100644 --- a/backend/app/api/admin.py +++ b/backend/app/api/admin.py @@ -379,6 +379,22 @@ async def trigger_dedup_videos(): return jsonify({"task_id": async_result.id, "status": "queued"}), 202 +@admin_bp.route("/maintenance/purge-gated-previews", methods=["POST"]) +async def trigger_purge_gated_previews(): + """Cleanup (#874 follow-up). Body {"dry_run": bool}: dry_run=true previews how + many blurred locked-preview images (grabbed from tier-gated Patreon posts + before the fix) would be removed WITHOUT deleting; dry_run=false applies it. + Re-walks every enabled Patreon source read-only and matches by content hash, so + real content downloaded when access existed is provably spared. Returns the + Celery task id — poll /maintenance/task-result/ for the summary.""" + from ..tasks.admin import purge_gated_previews_task + + body = await request.get_json(silent=True) or {} + dry_run = bool(body.get("dry_run", True)) # default to the SAFE preview + async_result = purge_gated_previews_task.delay(dry_run=dry_run) + return jsonify({"task_id": async_result.id, "status": "queued"}), 202 + + @admin_bp.route("/maintenance/task-result/", methods=["GET"]) async def maintenance_task_result(task_id: str): """Poll a maintenance Celery task's result (the summary dict it returns). diff --git a/backend/app/services/cleanup_service.py b/backend/app/services/cleanup_service.py index b5e84d7..ab9ad8d 100644 --- a/backend/app/services/cleanup_service.py +++ b/backend/app/services/cleanup_service.py @@ -17,7 +17,7 @@ from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any -from sqlalchemy import func, or_, select, update +from sqlalchemy import delete, func, or_, select, update from sqlalchemy.orm import Session from ..models import ( @@ -25,6 +25,8 @@ from ..models import ( ImageProvenance, ImageRecord, LibraryAuditRun, + PatreonFailedMedia, + PatreonSeenMedia, Post, PostAttachment, Tag, @@ -1138,3 +1140,199 @@ def dedup_videos( "files_deleted": deleted["files_deleted"], "relinked_posts": relinked, "sample": sample, } + + +# ---- Gated-post blurred-preview cleanup (#874 follow-up) ----------------- +# +# Before the #874 ingester fix, tier-gated Patreon posts (current_user_can_view +# == False) had their BLURRED locked-preview media downloaded as if real. That +# flag was never persisted (the sidecar/Post.raw_metadata store only +# category/id/title/content/url), so we can't tell from the DB which stored +# images are blurred previews. This cleanup RE-WALKS each feed to re-derive what +# Patreon serves as gated NOW, then matches by CONTENT HASH: an ImageRecord whose +# stored `source_filehash` equals a currently-served blurred file's hash IS that +# blurred preview. Because the hash is content-addressed, a real file the operator +# downloaded when they HAD access has a different hash and can never match — so +# regained-then-lost-access content is provably spared (operator's hard +# requirement, 2026-06-16). NULL source_filehash → can't verify → kept + reported. + + +def collect_gated_previews(client, campaign_id: str) -> dict[str, list[str]]: + """Read-only walk of one Patreon feed → `{external_post_id: [filehashes]}` for + every GATED post, where the list is the CDN filehashes of the blurred preview + media Patreon is serving for it right now. No downloads. + + `client` is a PatreonClient (or test stub) exposing the same seams the + ingester uses: `post_is_gated`, `iter_posts`, `extract_media`. extract_media + on a gated post yields exactly the blurred preview items (the original bug), so + their filehashes are the match keys the purge needs. + """ + gated: dict[str, list[str]] = {} + is_gated = getattr(client, "post_is_gated", None) + if is_gated is None: + return gated + for post, included, _cursor in client.iter_posts(campaign_id): + if not is_gated(post): + continue + pid = str(post.get("id") or "") + if not pid: + continue + hashes = sorted( + {m.filehash for m in client.extract_media(post, included) if m.filehash} + ) + gated[pid] = hashes + return gated + + +def _gated_source_hashes(gated_map: dict) -> dict[int, set[str]]: + """{source_id: set(blurred filehashes)} from the discovery map + {source_id: {external_post_id: [filehashes]}}.""" + out: dict[int, set[str]] = {} + for sid, posts in gated_map.items(): + bucket: set[str] = set() + for hashes in posts.values(): + bucket.update(hashes) + out[int(sid)] = bucket + return out + + +def _match_gated_preview_images( + session: Session, all_hashes: set[str], +) -> dict[int, int]: + """{image_id: size_bytes} for ImageRecords whose source_filehash is one of the + currently-served blurred filehashes — the confirmed blurred previews. Chunked + under the psycopg parameter ceiling.""" + matched: dict[int, int] = {} + hash_list = list(all_hashes) + for i in range(0, len(hash_list), 500): + chunk = hash_list[i:i + 500] + for rid, size in session.execute( + select(ImageRecord.id, ImageRecord.size_bytes) + .where(ImageRecord.source_filehash.in_(chunk)) + ).all(): + matched[rid] = size or 0 + return matched + + +def _count_unverifiable_gated_images(session: Session, gated_map: dict) -> int: + """Images linked (via provenance) to a gated post but with a NULL + source_filehash — we can't prove they're blurred previews, so they're KEPT and + only reported. Counts distinct images so one shared across posts counts once.""" + image_ids: set[int] = set() + for sid, posts in gated_map.items(): + ext_ids = list(posts.keys()) + if not ext_ids: + continue + post_ids = session.execute( + select(Post.id).where( + Post.source_id == int(sid), Post.external_post_id.in_(ext_ids) + ) + ).scalars().all() + if not post_ids: + continue + rows = session.execute( + select(ImageProvenance.image_record_id) + .join(ImageRecord, ImageRecord.id == ImageProvenance.image_record_id) + .where( + ImageProvenance.post_id.in_(list(post_ids)), + ImageRecord.source_filehash.is_(None), + ) + ).scalars().all() + image_ids.update(rows) + return len(image_ids) + + +def purge_gated_previews( + session: Session, *, gated_map: dict, images_root: Path, dry_run: bool = False, +) -> dict: + """Find and (unless dry_run) delete the blurred locked-preview images grabbed + from gated posts before the #874 fix (see module section above). + + `gated_map` is the discovery output `{source_id: {external_post_id: + [filehashes]}}` (assembled by collect_gated_previews per source). Matching is + by content hash, so real content is provably spared. dry_run shares the exact + same match predicate and returns the projection without deleting (rule 93). + + On apply: delete the matched ImageRecords + files (provenance cascades), clear + the seen/dead-letter ledger rows for those blurred hashes so the REAL media + re-ingests if access is later regained, and delete the gated post records that + are left bare. + """ + per_source = _gated_source_hashes(gated_map) + all_hashes: set[str] = set() + for bucket in per_source.values(): + all_hashes |= bucket + gated_posts = sum(len(posts) for posts in gated_map.values()) + + matched = _match_gated_preview_images(session, all_hashes) + reclaim = sum(matched.values()) + unverifiable = _count_unverifiable_gated_images(session, gated_map) + sample = [{"image_id": rid} for rid in sorted(matched)[:50]] + + projection = { + "gated_posts": gated_posts, + "matched": len(matched), + "reclaim_bytes": reclaim, + "unverifiable": unverifiable, + "sample": sample, + } + if dry_run: + return projection + + deleted = delete_images( + session, image_ids=list(matched), images_root=images_root, + ) + + # Clear the seen + dead-letter ledger for the blurred hashes so a later + # recovery/backfill re-ingests the REAL media if access is regained. + ledger_cleared = 0 + for sid, hashes in per_source.items(): + hl = list(hashes) + for i in range(0, len(hl), 500): + chunk = hl[i:i + 500] + res = session.execute( + delete(PatreonSeenMedia).where( + PatreonSeenMedia.source_id == sid, + PatreonSeenMedia.filehash.in_(chunk), + ) + ) + ledger_cleared += res.rowcount or 0 + session.execute( + delete(PatreonFailedMedia).where( + PatreonFailedMedia.source_id == sid, + PatreonFailedMedia.filehash.in_(chunk), + ) + ) + session.commit() + + # Delete gated post records left bare by the deletion (shares the bare-post + # predicate so a gated post that still has real content is never removed). + bare = _bare_post_conditions() + posts_deleted = 0 + for sid, posts in gated_map.items(): + ext_ids = list(posts.keys()) + for i in range(0, len(ext_ids), 500): + chunk = ext_ids[i:i + 500] + res = session.execute( + Post.__table__.delete().where( + Post.source_id == int(sid), + Post.external_post_id.in_(chunk), + *bare, + ) + ) + posts_deleted += res.rowcount or 0 + session.commit() + + log.info( + "gated-preview purge: %d gated post(s), %d blurred image(s) deleted, " + "%d ledger row(s) cleared, %d bare post(s) removed, %d unverifiable kept", + gated_posts, deleted["images_deleted"], ledger_cleared, posts_deleted, + unverifiable, + ) + return { + **projection, + "deleted": deleted["images_deleted"], + "files_deleted": deleted["files_deleted"], + "ledger_cleared": ledger_cleared, + "posts_deleted": posts_deleted, + } diff --git a/backend/app/tasks/admin.py b/backend/app/tasks/admin.py index 371c6c7..784b4da 100644 --- a/backend/app/tasks/admin.py +++ b/backend/app/tasks/admin.py @@ -143,6 +143,106 @@ def dedup_videos_task(self, dry_run: bool = False) -> dict: ) +# Wall-clock budget for the gated-preview re-walk: stop walking new sources past +# this and report partial (operator re-runs to finish). Sits under the soft limit +# so the task returns its summary cleanly instead of being SIGKILLed mid-walk. +_GATED_PURGE_BUDGET_SECONDS = 1500 + + +@celery.task( + name="backend.app.tasks.admin.purge_gated_previews_task", + bind=True, + autoretry_for=(OperationalError, DBAPIError), + retry_backoff=15, retry_backoff_max=180, max_retries=1, + soft_time_limit=1800, time_limit=2400, # 30 min / 40 min +) +def purge_gated_previews_task(self, dry_run: bool = True) -> dict: + """Cleanup (#874 follow-up): purge blurred locked-preview images grabbed from + tier-gated Patreon posts before the ingester fix. Re-walks every enabled + Patreon source (read-only, no downloads) to re-derive which posts are gated NOW + and the blurred filehashes Patreon serves for them, then matches by content + hash and (unless dry_run) deletes only those exact files — real content + downloaded when access existed has a different hash and is provably spared. + + dry_run=True returns the projection (gated posts / matched images / reclaimable + bytes / unverifiable kept) WITHOUT deleting; dry_run=false applies it. The + summary lands in task_run.metadata (FC-3i) for the Maintenance card. Time-boxed + across sources: a `partial` result means re-run to finish (idempotent — a + re-walk re-derives the same matches and already-deleted files stay deleted).""" + import asyncio + import time as _time + + from sqlalchemy import select as _select + + from ..models import Source + from ..services.credential_crypto import CredentialCrypto + from ..services.credential_service import CredentialService + from ..services.patreon_client import PatreonClient + from ..services.patreon_resolver import resolve_campaign_id_for_source + from ._async_session import async_session_factory + + key_path = IMAGES_ROOT / "secrets" / "credential_key.b64" + + async def _run() -> dict: + async_factory, async_engine = async_session_factory() + SessionLocal = _sync_session_factory() + try: + async with async_factory() as async_session: + cred = CredentialService(async_session, CredentialCrypto(key_path)) + cookies_obj = await cred.get_cookies_path("patreon") + cookies_path = str(cookies_obj) if cookies_obj else None + + with SessionLocal() as session: + sources = session.execute( + _select(Source.id, Source.url, Source.config_overrides) + .where(Source.platform == "patreon", Source.enabled.is_(True)) + ).all() + + gated_map: dict[int, dict] = {} + scanned = 0 + partial = False + loop = asyncio.get_running_loop() + start = _time.monotonic() + for sid, url, overrides in sources: + if _time.monotonic() - start >= _GATED_PURGE_BUDGET_SECONDS: + partial = True + break + campaign_id, _resolved = await resolve_campaign_id_for_source( + url, cookies_path, overrides or {} + ) + if not campaign_id: + log.warning( + "gated-purge: couldn't resolve campaign for source %s (%s) " + "— skipping", sid, url, + ) + continue + client = PatreonClient(cookies_path) + try: + gated = await loop.run_in_executor( + None, cleanup_service.collect_gated_previews, client, campaign_id + ) + except Exception as exc: # one bad feed must not strand the sweep + log.warning("gated-purge walk failed for source %s: %s", sid, exc) + continue + scanned += 1 + if gated: + gated_map[int(sid)] = gated + + with SessionLocal() as session: + summary = cleanup_service.purge_gated_previews( + session, gated_map=gated_map, images_root=IMAGES_ROOT, + dry_run=dry_run, + ) + summary["sources_scanned"] = scanned + summary["sources_total"] = len(sources) + summary["partial"] = partial + return summary + finally: + await async_engine.dispose() + + return asyncio.run(_run()) + + @celery.task( name="backend.app.tasks.admin.bulk_delete_images_task", bind=True, diff --git a/frontend/src/components/settings/GatedPurgeCard.vue b/frontend/src/components/settings/GatedPurgeCard.vue new file mode 100644 index 0000000..a08d528 --- /dev/null +++ b/frontend/src/components/settings/GatedPurgeCard.vue @@ -0,0 +1,173 @@ + + + diff --git a/frontend/src/components/settings/MaintenancePanel.vue b/frontend/src/components/settings/MaintenancePanel.vue index 2ddd31b..af21531 100644 --- a/frontend/src/components/settings/MaintenancePanel.vue +++ b/frontend/src/components/settings/MaintenancePanel.vue @@ -18,6 +18,7 @@ +