diff --git a/backend/app/api/admin.py b/backend/app/api/admin.py index 917a9cd..efd6468 100644 --- a/backend/app/api/admin.py +++ b/backend/app/api/admin.py @@ -379,6 +379,22 @@ async def trigger_dedup_videos(): return jsonify({"task_id": async_result.id, "status": "queued"}), 202 +@admin_bp.route("/maintenance/purge-gated-previews", methods=["POST"]) +async def trigger_purge_gated_previews(): + """Cleanup (#874 follow-up). Body {"dry_run": bool}: dry_run=true previews how + many blurred locked-preview images (grabbed from tier-gated Patreon posts + before the fix) would be removed WITHOUT deleting; dry_run=false applies it. + Re-walks every enabled Patreon source read-only and matches by content hash, so + real content downloaded when access existed is provably spared. Returns the + Celery task id — poll /maintenance/task-result/ for the summary.""" + from ..tasks.admin import purge_gated_previews_task + + body = await request.get_json(silent=True) or {} + dry_run = bool(body.get("dry_run", True)) # default to the SAFE preview + async_result = purge_gated_previews_task.delay(dry_run=dry_run) + return jsonify({"task_id": async_result.id, "status": "queued"}), 202 + + @admin_bp.route("/maintenance/task-result/", methods=["GET"]) async def maintenance_task_result(task_id: str): """Poll a maintenance Celery task's result (the summary dict it returns). diff --git a/backend/app/services/cleanup_service.py b/backend/app/services/cleanup_service.py index b5e84d7..ab9ad8d 100644 --- a/backend/app/services/cleanup_service.py +++ b/backend/app/services/cleanup_service.py @@ -17,7 +17,7 @@ from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any -from sqlalchemy import func, or_, select, update +from sqlalchemy import delete, func, or_, select, update from sqlalchemy.orm import Session from ..models import ( @@ -25,6 +25,8 @@ from ..models import ( ImageProvenance, ImageRecord, LibraryAuditRun, + PatreonFailedMedia, + PatreonSeenMedia, Post, PostAttachment, Tag, @@ -1138,3 +1140,199 @@ def dedup_videos( "files_deleted": deleted["files_deleted"], "relinked_posts": relinked, "sample": sample, } + + +# ---- Gated-post blurred-preview cleanup (#874 follow-up) ----------------- +# +# Before the #874 ingester fix, tier-gated Patreon posts (current_user_can_view +# == False) had their BLURRED locked-preview media downloaded as if real. That +# flag was never persisted (the sidecar/Post.raw_metadata store only +# category/id/title/content/url), so we can't tell from the DB which stored +# images are blurred previews. This cleanup RE-WALKS each feed to re-derive what +# Patreon serves as gated NOW, then matches by CONTENT HASH: an ImageRecord whose +# stored `source_filehash` equals a currently-served blurred file's hash IS that +# blurred preview. Because the hash is content-addressed, a real file the operator +# downloaded when they HAD access has a different hash and can never match — so +# regained-then-lost-access content is provably spared (operator's hard +# requirement, 2026-06-16). NULL source_filehash → can't verify → kept + reported. + + +def collect_gated_previews(client, campaign_id: str) -> dict[str, list[str]]: + """Read-only walk of one Patreon feed → `{external_post_id: [filehashes]}` for + every GATED post, where the list is the CDN filehashes of the blurred preview + media Patreon is serving for it right now. No downloads. + + `client` is a PatreonClient (or test stub) exposing the same seams the + ingester uses: `post_is_gated`, `iter_posts`, `extract_media`. extract_media + on a gated post yields exactly the blurred preview items (the original bug), so + their filehashes are the match keys the purge needs. + """ + gated: dict[str, list[str]] = {} + is_gated = getattr(client, "post_is_gated", None) + if is_gated is None: + return gated + for post, included, _cursor in client.iter_posts(campaign_id): + if not is_gated(post): + continue + pid = str(post.get("id") or "") + if not pid: + continue + hashes = sorted( + {m.filehash for m in client.extract_media(post, included) if m.filehash} + ) + gated[pid] = hashes + return gated + + +def _gated_source_hashes(gated_map: dict) -> dict[int, set[str]]: + """{source_id: set(blurred filehashes)} from the discovery map + {source_id: {external_post_id: [filehashes]}}.""" + out: dict[int, set[str]] = {} + for sid, posts in gated_map.items(): + bucket: set[str] = set() + for hashes in posts.values(): + bucket.update(hashes) + out[int(sid)] = bucket + return out + + +def _match_gated_preview_images( + session: Session, all_hashes: set[str], +) -> dict[int, int]: + """{image_id: size_bytes} for ImageRecords whose source_filehash is one of the + currently-served blurred filehashes — the confirmed blurred previews. Chunked + under the psycopg parameter ceiling.""" + matched: dict[int, int] = {} + hash_list = list(all_hashes) + for i in range(0, len(hash_list), 500): + chunk = hash_list[i:i + 500] + for rid, size in session.execute( + select(ImageRecord.id, ImageRecord.size_bytes) + .where(ImageRecord.source_filehash.in_(chunk)) + ).all(): + matched[rid] = size or 0 + return matched + + +def _count_unverifiable_gated_images(session: Session, gated_map: dict) -> int: + """Images linked (via provenance) to a gated post but with a NULL + source_filehash — we can't prove they're blurred previews, so they're KEPT and + only reported. Counts distinct images so one shared across posts counts once.""" + image_ids: set[int] = set() + for sid, posts in gated_map.items(): + ext_ids = list(posts.keys()) + if not ext_ids: + continue + post_ids = session.execute( + select(Post.id).where( + Post.source_id == int(sid), Post.external_post_id.in_(ext_ids) + ) + ).scalars().all() + if not post_ids: + continue + rows = session.execute( + select(ImageProvenance.image_record_id) + .join(ImageRecord, ImageRecord.id == ImageProvenance.image_record_id) + .where( + ImageProvenance.post_id.in_(list(post_ids)), + ImageRecord.source_filehash.is_(None), + ) + ).scalars().all() + image_ids.update(rows) + return len(image_ids) + + +def purge_gated_previews( + session: Session, *, gated_map: dict, images_root: Path, dry_run: bool = False, +) -> dict: + """Find and (unless dry_run) delete the blurred locked-preview images grabbed + from gated posts before the #874 fix (see module section above). + + `gated_map` is the discovery output `{source_id: {external_post_id: + [filehashes]}}` (assembled by collect_gated_previews per source). Matching is + by content hash, so real content is provably spared. dry_run shares the exact + same match predicate and returns the projection without deleting (rule 93). + + On apply: delete the matched ImageRecords + files (provenance cascades), clear + the seen/dead-letter ledger rows for those blurred hashes so the REAL media + re-ingests if access is later regained, and delete the gated post records that + are left bare. + """ + per_source = _gated_source_hashes(gated_map) + all_hashes: set[str] = set() + for bucket in per_source.values(): + all_hashes |= bucket + gated_posts = sum(len(posts) for posts in gated_map.values()) + + matched = _match_gated_preview_images(session, all_hashes) + reclaim = sum(matched.values()) + unverifiable = _count_unverifiable_gated_images(session, gated_map) + sample = [{"image_id": rid} for rid in sorted(matched)[:50]] + + projection = { + "gated_posts": gated_posts, + "matched": len(matched), + "reclaim_bytes": reclaim, + "unverifiable": unverifiable, + "sample": sample, + } + if dry_run: + return projection + + deleted = delete_images( + session, image_ids=list(matched), images_root=images_root, + ) + + # Clear the seen + dead-letter ledger for the blurred hashes so a later + # recovery/backfill re-ingests the REAL media if access is regained. + ledger_cleared = 0 + for sid, hashes in per_source.items(): + hl = list(hashes) + for i in range(0, len(hl), 500): + chunk = hl[i:i + 500] + res = session.execute( + delete(PatreonSeenMedia).where( + PatreonSeenMedia.source_id == sid, + PatreonSeenMedia.filehash.in_(chunk), + ) + ) + ledger_cleared += res.rowcount or 0 + session.execute( + delete(PatreonFailedMedia).where( + PatreonFailedMedia.source_id == sid, + PatreonFailedMedia.filehash.in_(chunk), + ) + ) + session.commit() + + # Delete gated post records left bare by the deletion (shares the bare-post + # predicate so a gated post that still has real content is never removed). + bare = _bare_post_conditions() + posts_deleted = 0 + for sid, posts in gated_map.items(): + ext_ids = list(posts.keys()) + for i in range(0, len(ext_ids), 500): + chunk = ext_ids[i:i + 500] + res = session.execute( + Post.__table__.delete().where( + Post.source_id == int(sid), + Post.external_post_id.in_(chunk), + *bare, + ) + ) + posts_deleted += res.rowcount or 0 + session.commit() + + log.info( + "gated-preview purge: %d gated post(s), %d blurred image(s) deleted, " + "%d ledger row(s) cleared, %d bare post(s) removed, %d unverifiable kept", + gated_posts, deleted["images_deleted"], ledger_cleared, posts_deleted, + unverifiable, + ) + return { + **projection, + "deleted": deleted["images_deleted"], + "files_deleted": deleted["files_deleted"], + "ledger_cleared": ledger_cleared, + "posts_deleted": posts_deleted, + } diff --git a/backend/app/services/ingest_core.py b/backend/app/services/ingest_core.py index 072a19c..ba628f7 100644 --- a/backend/app/services/ingest_core.py +++ b/backend/app/services/ingest_core.py @@ -155,6 +155,11 @@ class Ingester: # comes from the per-media sidecar (gallery-dl path). See [[post-first-ingest-contract]]. post_record_key = getattr(self.client, "post_record_key", None) write_post_record = getattr(self.downloader, "write_post_record", None) + # #874: optional client seam — skip tier-gated posts (the account can't + # view them, so Patreon serves only blurred locked-preview media) ENTIRELY: + # no media download, no post-record stub. Absent on stub/not-yet-migrated + # clients → nothing is ever treated as gated. + post_is_gated = getattr(self.client, "post_is_gated", None) start = time.monotonic() last_live = start # plan #709: last live-progress write timestamp log_lines: list[str] = [] @@ -177,6 +182,9 @@ class Ingester: # this walk; posts_with_body = how many yielded a non-empty body. posts_recorded = 0 posts_with_body = 0 + # Tier-gated posts skipped entirely this walk (#874) — surfaced in the + # run summary for diagnostics ("a lot of these now"). + gated_skipped = 0 # Net-new posts THIS chunk for the live progress badge (plan #704 #5); # excludes the re-walked resume page so _backfill_posts stays a monotonic # absolute across chunks instead of an inflating sum. posts_processed @@ -270,6 +278,20 @@ class Ingester: # resume_cursor None, so everything counts. if not (resume_cursor and page_cursor == resume_cursor): chunk_new_posts += 1 + # Tier-gated post (#874): the account can't fully view it, so + # Patreon serves only blurred locked-preview media. Skip it + # ENTIRELY — no media download AND no post-record stub (operator + # decision: gated content leaves no trace; a later walk re-ingests + # it for real once access is gained). Skipped BEFORE the + # post-record block so gated posts never inflate the #862 body + # canary's sample. post_is_gated gates only on an explicit + # current_user_can_view=False (missing/None → viewable). + if post_is_gated and post_is_gated(post): + gated_skipped += 1 + log_lines.append( + f" post {post.get('id')} — gated (skipped, no access)" + ) + continue # Capture the post body + external links ONCE per post (gated by # the synthetic post key in the seen-ledger), for EVERY post — # whether or not it has downloadable media. This is what makes a @@ -449,6 +471,7 @@ class Ingester: # threshold, surfacing the ratio makes a partial extraction regression # visible in the Raw stdout (e.g. "bodies 3/180" reads as off). + (f", bodies {posts_with_body}/{posts_recorded}" if posts_recorded else "") + + (f", {gated_skipped} gated-skipped" if gated_skipped else "") + (", reached end" if reached_bottom else "") + (", time-boxed" if budget_hit else "") ) @@ -534,6 +557,10 @@ class Ingester: sample: list[dict] = [] unset = object() last_page: object = unset + # #874: same gated-post gate as run() — the preview must not count + # blurred locked-preview media as "new", or it would overstate a gated + # source's backlog (preview/apply parity, rule 93). + post_is_gated = getattr(self.client, "post_is_gated", None) for post, included, page_cursor in self.client.iter_posts( campaign_id, cursor=None ): @@ -545,6 +572,8 @@ class Ingester: pages_scanned = page_limit break posts_scanned += 1 + if post_is_gated and post_is_gated(post): + continue media = self.client.extract_media(post, included) if not media: continue diff --git a/backend/app/services/patreon_client.py b/backend/app/services/patreon_client.py index dea59df..907ce73 100644 --- a/backend/app/services/patreon_client.py +++ b/backend/app/services/patreon_client.py @@ -524,6 +524,23 @@ class PatreonClient: "date": published if isinstance(published, str) else None, } + @staticmethod + def post_is_gated(post: dict) -> bool: + """True when the authenticated account CANNOT view this post's content + (#874). Patreon serves only BLURRED locked-preview thumbnails for + paywalled / insufficient-tier posts, and `current_user_can_view` on the + post attributes is the access flag (it IS in `_FIELDS_POST`). The walk + skips a gated post ENTIRELY — no media, no post-record stub — so those + unusable previews never get downloaded. + + Gate ONLY on an explicit `current_user_can_view == False`. A missing / + None flag (older posts, a sparse fieldset, or API drift) is treated as + viewable, so we never over-filter accessible posts on an absent field. + Part of the client contract the core consumes via getattr — an optional + seam, so stub clients / not-yet-migrated platforms simply never gate.""" + attrs = post.get("attributes") or {} + return attrs.get("current_user_can_view") is False + @staticmethod def post_record_key(post: dict) -> tuple[str, str] | None: """`(ledger_key, post_id)` for a media-less post's seen-ledger entry, or diff --git a/backend/app/tasks/admin.py b/backend/app/tasks/admin.py index 371c6c7..784b4da 100644 --- a/backend/app/tasks/admin.py +++ b/backend/app/tasks/admin.py @@ -143,6 +143,106 @@ def dedup_videos_task(self, dry_run: bool = False) -> dict: ) +# Wall-clock budget for the gated-preview re-walk: stop walking new sources past +# this and report partial (operator re-runs to finish). Sits under the soft limit +# so the task returns its summary cleanly instead of being SIGKILLed mid-walk. +_GATED_PURGE_BUDGET_SECONDS = 1500 + + +@celery.task( + name="backend.app.tasks.admin.purge_gated_previews_task", + bind=True, + autoretry_for=(OperationalError, DBAPIError), + retry_backoff=15, retry_backoff_max=180, max_retries=1, + soft_time_limit=1800, time_limit=2400, # 30 min / 40 min +) +def purge_gated_previews_task(self, dry_run: bool = True) -> dict: + """Cleanup (#874 follow-up): purge blurred locked-preview images grabbed from + tier-gated Patreon posts before the ingester fix. Re-walks every enabled + Patreon source (read-only, no downloads) to re-derive which posts are gated NOW + and the blurred filehashes Patreon serves for them, then matches by content + hash and (unless dry_run) deletes only those exact files — real content + downloaded when access existed has a different hash and is provably spared. + + dry_run=True returns the projection (gated posts / matched images / reclaimable + bytes / unverifiable kept) WITHOUT deleting; dry_run=false applies it. The + summary lands in task_run.metadata (FC-3i) for the Maintenance card. Time-boxed + across sources: a `partial` result means re-run to finish (idempotent — a + re-walk re-derives the same matches and already-deleted files stay deleted).""" + import asyncio + import time as _time + + from sqlalchemy import select as _select + + from ..models import Source + from ..services.credential_crypto import CredentialCrypto + from ..services.credential_service import CredentialService + from ..services.patreon_client import PatreonClient + from ..services.patreon_resolver import resolve_campaign_id_for_source + from ._async_session import async_session_factory + + key_path = IMAGES_ROOT / "secrets" / "credential_key.b64" + + async def _run() -> dict: + async_factory, async_engine = async_session_factory() + SessionLocal = _sync_session_factory() + try: + async with async_factory() as async_session: + cred = CredentialService(async_session, CredentialCrypto(key_path)) + cookies_obj = await cred.get_cookies_path("patreon") + cookies_path = str(cookies_obj) if cookies_obj else None + + with SessionLocal() as session: + sources = session.execute( + _select(Source.id, Source.url, Source.config_overrides) + .where(Source.platform == "patreon", Source.enabled.is_(True)) + ).all() + + gated_map: dict[int, dict] = {} + scanned = 0 + partial = False + loop = asyncio.get_running_loop() + start = _time.monotonic() + for sid, url, overrides in sources: + if _time.monotonic() - start >= _GATED_PURGE_BUDGET_SECONDS: + partial = True + break + campaign_id, _resolved = await resolve_campaign_id_for_source( + url, cookies_path, overrides or {} + ) + if not campaign_id: + log.warning( + "gated-purge: couldn't resolve campaign for source %s (%s) " + "— skipping", sid, url, + ) + continue + client = PatreonClient(cookies_path) + try: + gated = await loop.run_in_executor( + None, cleanup_service.collect_gated_previews, client, campaign_id + ) + except Exception as exc: # one bad feed must not strand the sweep + log.warning("gated-purge walk failed for source %s: %s", sid, exc) + continue + scanned += 1 + if gated: + gated_map[int(sid)] = gated + + with SessionLocal() as session: + summary = cleanup_service.purge_gated_previews( + session, gated_map=gated_map, images_root=IMAGES_ROOT, + dry_run=dry_run, + ) + summary["sources_scanned"] = scanned + summary["sources_total"] = len(sources) + summary["partial"] = partial + return summary + finally: + await async_engine.dispose() + + return asyncio.run(_run()) + + @celery.task( name="backend.app.tasks.admin.bulk_delete_images_task", bind=True, diff --git a/frontend/src/components/settings/GatedPurgeCard.vue b/frontend/src/components/settings/GatedPurgeCard.vue new file mode 100644 index 0000000..a08d528 --- /dev/null +++ b/frontend/src/components/settings/GatedPurgeCard.vue @@ -0,0 +1,173 @@ + + + diff --git a/frontend/src/components/settings/MaintenancePanel.vue b/frontend/src/components/settings/MaintenancePanel.vue index 2ddd31b..af21531 100644 --- a/frontend/src/components/settings/MaintenancePanel.vue +++ b/frontend/src/components/settings/MaintenancePanel.vue @@ -18,6 +18,7 @@ +