feat(cleanup): purge misgrabbed gated-post blurred previews (#874 follow-up)
A one-shot Maintenance action to remove the blurred locked-preview images the ingester downloaded from tier-gated Patreon posts before #874. current_user_can_view was never persisted, so the cleanup re-walks each enabled Patreon source (read-only) to re-derive which posts are gated now and the blurred filehashes Patreon serves for them, then matches by CONTENT HASH against stored source_filehash. Because the hash is content-addressed, a real file downloaded when access existed has a different hash and can never match — regained-then-lost-access content is provably spared (operator's hard requirement). NULL source_filehash => unverifiable, kept + reported. On apply: delete matched ImageRecords + files (provenance cascades), clear seen/dead-letter ledger rows for those hashes so the real media re-ingests if access returns, and delete gated posts left bare. Shares one match predicate between preview and apply (rule 93). - cleanup_service: collect_gated_previews + purge_gated_previews - tasks.admin: purge_gated_previews_task (async re-walk bridge, timeboxed) - api.admin: POST /maintenance/purge-gated-previews - GatedPurgeCard.vue in Settings > Maintenance (preview -> confirm -> apply) - tests: collect predicate, hash-match delete/spare/unverifiable, ledger clear, bare-post removal, no-op Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -379,6 +379,22 @@ async def trigger_dedup_videos():
|
||||
return jsonify({"task_id": async_result.id, "status": "queued"}), 202
|
||||
|
||||
|
||||
@admin_bp.route("/maintenance/purge-gated-previews", methods=["POST"])
|
||||
async def trigger_purge_gated_previews():
|
||||
"""Cleanup (#874 follow-up). Body {"dry_run": bool}: dry_run=true previews how
|
||||
many blurred locked-preview images (grabbed from tier-gated Patreon posts
|
||||
before the fix) would be removed WITHOUT deleting; dry_run=false applies it.
|
||||
Re-walks every enabled Patreon source read-only and matches by content hash, so
|
||||
real content downloaded when access existed is provably spared. Returns the
|
||||
Celery task id — poll /maintenance/task-result/<id> for the summary."""
|
||||
from ..tasks.admin import purge_gated_previews_task
|
||||
|
||||
body = await request.get_json(silent=True) or {}
|
||||
dry_run = bool(body.get("dry_run", True)) # default to the SAFE preview
|
||||
async_result = purge_gated_previews_task.delay(dry_run=dry_run)
|
||||
return jsonify({"task_id": async_result.id, "status": "queued"}), 202
|
||||
|
||||
|
||||
@admin_bp.route("/maintenance/task-result/<task_id>", methods=["GET"])
|
||||
async def maintenance_task_result(task_id: str):
|
||||
"""Poll a maintenance Celery task's result (the summary dict it returns).
|
||||
|
||||
@@ -17,7 +17,7 @@ from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func, or_, select, update
|
||||
from sqlalchemy import delete, func, or_, select, update
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import (
|
||||
@@ -25,6 +25,8 @@ from ..models import (
|
||||
ImageProvenance,
|
||||
ImageRecord,
|
||||
LibraryAuditRun,
|
||||
PatreonFailedMedia,
|
||||
PatreonSeenMedia,
|
||||
Post,
|
||||
PostAttachment,
|
||||
Tag,
|
||||
@@ -1138,3 +1140,199 @@ def dedup_videos(
|
||||
"files_deleted": deleted["files_deleted"],
|
||||
"relinked_posts": relinked, "sample": sample,
|
||||
}
|
||||
|
||||
|
||||
# ---- Gated-post blurred-preview cleanup (#874 follow-up) -----------------
|
||||
#
|
||||
# Before the #874 ingester fix, tier-gated Patreon posts (current_user_can_view
|
||||
# == False) had their BLURRED locked-preview media downloaded as if real. That
|
||||
# flag was never persisted (the sidecar/Post.raw_metadata store only
|
||||
# category/id/title/content/url), so we can't tell from the DB which stored
|
||||
# images are blurred previews. This cleanup RE-WALKS each feed to re-derive what
|
||||
# Patreon serves as gated NOW, then matches by CONTENT HASH: an ImageRecord whose
|
||||
# stored `source_filehash` equals a currently-served blurred file's hash IS that
|
||||
# blurred preview. Because the hash is content-addressed, a real file the operator
|
||||
# downloaded when they HAD access has a different hash and can never match — so
|
||||
# regained-then-lost-access content is provably spared (operator's hard
|
||||
# requirement, 2026-06-16). NULL source_filehash → can't verify → kept + reported.
|
||||
|
||||
|
||||
def collect_gated_previews(client, campaign_id: str) -> dict[str, list[str]]:
|
||||
"""Read-only walk of one Patreon feed → `{external_post_id: [filehashes]}` for
|
||||
every GATED post, where the list is the CDN filehashes of the blurred preview
|
||||
media Patreon is serving for it right now. No downloads.
|
||||
|
||||
`client` is a PatreonClient (or test stub) exposing the same seams the
|
||||
ingester uses: `post_is_gated`, `iter_posts`, `extract_media`. extract_media
|
||||
on a gated post yields exactly the blurred preview items (the original bug), so
|
||||
their filehashes are the match keys the purge needs.
|
||||
"""
|
||||
gated: dict[str, list[str]] = {}
|
||||
is_gated = getattr(client, "post_is_gated", None)
|
||||
if is_gated is None:
|
||||
return gated
|
||||
for post, included, _cursor in client.iter_posts(campaign_id):
|
||||
if not is_gated(post):
|
||||
continue
|
||||
pid = str(post.get("id") or "")
|
||||
if not pid:
|
||||
continue
|
||||
hashes = sorted(
|
||||
{m.filehash for m in client.extract_media(post, included) if m.filehash}
|
||||
)
|
||||
gated[pid] = hashes
|
||||
return gated
|
||||
|
||||
|
||||
def _gated_source_hashes(gated_map: dict) -> dict[int, set[str]]:
|
||||
"""{source_id: set(blurred filehashes)} from the discovery map
|
||||
{source_id: {external_post_id: [filehashes]}}."""
|
||||
out: dict[int, set[str]] = {}
|
||||
for sid, posts in gated_map.items():
|
||||
bucket: set[str] = set()
|
||||
for hashes in posts.values():
|
||||
bucket.update(hashes)
|
||||
out[int(sid)] = bucket
|
||||
return out
|
||||
|
||||
|
||||
def _match_gated_preview_images(
|
||||
session: Session, all_hashes: set[str],
|
||||
) -> dict[int, int]:
|
||||
"""{image_id: size_bytes} for ImageRecords whose source_filehash is one of the
|
||||
currently-served blurred filehashes — the confirmed blurred previews. Chunked
|
||||
under the psycopg parameter ceiling."""
|
||||
matched: dict[int, int] = {}
|
||||
hash_list = list(all_hashes)
|
||||
for i in range(0, len(hash_list), 500):
|
||||
chunk = hash_list[i:i + 500]
|
||||
for rid, size in session.execute(
|
||||
select(ImageRecord.id, ImageRecord.size_bytes)
|
||||
.where(ImageRecord.source_filehash.in_(chunk))
|
||||
).all():
|
||||
matched[rid] = size or 0
|
||||
return matched
|
||||
|
||||
|
||||
def _count_unverifiable_gated_images(session: Session, gated_map: dict) -> int:
|
||||
"""Images linked (via provenance) to a gated post but with a NULL
|
||||
source_filehash — we can't prove they're blurred previews, so they're KEPT and
|
||||
only reported. Counts distinct images so one shared across posts counts once."""
|
||||
image_ids: set[int] = set()
|
||||
for sid, posts in gated_map.items():
|
||||
ext_ids = list(posts.keys())
|
||||
if not ext_ids:
|
||||
continue
|
||||
post_ids = session.execute(
|
||||
select(Post.id).where(
|
||||
Post.source_id == int(sid), Post.external_post_id.in_(ext_ids)
|
||||
)
|
||||
).scalars().all()
|
||||
if not post_ids:
|
||||
continue
|
||||
rows = session.execute(
|
||||
select(ImageProvenance.image_record_id)
|
||||
.join(ImageRecord, ImageRecord.id == ImageProvenance.image_record_id)
|
||||
.where(
|
||||
ImageProvenance.post_id.in_(list(post_ids)),
|
||||
ImageRecord.source_filehash.is_(None),
|
||||
)
|
||||
).scalars().all()
|
||||
image_ids.update(rows)
|
||||
return len(image_ids)
|
||||
|
||||
|
||||
def purge_gated_previews(
|
||||
session: Session, *, gated_map: dict, images_root: Path, dry_run: bool = False,
|
||||
) -> dict:
|
||||
"""Find and (unless dry_run) delete the blurred locked-preview images grabbed
|
||||
from gated posts before the #874 fix (see module section above).
|
||||
|
||||
`gated_map` is the discovery output `{source_id: {external_post_id:
|
||||
[filehashes]}}` (assembled by collect_gated_previews per source). Matching is
|
||||
by content hash, so real content is provably spared. dry_run shares the exact
|
||||
same match predicate and returns the projection without deleting (rule 93).
|
||||
|
||||
On apply: delete the matched ImageRecords + files (provenance cascades), clear
|
||||
the seen/dead-letter ledger rows for those blurred hashes so the REAL media
|
||||
re-ingests if access is later regained, and delete the gated post records that
|
||||
are left bare.
|
||||
"""
|
||||
per_source = _gated_source_hashes(gated_map)
|
||||
all_hashes: set[str] = set()
|
||||
for bucket in per_source.values():
|
||||
all_hashes |= bucket
|
||||
gated_posts = sum(len(posts) for posts in gated_map.values())
|
||||
|
||||
matched = _match_gated_preview_images(session, all_hashes)
|
||||
reclaim = sum(matched.values())
|
||||
unverifiable = _count_unverifiable_gated_images(session, gated_map)
|
||||
sample = [{"image_id": rid} for rid in sorted(matched)[:50]]
|
||||
|
||||
projection = {
|
||||
"gated_posts": gated_posts,
|
||||
"matched": len(matched),
|
||||
"reclaim_bytes": reclaim,
|
||||
"unverifiable": unverifiable,
|
||||
"sample": sample,
|
||||
}
|
||||
if dry_run:
|
||||
return projection
|
||||
|
||||
deleted = delete_images(
|
||||
session, image_ids=list(matched), images_root=images_root,
|
||||
)
|
||||
|
||||
# Clear the seen + dead-letter ledger for the blurred hashes so a later
|
||||
# recovery/backfill re-ingests the REAL media if access is regained.
|
||||
ledger_cleared = 0
|
||||
for sid, hashes in per_source.items():
|
||||
hl = list(hashes)
|
||||
for i in range(0, len(hl), 500):
|
||||
chunk = hl[i:i + 500]
|
||||
res = session.execute(
|
||||
delete(PatreonSeenMedia).where(
|
||||
PatreonSeenMedia.source_id == sid,
|
||||
PatreonSeenMedia.filehash.in_(chunk),
|
||||
)
|
||||
)
|
||||
ledger_cleared += res.rowcount or 0
|
||||
session.execute(
|
||||
delete(PatreonFailedMedia).where(
|
||||
PatreonFailedMedia.source_id == sid,
|
||||
PatreonFailedMedia.filehash.in_(chunk),
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
# Delete gated post records left bare by the deletion (shares the bare-post
|
||||
# predicate so a gated post that still has real content is never removed).
|
||||
bare = _bare_post_conditions()
|
||||
posts_deleted = 0
|
||||
for sid, posts in gated_map.items():
|
||||
ext_ids = list(posts.keys())
|
||||
for i in range(0, len(ext_ids), 500):
|
||||
chunk = ext_ids[i:i + 500]
|
||||
res = session.execute(
|
||||
Post.__table__.delete().where(
|
||||
Post.source_id == int(sid),
|
||||
Post.external_post_id.in_(chunk),
|
||||
*bare,
|
||||
)
|
||||
)
|
||||
posts_deleted += res.rowcount or 0
|
||||
session.commit()
|
||||
|
||||
log.info(
|
||||
"gated-preview purge: %d gated post(s), %d blurred image(s) deleted, "
|
||||
"%d ledger row(s) cleared, %d bare post(s) removed, %d unverifiable kept",
|
||||
gated_posts, deleted["images_deleted"], ledger_cleared, posts_deleted,
|
||||
unverifiable,
|
||||
)
|
||||
return {
|
||||
**projection,
|
||||
"deleted": deleted["images_deleted"],
|
||||
"files_deleted": deleted["files_deleted"],
|
||||
"ledger_cleared": ledger_cleared,
|
||||
"posts_deleted": posts_deleted,
|
||||
}
|
||||
|
||||
@@ -143,6 +143,106 @@ def dedup_videos_task(self, dry_run: bool = False) -> dict:
|
||||
)
|
||||
|
||||
|
||||
# Wall-clock budget for the gated-preview re-walk: stop walking new sources past
|
||||
# this and report partial (operator re-runs to finish). Sits under the soft limit
|
||||
# so the task returns its summary cleanly instead of being SIGKILLed mid-walk.
|
||||
_GATED_PURGE_BUDGET_SECONDS = 1500
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.admin.purge_gated_previews_task",
|
||||
bind=True,
|
||||
autoretry_for=(OperationalError, DBAPIError),
|
||||
retry_backoff=15, retry_backoff_max=180, max_retries=1,
|
||||
soft_time_limit=1800, time_limit=2400, # 30 min / 40 min
|
||||
)
|
||||
def purge_gated_previews_task(self, dry_run: bool = True) -> dict:
|
||||
"""Cleanup (#874 follow-up): purge blurred locked-preview images grabbed from
|
||||
tier-gated Patreon posts before the ingester fix. Re-walks every enabled
|
||||
Patreon source (read-only, no downloads) to re-derive which posts are gated NOW
|
||||
and the blurred filehashes Patreon serves for them, then matches by content
|
||||
hash and (unless dry_run) deletes only those exact files — real content
|
||||
downloaded when access existed has a different hash and is provably spared.
|
||||
|
||||
dry_run=True returns the projection (gated posts / matched images / reclaimable
|
||||
bytes / unverifiable kept) WITHOUT deleting; dry_run=false applies it. The
|
||||
summary lands in task_run.metadata (FC-3i) for the Maintenance card. Time-boxed
|
||||
across sources: a `partial` result means re-run to finish (idempotent — a
|
||||
re-walk re-derives the same matches and already-deleted files stay deleted)."""
|
||||
import asyncio
|
||||
import time as _time
|
||||
|
||||
from sqlalchemy import select as _select
|
||||
|
||||
from ..models import Source
|
||||
from ..services.credential_crypto import CredentialCrypto
|
||||
from ..services.credential_service import CredentialService
|
||||
from ..services.patreon_client import PatreonClient
|
||||
from ..services.patreon_resolver import resolve_campaign_id_for_source
|
||||
from ._async_session import async_session_factory
|
||||
|
||||
key_path = IMAGES_ROOT / "secrets" / "credential_key.b64"
|
||||
|
||||
async def _run() -> dict:
|
||||
async_factory, async_engine = async_session_factory()
|
||||
SessionLocal = _sync_session_factory()
|
||||
try:
|
||||
async with async_factory() as async_session:
|
||||
cred = CredentialService(async_session, CredentialCrypto(key_path))
|
||||
cookies_obj = await cred.get_cookies_path("patreon")
|
||||
cookies_path = str(cookies_obj) if cookies_obj else None
|
||||
|
||||
with SessionLocal() as session:
|
||||
sources = session.execute(
|
||||
_select(Source.id, Source.url, Source.config_overrides)
|
||||
.where(Source.platform == "patreon", Source.enabled.is_(True))
|
||||
).all()
|
||||
|
||||
gated_map: dict[int, dict] = {}
|
||||
scanned = 0
|
||||
partial = False
|
||||
loop = asyncio.get_running_loop()
|
||||
start = _time.monotonic()
|
||||
for sid, url, overrides in sources:
|
||||
if _time.monotonic() - start >= _GATED_PURGE_BUDGET_SECONDS:
|
||||
partial = True
|
||||
break
|
||||
campaign_id, _resolved = await resolve_campaign_id_for_source(
|
||||
url, cookies_path, overrides or {}
|
||||
)
|
||||
if not campaign_id:
|
||||
log.warning(
|
||||
"gated-purge: couldn't resolve campaign for source %s (%s) "
|
||||
"— skipping", sid, url,
|
||||
)
|
||||
continue
|
||||
client = PatreonClient(cookies_path)
|
||||
try:
|
||||
gated = await loop.run_in_executor(
|
||||
None, cleanup_service.collect_gated_previews, client, campaign_id
|
||||
)
|
||||
except Exception as exc: # one bad feed must not strand the sweep
|
||||
log.warning("gated-purge walk failed for source %s: %s", sid, exc)
|
||||
continue
|
||||
scanned += 1
|
||||
if gated:
|
||||
gated_map[int(sid)] = gated
|
||||
|
||||
with SessionLocal() as session:
|
||||
summary = cleanup_service.purge_gated_previews(
|
||||
session, gated_map=gated_map, images_root=IMAGES_ROOT,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
summary["sources_scanned"] = scanned
|
||||
summary["sources_total"] = len(sources)
|
||||
summary["partial"] = partial
|
||||
return summary
|
||||
finally:
|
||||
await async_engine.dispose()
|
||||
|
||||
return asyncio.run(_run())
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.admin.bulk_delete_images_task",
|
||||
bind=True,
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
<template>
|
||||
<!-- #874 follow-up: purge blurred locked-preview images grabbed from
|
||||
tier-gated Patreon posts before the ingester fix. Re-walks each feed and
|
||||
matches by content hash, so real content downloaded when access existed is
|
||||
provably spared. Preview first, then apply (destructive: deletes the
|
||||
matched blurred-preview files). -->
|
||||
<v-card>
|
||||
<v-card-title>Clean up gated-post previews</v-card-title>
|
||||
<v-card-text>
|
||||
<p class="text-body-2 mb-3">
|
||||
Removes the blurred locked-preview images that were grabbed from
|
||||
tier-gated Patreon posts before they were filtered out. It re-walks every
|
||||
enabled Patreon source and matches by exact content hash, so anything you
|
||||
downloaded while you <em>had</em> access is left untouched —
|
||||
only the blurred previews are removed. <strong>Preview</strong> first to
|
||||
see the count; <strong>Apply</strong> deletes the matched files and clears
|
||||
their download ledger so the real media can re-download if you regain
|
||||
access. The re-walk can take a while across many sources.
|
||||
</p>
|
||||
|
||||
<div class="d-flex align-center flex-wrap" style="gap: 12px;">
|
||||
<v-btn
|
||||
color="primary" variant="tonal" rounded="pill"
|
||||
:loading="previewing" :disabled="applying" @click="preview"
|
||||
>
|
||||
<v-icon start>mdi-magnify</v-icon> Preview
|
||||
</v-btn>
|
||||
<v-btn
|
||||
color="error" rounded="pill"
|
||||
:loading="applying"
|
||||
:disabled="previewing || !canApply"
|
||||
@click="confirmOpen = true"
|
||||
>
|
||||
<v-icon start>mdi-image-off</v-icon> Apply
|
||||
</v-btn>
|
||||
</div>
|
||||
|
||||
<v-alert
|
||||
v-if="summary" :type="summaryType" variant="tonal" class="mt-4"
|
||||
density="comfortable"
|
||||
>
|
||||
<span v-if="applied">
|
||||
Removed {{ summary.deleted }} blurred preview image(s) across
|
||||
{{ summary.gated_posts }} gated post(s); reclaimed
|
||||
{{ humanBytes(summary.reclaim_bytes) }};
|
||||
removed {{ summary.posts_deleted }} now-empty post(s).
|
||||
<template v-if="summary.unverifiable > 0">
|
||||
{{ summary.unverifiable }} image(s) couldn't be verified (no stored
|
||||
source hash) and were kept.
|
||||
</template>
|
||||
</span>
|
||||
<span v-else-if="summary.matched > 0">
|
||||
{{ summary.matched }} blurred preview image(s) across
|
||||
{{ summary.gated_posts }} gated post(s) —
|
||||
{{ humanBytes(summary.reclaim_bytes) }} reclaimable. Click
|
||||
<strong>Apply</strong> to delete them.
|
||||
<template v-if="summary.unverifiable > 0">
|
||||
({{ summary.unverifiable }} image(s) on gated posts have no stored
|
||||
source hash, so they can't be verified and will be kept.)
|
||||
</template>
|
||||
</span>
|
||||
<span v-else>
|
||||
No blurred gated-post previews found
|
||||
<template v-if="summary.gated_posts > 0">
|
||||
across {{ summary.gated_posts }} gated post(s)</template>.
|
||||
</span>
|
||||
<template v-if="summary.partial">
|
||||
<br>
|
||||
<small>
|
||||
Scanned {{ summary.sources_scanned }}/{{ summary.sources_total }}
|
||||
sources before the time budget — run again to finish the rest.
|
||||
</small>
|
||||
</template>
|
||||
</v-alert>
|
||||
|
||||
<QueueStatusBar queue="maintenance" queue-label="Maintenance" />
|
||||
</v-card-text>
|
||||
|
||||
<v-dialog v-model="confirmOpen" max-width="460">
|
||||
<v-card>
|
||||
<v-card-title>Delete gated-post previews?</v-card-title>
|
||||
<v-card-text class="text-body-2">
|
||||
This permanently deletes
|
||||
<strong>{{ summary?.matched ?? 0 }}</strong> blurred preview file(s)
|
||||
matched by exact content hash. Real content you downloaded with access
|
||||
has a different hash and is not affected.
|
||||
</v-card-text>
|
||||
<v-card-actions>
|
||||
<v-spacer />
|
||||
<v-btn variant="text" @click="confirmOpen = false">Cancel</v-btn>
|
||||
<v-btn color="error" @click="apply">Delete previews</v-btn>
|
||||
</v-card-actions>
|
||||
</v-card>
|
||||
</v-dialog>
|
||||
</v-card>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed, ref } from 'vue'
|
||||
|
||||
import { useApi } from '../../composables/useApi.js'
|
||||
import { toast } from '../../utils/toast.js'
|
||||
import QueueStatusBar from './QueueStatusBar.vue'
|
||||
|
||||
const api = useApi()
|
||||
const previewing = ref(false)
|
||||
const applying = ref(false)
|
||||
const confirmOpen = ref(false)
|
||||
const summary = ref(null)
|
||||
const applied = ref(false)
|
||||
|
||||
const canApply = computed(() => !!summary.value && !applied.value && summary.value.matched > 0)
|
||||
const summaryType = computed(() => {
|
||||
if (applied.value) return 'success'
|
||||
return summary.value && summary.value.matched > 0 ? 'info' : 'success'
|
||||
})
|
||||
|
||||
function humanBytes (n) {
|
||||
const b = Number(n || 0)
|
||||
if (b >= 1 << 30) return (b / (1 << 30)).toFixed(1) + ' GB'
|
||||
if (b >= 1 << 20) return (b / (1 << 20)).toFixed(1) + ' MB'
|
||||
if (b >= 1 << 10) return (b / (1 << 10)).toFixed(1) + ' KB'
|
||||
return b + ' B'
|
||||
}
|
||||
|
||||
// Poll the maintenance task result until ready. The re-walk hits every Patreon
|
||||
// feed, so allow a generous window.
|
||||
async function pollResult (taskId) {
|
||||
for (let i = 0; i < 300; i++) {
|
||||
await new Promise(r => setTimeout(r, 2000))
|
||||
const r = await api.get(`/api/admin/maintenance/task-result/${taskId}`)
|
||||
if (r.ready) {
|
||||
if (!r.successful) throw new Error('the cleanup task failed — check the worker logs')
|
||||
return r.result
|
||||
}
|
||||
}
|
||||
throw new Error('timed out waiting for the cleanup task — check the task dashboard')
|
||||
}
|
||||
|
||||
async function run (dryRun) {
|
||||
const { task_id: taskId } = await api.post(
|
||||
'/api/admin/maintenance/purge-gated-previews', { body: { dry_run: dryRun } },
|
||||
)
|
||||
return pollResult(taskId)
|
||||
}
|
||||
|
||||
async function preview () {
|
||||
previewing.value = true
|
||||
applied.value = false
|
||||
summary.value = null
|
||||
try {
|
||||
summary.value = await run(true)
|
||||
} catch (e) {
|
||||
toast({ text: e?.body?.detail || e?.message || 'Preview failed', type: 'error' })
|
||||
} finally {
|
||||
previewing.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function apply () {
|
||||
confirmOpen.value = false
|
||||
applying.value = true
|
||||
try {
|
||||
summary.value = await run(false)
|
||||
applied.value = true
|
||||
toast({ text: 'Gated-post previews removed', type: 'success' })
|
||||
} catch (e) {
|
||||
toast({ text: e?.body?.detail || e?.message || 'Apply failed', type: 'error' })
|
||||
} finally {
|
||||
applying.value = false
|
||||
}
|
||||
}
|
||||
</script>
|
||||
@@ -18,6 +18,7 @@
|
||||
<ArchiveReextractCard class="mt-6" />
|
||||
<MissingFileRepairCard class="mt-6" />
|
||||
<VideoDedupCard class="mt-6" />
|
||||
<GatedPurgeCard class="mt-6" />
|
||||
<BackupCard class="mt-6" />
|
||||
<!-- TagMaintenanceCard moved to Cleanup tab (v26.05.25.7) — it
|
||||
operates on the existing library which fits the Cleanup-tab
|
||||
@@ -39,6 +40,7 @@ import DbMaintenanceCard from './DbMaintenanceCard.vue'
|
||||
import ArchiveReextractCard from './ArchiveReextractCard.vue'
|
||||
import MissingFileRepairCard from './MissingFileRepairCard.vue'
|
||||
import VideoDedupCard from './VideoDedupCard.vue'
|
||||
import GatedPurgeCard from './GatedPurgeCard.vue'
|
||||
import BackupCard from './BackupCard.vue'
|
||||
import { useSystemActivityStore } from '../../stores/systemActivity.js'
|
||||
|
||||
|
||||
@@ -12,8 +12,10 @@ from backend.app.models import (
|
||||
Artist,
|
||||
ImageProvenance,
|
||||
ImageRecord,
|
||||
PatreonSeenMedia,
|
||||
Post,
|
||||
PostAttachment,
|
||||
Source,
|
||||
Tag,
|
||||
TagKind,
|
||||
)
|
||||
@@ -607,3 +609,161 @@ def test_dedup_videos_distinct_durations_not_grouped(db_sync, tmp_path):
|
||||
assert db_sync.execute(
|
||||
select(func.count(ImageRecord.id))
|
||||
).scalar_one() == 2
|
||||
|
||||
|
||||
# ---- Gated-post blurred-preview cleanup (#874 follow-up) ------------------
|
||||
|
||||
|
||||
class _GatedFakeClient:
|
||||
"""Stub PatreonClient for collect_gated_previews: `posts` is a list of
|
||||
(post_id, gated_bool, [filehashes])."""
|
||||
|
||||
def __init__(self, posts):
|
||||
self._posts = posts
|
||||
|
||||
@staticmethod
|
||||
def post_is_gated(post):
|
||||
return (post.get("attributes") or {}).get("current_user_can_view") is False
|
||||
|
||||
def iter_posts(self, campaign_id, cursor=None):
|
||||
for pid, gated, hashes in self._posts:
|
||||
yield (
|
||||
{
|
||||
"id": pid,
|
||||
"attributes": {"current_user_can_view": not gated},
|
||||
"_h": hashes,
|
||||
},
|
||||
{},
|
||||
None,
|
||||
)
|
||||
|
||||
def extract_media(self, post, included):
|
||||
from backend.app.services.patreon_client import MediaItem
|
||||
return [
|
||||
MediaItem(
|
||||
url=f"https://cdn.patreon.com/{h}/x.jpg", filename="x.jpg",
|
||||
kind="images", filehash=h, post_id=str(post["id"]),
|
||||
)
|
||||
for h in post["_h"]
|
||||
]
|
||||
|
||||
|
||||
def test_collect_gated_previews_only_gated_posts():
|
||||
client = _GatedFakeClient([
|
||||
("g1", True, ["aa", "bb"]),
|
||||
("open1", False, ["cc"]),
|
||||
("g2", True, []), # gated but feed served no media
|
||||
])
|
||||
out = cleanup_service.collect_gated_previews(client, "camp")
|
||||
assert out == {"g1": ["aa", "bb"], "g2": []} # accessible post excluded
|
||||
|
||||
|
||||
def _make_src(db_sync, artist):
|
||||
s = Source(
|
||||
artist_id=artist.id, platform="patreon",
|
||||
url="https://patreon.com/x", enabled=True, config_overrides={},
|
||||
)
|
||||
db_sync.add(s)
|
||||
db_sync.flush()
|
||||
return s
|
||||
|
||||
|
||||
def _make_dl_image(db_sync, *, artist, path, sha256, fh, size=500):
|
||||
img = ImageRecord(
|
||||
artist_id=artist.id, path=path, sha256=sha256, size_bytes=size,
|
||||
mime="image/jpeg", origin="downloaded", source_filehash=fh,
|
||||
)
|
||||
db_sync.add(img)
|
||||
db_sync.flush()
|
||||
return img
|
||||
|
||||
|
||||
def test_purge_gated_previews_deletes_only_hash_matched(db_sync, tmp_path):
|
||||
"""#874: hash-matched blurred previews are deleted; a real file with a
|
||||
different source_filehash (downloaded when access existed) and a
|
||||
null-source_filehash image are both spared. A post left bare is removed; a post
|
||||
that still has real content is kept. Ledger rows for the blurred hash are
|
||||
cleared (so real media can re-ingest) while the real hash stays."""
|
||||
a = _make_artist(db_sync, slug="gp", name="GP")
|
||||
s = _make_src(db_sync, a)
|
||||
blurred = _make_dl_image(
|
||||
db_sync, artist=a, path=str(tmp_path / "blur.jpg"), sha256="1" * 64,
|
||||
fh="b" * 32, size=500,
|
||||
)
|
||||
real = _make_dl_image(
|
||||
db_sync, artist=a, path=str(tmp_path / "real.jpg"), sha256="2" * 64,
|
||||
fh="r" * 32, size=999,
|
||||
)
|
||||
nohash = _make_dl_image(
|
||||
db_sync, artist=a, path=str(tmp_path / "nh.jpg"), sha256="3" * 64,
|
||||
fh=None, size=100,
|
||||
)
|
||||
blurred2 = _make_dl_image(
|
||||
db_sync, artist=a, path=str(tmp_path / "blur2.jpg"), sha256="4" * 64,
|
||||
fh="c" * 32, size=700,
|
||||
)
|
||||
for name in ("blur.jpg", "real.jpg", "nh.jpg", "blur2.jpg"):
|
||||
(tmp_path / name).write_bytes(b"x")
|
||||
p1 = Post(artist_id=a.id, source_id=s.id, external_post_id="g1")
|
||||
p2 = Post(artist_id=a.id, source_id=s.id, external_post_id="g2")
|
||||
db_sync.add_all([p1, p2])
|
||||
db_sync.flush()
|
||||
# g1: blurred + real + nohash; g2: only blurred2.
|
||||
for img in (blurred, real, nohash):
|
||||
db_sync.add(ImageProvenance(image_record_id=img.id, post_id=p1.id, source_id=s.id))
|
||||
db_sync.add(ImageProvenance(image_record_id=blurred2.id, post_id=p2.id, source_id=s.id))
|
||||
db_sync.add_all([
|
||||
PatreonSeenMedia(source_id=s.id, filehash="b" * 32, post_id="g1"),
|
||||
PatreonSeenMedia(source_id=s.id, filehash="r" * 32, post_id="g1"),
|
||||
PatreonSeenMedia(source_id=s.id, filehash="c" * 32, post_id="g2"),
|
||||
])
|
||||
db_sync.commit()
|
||||
ids = {
|
||||
"blurred": blurred.id, "real": real.id, "nohash": nohash.id,
|
||||
"blurred2": blurred2.id, "p1": p1.id, "p2": p2.id,
|
||||
}
|
||||
|
||||
# The gated feed now serves the blurred hashes for g1 + g2.
|
||||
gated_map = {s.id: {"g1": ["b" * 32], "g2": ["c" * 32]}}
|
||||
|
||||
# Preview (rule 93): reports matches without deleting.
|
||||
prev = cleanup_service.purge_gated_previews(
|
||||
db_sync, gated_map=gated_map, images_root=tmp_path, dry_run=True,
|
||||
)
|
||||
assert prev["matched"] == 2
|
||||
assert prev["gated_posts"] == 2
|
||||
assert prev["unverifiable"] == 1 # the null-source_filehash image on g1
|
||||
assert prev["reclaim_bytes"] == 500 + 700
|
||||
assert db_sync.get(ImageRecord, ids["blurred"]) is not None # dry-run kept
|
||||
|
||||
# Apply.
|
||||
out = cleanup_service.purge_gated_previews(
|
||||
db_sync, gated_map=gated_map, images_root=tmp_path, dry_run=False,
|
||||
)
|
||||
assert out["deleted"] == 2
|
||||
assert out["posts_deleted"] == 1 # g2 went bare; g1 still has real content
|
||||
assert out["ledger_cleared"] == 2 # b* and c* cleared; r* retained
|
||||
|
||||
db_sync.expire_all()
|
||||
assert db_sync.get(ImageRecord, ids["blurred"]) is None # blurred deleted
|
||||
assert db_sync.get(ImageRecord, ids["blurred2"]) is None
|
||||
assert db_sync.get(ImageRecord, ids["real"]) is not None # real spared
|
||||
assert db_sync.get(ImageRecord, ids["nohash"]) is not None # unverifiable kept
|
||||
assert not (tmp_path / "blur.jpg").exists()
|
||||
assert (tmp_path / "real.jpg").exists()
|
||||
# Ledger: blurred hashes cleared so the real media can re-ingest; real kept.
|
||||
remaining = db_sync.execute(
|
||||
select(PatreonSeenMedia.filehash).where(PatreonSeenMedia.source_id == s.id)
|
||||
).scalars().all()
|
||||
assert set(remaining) == {"r" * 32}
|
||||
assert db_sync.get(Post, ids["p1"]) is not None # still has real + nohash
|
||||
assert db_sync.get(Post, ids["p2"]) is None # bare → removed
|
||||
|
||||
|
||||
def test_purge_gated_previews_no_gated_posts_is_noop(db_sync, tmp_path):
|
||||
out = cleanup_service.purge_gated_previews(
|
||||
db_sync, gated_map={}, images_root=tmp_path, dry_run=False,
|
||||
)
|
||||
assert out["matched"] == 0
|
||||
assert out["deleted"] == 0
|
||||
assert out["gated_posts"] == 0
|
||||
|
||||
Reference in New Issue
Block a user