diff --git a/backend/app/api/admin.py b/backend/app/api/admin.py index f366394..917a9cd 100644 --- a/backend/app/api/admin.py +++ b/backend/app/api/admin.py @@ -361,3 +361,34 @@ async def trigger_prune_missing_files(): async_result = prune_missing_file_records_task.delay() return jsonify({"task_id": async_result.id, "status": "queued"}), 202 + + +@admin_bp.route("/maintenance/dedup-videos", methods=["POST"]) +async def trigger_dedup_videos(): + """Tier-1 video dedup (#871). Body {"dry_run": bool}: dry_run=true previews + what would be removed (groups / redundant count / reclaimable bytes) WITHOUT + deleting; dry_run=false applies it (re-link posts to the keeper, then delete + the redundant copies). Either way it first re-probes NULL-duration videos so + the existing library participates. Returns the Celery task id — poll + /maintenance/task-result/ for the summary.""" + from ..tasks.admin import dedup_videos_task + + body = await request.get_json(silent=True) or {} + dry_run = bool(body.get("dry_run", True)) # default to the SAFE preview + async_result = dedup_videos_task.delay(dry_run=dry_run) + return jsonify({"task_id": async_result.id, "status": "queued"}), 202 + + +@admin_bp.route("/maintenance/task-result/", methods=["GET"]) +async def maintenance_task_result(task_id: str): + """Poll a maintenance Celery task's result (the summary dict it returns). + Used by the video-dedup card to show the dry-run projection before apply.""" + from ..celery_app import celery + + res = celery.AsyncResult(task_id) + ready = res.ready() + return jsonify({ + "ready": ready, + "successful": res.successful() if ready else None, + "result": res.result if (ready and res.successful()) else None, + }) diff --git a/backend/app/services/cleanup_service.py b/backend/app/services/cleanup_service.py index 6d71dc3..b5e84d7 100644 --- a/backend/app/services/cleanup_service.py +++ b/backend/app/services/cleanup_service.py @@ -32,9 +32,17 @@ from ..models import ( from ..models.series_chapter import SeriesChapter from ..models.series_page import SeriesPage from ..models.tag import image_tag +from ..utils import safe_probe +from .importer import _VIDEO_DUP_ASPECT_TOL, _VIDEO_DUP_DURATION_TOL_SECONDS log = logging.getLogger(__name__) +# Sentinel written to duration_seconds when a video was probed but ffprobe +# reported no usable duration (missing/corrupt file) — distinct from NULL (never +# probed) so the backfill doesn't re-probe it forever, and < 0 so it can never +# match a real duration in the dedup grouping (#871). +_VIDEO_DURATION_UNKNOWN = -1.0 + def project_artist_cascade(session: Session, *, slug: str) -> dict: """Read-only projection of what delete_artist_cascade would touch. @@ -940,3 +948,193 @@ def reextract_archive_attachments( except Exception as exc: log.warning("re-extract enqueue failed for image %s: %s", img_id, exc) return summary + + +# ---- Tier-1 video dedup (#871) ------------------------------------------ + + +def _aspect_matches(w, h, cw, ch) -> bool: + """Same aspect ratio within tolerance; missing dims don't block (duration is + the primary signal). Mirrors Importer._video_aspect_matches.""" + if not (w and h and cw and ch): + return True + return abs((w / h) - (cw / ch)) <= _VIDEO_DUP_ASPECT_TOL + + +def backfill_video_durations(session: Session) -> int: + """Populate image_record.duration_seconds for video rows imported before #871 + (NULL). Idempotent — only NULL rows are touched, so a re-run after a timeout + naturally resumes. A probe that yields no duration writes the + _VIDEO_DURATION_UNKNOWN sentinel so the file isn't re-probed forever (and can + never match a real duration). Returns the count of rows given a real duration. + """ + populated = 0 + while True: + rows = session.execute( + select(ImageRecord.id, ImageRecord.path) + .where( + ImageRecord.mime.like("video/%"), + ImageRecord.duration_seconds.is_(None), + ) + .order_by(ImageRecord.id) + .limit(500) + ).all() + if not rows: + break + for rid, path in rows: + probe = safe_probe.probe_video(Path(path)) + dur = probe.duration if probe.ok and probe.duration else None + session.execute( + update(ImageRecord) + .where(ImageRecord.id == rid) + .values( + duration_seconds=dur if dur is not None else _VIDEO_DURATION_UNKNOWN + ) + ) + if dur is not None: + populated += 1 + session.commit() + return populated + + +def _video_dup_group(members: list) -> dict: + """Pick the keeper (highest pixel area, then largest bytes, then lowest id for + stability) and describe the group.""" + keeper = max( + members, + key=lambda m: ((m.width or 0) * (m.height or 0), m.size_bytes or 0, -m.id), + ) + losers = [m for m in members if m.id != keeper.id] + return { + "artist_id": keeper.artist_id, + "keeper_id": keeper.id, + "loser_ids": [m.id for m in losers], + "duration": keeper.duration_seconds, + "count": len(members), + "reclaim_bytes": sum((m.size_bytes or 0) for m in losers), + } + + +def find_video_dup_groups(session: Session) -> list[dict]: + """Cluster videos that are the same content (#871): same artist, duration + within tolerance, matching aspect ratio. Returns groups of >1 member. Greedy + sweep over duration-sorted rows, anchored to each cluster's first member so the + cluster's duration span never exceeds the tolerance (no chain drift).""" + rows = session.execute( + select( + ImageRecord.id, ImageRecord.artist_id, ImageRecord.duration_seconds, + ImageRecord.width, ImageRecord.height, ImageRecord.size_bytes, + ) + .where( + ImageRecord.mime.like("video/%"), + ImageRecord.duration_seconds.is_not(None), + ImageRecord.duration_seconds > 0, + ImageRecord.artist_id.is_not(None), + ) + .order_by( + ImageRecord.artist_id, ImageRecord.duration_seconds, ImageRecord.id + ) + ).all() + groups: list[dict] = [] + cluster: list = [] + anchor = None + for r in rows: + if ( + anchor is not None + and r.artist_id == anchor.artist_id + and (r.duration_seconds - anchor.duration_seconds) + <= _VIDEO_DUP_DURATION_TOL_SECONDS + and _aspect_matches(r.width, r.height, anchor.width, anchor.height) + ): + cluster.append(r) + else: + if len(cluster) > 1: + groups.append(_video_dup_group(cluster)) + cluster = [r] + anchor = r + if len(cluster) > 1: + groups.append(_video_dup_group(cluster)) + return groups + + +def _relink_provenance_to_keeper( + session: Session, *, loser_id: int, keeper_id: int +) -> int: + """Ensure the keeper has an ImageProvenance row for every post the loser was + linked to, so deleting the loser never drops the video off a post. Returns the + number of new keeper↔post links added.""" + rows = session.execute( + select(ImageProvenance.post_id, ImageProvenance.source_id) + .where(ImageProvenance.image_record_id == loser_id) + ).all() + added = 0 + for post_id, source_id in rows: + exists = session.execute( + select(ImageProvenance.id).where( + ImageProvenance.image_record_id == keeper_id, + ImageProvenance.post_id == post_id, + ) + ).scalar_one_or_none() + if exists is None: + session.add(ImageProvenance( + image_record_id=keeper_id, post_id=post_id, source_id=source_id, + )) + session.flush() + added += 1 + return added + + +def dedup_videos( + session: Session, *, images_root: Path, dry_run: bool = False +) -> dict: + """Find and (unless dry_run) collapse Tier-1 video duplicates (#871). + + Re-probes NULL-duration videos first so the existing library participates, + then clusters by artist + duration + aspect and keeps the highest-res copy per + cluster. On apply, each loser's post links are re-pointed to the keeper BEFORE + the loser record + file are deleted, so no post loses the video. dry_run shares + the same discovery predicate and returns the projection without deleting + (rule 93). + + NOTE: tags/curation on a loser are NOT merged onto the keeper — videos rarely + carry hand-curation and merging would add FK-juggling risk. Flagged as a + follow-up if it ever matters. + """ + backfill_video_durations(session) + groups = find_video_dup_groups(session) + redundant = sum(len(g["loser_ids"]) for g in groups) + reclaim = sum(g["reclaim_bytes"] for g in groups) + sample = [ + { + "keeper_id": g["keeper_id"], + "redundant": len(g["loser_ids"]), + "duration": round(g["duration"], 1), + } + for g in groups[:50] + ] + if dry_run: + return { + "groups": len(groups), "redundant": redundant, + "reclaim_bytes": reclaim, "sample": sample, + } + + relinked = 0 + for g in groups: + for loser_id in g["loser_ids"]: + relinked += _relink_provenance_to_keeper( + session, loser_id=loser_id, keeper_id=g["keeper_id"] + ) + session.commit() + + loser_ids = [lid for g in groups for lid in g["loser_ids"]] + deleted = delete_images(session, image_ids=loser_ids, images_root=images_root) + log.info( + "video dedup: %d group(s), %d redundant removed, %d post link(s) re-pointed", + len(groups), deleted["images_deleted"], relinked, + ) + return { + "groups": len(groups), "redundant": redundant, + "reclaim_bytes": reclaim, "deleted": deleted["images_deleted"], + "files_deleted": deleted["files_deleted"], + "relinked_posts": relinked, "sample": sample, + } diff --git a/backend/app/tasks/admin.py b/backend/app/tasks/admin.py index c9b7ca8..371c6c7 100644 --- a/backend/app/tasks/admin.py +++ b/backend/app/tasks/admin.py @@ -122,6 +122,27 @@ def prune_missing_file_records_task(self) -> dict: return {"checked": checked, "missing": len(missing_ids), "deleted": deleted} +@celery.task( + name="backend.app.tasks.admin.dedup_videos_task", + bind=True, + autoretry_for=(OperationalError, DBAPIError), + retry_backoff=15, retry_backoff_max=180, max_retries=1, + soft_time_limit=1800, time_limit=2400, # 30 min / 40 min +) +def dedup_videos_task(self, dry_run: bool = False) -> dict: + """Tier-1 video dedup (#871): re-probe NULL-duration videos, cluster by + artist + duration + aspect, keep the highest-res copy per cluster. dry_run + returns the projection (groups/redundant/reclaimable bytes) WITHOUT deleting; + apply re-points each loser's post links to the keeper then deletes the + redundant records + files. Operator-triggered; the summary lands in + task_run.metadata (FC-3i) for the Maintenance card to surface.""" + SessionLocal = _sync_session_factory() + with SessionLocal() as session: + return cleanup_service.dedup_videos( + session, images_root=IMAGES_ROOT, dry_run=dry_run, + ) + + @celery.task( name="backend.app.tasks.admin.bulk_delete_images_task", bind=True, diff --git a/frontend/src/components/settings/MaintenancePanel.vue b/frontend/src/components/settings/MaintenancePanel.vue index 32c85fd..2ddd31b 100644 --- a/frontend/src/components/settings/MaintenancePanel.vue +++ b/frontend/src/components/settings/MaintenancePanel.vue @@ -17,6 +17,7 @@ + + + Deduplicate videos + +

+ Finds videos of the same artist that are the same content across + re-encodes (matching duration + aspect ratio) and keeps the + highest-resolution copy. Preview first to see what would + be removed; Apply re-points each post to the kept copy, + then deletes the redundant videos and their files. The first run also + computes durations for older videos, so it may take a while. +

+ +
+ + mdi-magnify Preview + + + mdi-content-duplicate Apply + +
+ + + + Removed {{ summary.deleted }} redundant video(s) across + {{ summary.groups }} group(s); re-pointed {{ summary.relinked_posts }} + post link(s); reclaimed {{ humanBytes(summary.reclaim_bytes) }}. + + + {{ summary.redundant }} redundant video(s) across {{ summary.groups }} + group(s) — {{ humanBytes(summary.reclaim_bytes) }} reclaimable. Click + Apply to remove them (keeping the best copy). + + No duplicate videos found. + + + + + + + + Remove duplicate videos? + + This permanently deletes + {{ summary?.redundant ?? 0 }} redundant video file(s), + keeping the highest-resolution copy in each group. Posts are re-pointed + to the kept copy first, so nothing disappears from a post. + + + + Cancel + Remove duplicates + + + + + + + diff --git a/tests/test_cleanup_service.py b/tests/test_cleanup_service.py index 3607465..d1a8b89 100644 --- a/tests/test_cleanup_service.py +++ b/tests/test_cleanup_service.py @@ -511,3 +511,99 @@ def test_reset_content_tagging_deletes_content_keeps_fandom_series(db_sync, tmp_ # Only the series image_tag association survived (content ones cascaded). remaining = db_sync.execute(select(image_tag.c.tag_id)).scalars().all() assert remaining == [s.id] + + +# ---- Tier-1 video dedup (#871) ------------------------------------------ + + +def _make_video(db_sync, *, artist, path, sha256, duration, w, h, size=2000): + img = ImageRecord( + artist_id=artist.id, path=path, sha256=sha256, size_bytes=size, + mime="video/mp4", origin="downloaded", + width=w, height=h, duration_seconds=duration, + ) + db_sync.add(img) + db_sync.flush() + return img + + +def test_dedup_videos_collapses_and_relinks(db_sync, tmp_path): + """Apply: a same-artist, same-duration, same-aspect video pair collapses to the + higher-res keeper, and the loser's post is re-pointed to the keeper first so it + doesn't lose the video.""" + a = _make_artist(db_sync, slug="vd", name="VD") + keeper = _make_video( + db_sync, artist=a, path=str(tmp_path / "k.mp4"), sha256="k" * 64, + duration=120.0, w=1920, h=1080, + ) + loser = _make_video( + db_sync, artist=a, path=str(tmp_path / "l.mp4"), sha256="l" * 64, + duration=120.3, w=1280, h=720, # within 1s, same 16:9, smaller + ) + (tmp_path / "k.mp4").write_bytes(b"k") + (tmp_path / "l.mp4").write_bytes(b"l") + pa = Post(artist_id=a.id, external_post_id="pa") + pb = Post(artist_id=a.id, external_post_id="pb") + db_sync.add_all([pa, pb]) + db_sync.flush() + db_sync.add(ImageProvenance(image_record_id=keeper.id, post_id=pa.id)) + db_sync.add(ImageProvenance(image_record_id=loser.id, post_id=pb.id)) + db_sync.commit() + keeper_id, loser_id, pb_id = keeper.id, loser.id, pb.id + + out = cleanup_service.dedup_videos(db_sync, images_root=tmp_path, dry_run=False) + assert out["groups"] == 1 + assert out["deleted"] == 1 + assert out["relinked_posts"] == 1 + + db_sync.expire_all() + assert db_sync.get(ImageRecord, loser_id) is None + assert db_sync.get(ImageRecord, keeper_id) is not None + linked = db_sync.execute( + select(ImageProvenance.post_id) + .where(ImageProvenance.image_record_id == keeper_id) + ).scalars().all() + assert pb_id in set(linked) # keeper inherited the loser's post + assert not (tmp_path / "l.mp4").exists() # redundant file removed + + +def test_dedup_videos_dry_run_keeps_everything(db_sync, tmp_path): + """Preview reports the group without deleting (rule 93 shared predicate).""" + a = _make_artist(db_sync, slug="vd2", name="VD2") + _make_video( + db_sync, artist=a, path=str(tmp_path / "k.mp4"), sha256="a" * 64, + duration=90.0, w=1920, h=1080, + ) + _make_video( + db_sync, artist=a, path=str(tmp_path / "l.mp4"), sha256="b" * 64, + duration=90.2, w=1280, h=720, + ) + db_sync.commit() + + out = cleanup_service.dedup_videos(db_sync, images_root=tmp_path, dry_run=True) + assert out["groups"] == 1 + assert out["redundant"] == 1 + assert "deleted" not in out + assert db_sync.execute( + select(func.count(ImageRecord.id)) + ).scalar_one() == 2 + + +def test_dedup_videos_distinct_durations_not_grouped(db_sync, tmp_path): + """False-merge guard: clearly different durations stay separate.""" + a = _make_artist(db_sync, slug="vd3", name="VD3") + _make_video( + db_sync, artist=a, path=str(tmp_path / "x.mp4"), sha256="c" * 64, + duration=60.0, w=1920, h=1080, + ) + _make_video( + db_sync, artist=a, path=str(tmp_path / "y.mp4"), sha256="d" * 64, + duration=200.0, w=1920, h=1080, + ) + db_sync.commit() + + out = cleanup_service.dedup_videos(db_sync, images_root=tmp_path, dry_run=True) + assert out["groups"] == 0 + assert db_sync.execute( + select(func.count(ImageRecord.id)) + ).scalar_one() == 2 diff --git a/tests/test_tasks_admin.py b/tests/test_tasks_admin.py index 4d7ac77..ed54402 100644 --- a/tests/test_tasks_admin.py +++ b/tests/test_tasks_admin.py @@ -42,6 +42,10 @@ def test_bulk_delete_images_task_registered(): ) +def test_dedup_videos_task_registered(): + assert "backend.app.tasks.admin.dedup_videos_task" in celery.tasks + + # --- delete_artist_cascade_task -------------------------------------