feat(maintenance): retroactive video-dedup action — preview + apply (#871)

Phase 2 of #871: clean up the duplicate videos already in the library (the #859
"same video from multiple sources" clutter). Import-time dedup (Phase 1) only
prevents NEW dups; this is the operator-triggered cleanup of existing ones.

cleanup_service.dedup_videos(dry_run):
- backfill_video_durations: re-probe NULL-duration videos (pre-#871 rows) so the
  existing library participates; idempotent (only NULL rows), writes a negative
  sentinel for un-probeable files so they're neither re-probed forever nor matched.
- find_video_dup_groups: cluster same-artist videos by duration (±tol) + aspect,
  anchored per cluster to bound the span (no chain drift); keeper = highest pixel
  area then bytes. Reuses the importer's _VIDEO_DUP_* tolerances.
- apply: re-point each loser's post links to the keeper (so no post loses the
  video) THEN delete the redundant records + files via delete_images (cascade).
  dry_run shares the same discovery predicate and returns the projection only
  (rule 93). Tags on a loser are NOT merged (noted; videos rarely hand-curated).

- dedup_videos_task (maintenance queue; summary → task_run.metadata).
- POST /maintenance/dedup-videos {dry_run} + GET /maintenance/task-result/<id> so
  the card shows the dry-run projection before the destructive apply.
- VideoDedupCard: Preview → shows groups/redundant/reclaimable, then Apply behind
  a confirm dialog. Mounted in the Maintenance panel.

Tests: dedup collapses + re-links the loser's post to the keeper + removes the
file; dry-run deletes nothing; distinct durations aren't grouped; task registered.
(Migration 0052 for duration_seconds already shipped with Phase 1.)

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

backend/app/api/admin.py

@@ -361,3 +361,34 @@ async def trigger_prune_missing_files():
 
     async_result = prune_missing_file_records_task.delay()
     return jsonify({"task_id": async_result.id, "status": "queued"}), 202
+
+
+@admin_bp.route("/maintenance/dedup-videos", methods=["POST"])
+async def trigger_dedup_videos():
+    """Tier-1 video dedup (#871). Body {"dry_run": bool}: dry_run=true previews
+    what would be removed (groups / redundant count / reclaimable bytes) WITHOUT
+    deleting; dry_run=false applies it (re-link posts to the keeper, then delete
+    the redundant copies). Either way it first re-probes NULL-duration videos so
+    the existing library participates. Returns the Celery task id — poll
+    /maintenance/task-result/<id> for the summary."""
+    from ..tasks.admin import dedup_videos_task
+
+    body = await request.get_json(silent=True) or {}
+    dry_run = bool(body.get("dry_run", True))  # default to the SAFE preview
+    async_result = dedup_videos_task.delay(dry_run=dry_run)
+    return jsonify({"task_id": async_result.id, "status": "queued"}), 202
+
+
+@admin_bp.route("/maintenance/task-result/<task_id>", methods=["GET"])
+async def maintenance_task_result(task_id: str):
+    """Poll a maintenance Celery task's result (the summary dict it returns).
+    Used by the video-dedup card to show the dry-run projection before apply."""
+    from ..celery_app import celery
+
+    res = celery.AsyncResult(task_id)
+    ready = res.ready()
+    return jsonify({
+        "ready": ready,
+        "successful": res.successful() if ready else None,
+        "result": res.result if (ready and res.successful()) else None,
+    })

backend/app/services/cleanup_service.py

@@ -32,9 +32,17 @@ from ..models import (
 from ..models.series_chapter import SeriesChapter
 from ..models.series_page import SeriesPage
 from ..models.tag import image_tag
+from ..utils import safe_probe
+from .importer import _VIDEO_DUP_ASPECT_TOL, _VIDEO_DUP_DURATION_TOL_SECONDS
 
 log = logging.getLogger(__name__)
 
+# Sentinel written to duration_seconds when a video was probed but ffprobe
+# reported no usable duration (missing/corrupt file) — distinct from NULL (never
+# probed) so the backfill doesn't re-probe it forever, and < 0 so it can never
+# match a real duration in the dedup grouping (#871).
+_VIDEO_DURATION_UNKNOWN = -1.0
+
 
 def project_artist_cascade(session: Session, *, slug: str) -> dict:
     """Read-only projection of what delete_artist_cascade would touch.
@@ -940,3 +948,193 @@ def reextract_archive_attachments(
         except Exception as exc:
             log.warning("re-extract enqueue failed for image %s: %s", img_id, exc)
     return summary
+
+
+# ---- Tier-1 video dedup (#871) ------------------------------------------
+
+
+def _aspect_matches(w, h, cw, ch) -> bool:
+    """Same aspect ratio within tolerance; missing dims don't block (duration is
+    the primary signal). Mirrors Importer._video_aspect_matches."""
+    if not (w and h and cw and ch):
+        return True
+    return abs((w / h) - (cw / ch)) <= _VIDEO_DUP_ASPECT_TOL
+
+
+def backfill_video_durations(session: Session) -> int:
+    """Populate image_record.duration_seconds for video rows imported before #871
+    (NULL). Idempotent — only NULL rows are touched, so a re-run after a timeout
+    naturally resumes. A probe that yields no duration writes the
+    _VIDEO_DURATION_UNKNOWN sentinel so the file isn't re-probed forever (and can
+    never match a real duration). Returns the count of rows given a real duration.
+    """
+    populated = 0
+    while True:
+        rows = session.execute(
+            select(ImageRecord.id, ImageRecord.path)
+            .where(
+                ImageRecord.mime.like("video/%"),
+                ImageRecord.duration_seconds.is_(None),
+            )
+            .order_by(ImageRecord.id)
+            .limit(500)
+        ).all()
+        if not rows:
+            break
+        for rid, path in rows:
+            probe = safe_probe.probe_video(Path(path))
+            dur = probe.duration if probe.ok and probe.duration else None
+            session.execute(
+                update(ImageRecord)
+                .where(ImageRecord.id == rid)
+                .values(
+                    duration_seconds=dur if dur is not None else _VIDEO_DURATION_UNKNOWN
+                )
+            )
+            if dur is not None:
+                populated += 1
+        session.commit()
+    return populated
+
+
+def _video_dup_group(members: list) -> dict:
+    """Pick the keeper (highest pixel area, then largest bytes, then lowest id for
+    stability) and describe the group."""
+    keeper = max(
+        members,
+        key=lambda m: ((m.width or 0) * (m.height or 0), m.size_bytes or 0, -m.id),
+    )
+    losers = [m for m in members if m.id != keeper.id]
+    return {
+        "artist_id": keeper.artist_id,
+        "keeper_id": keeper.id,
+        "loser_ids": [m.id for m in losers],
+        "duration": keeper.duration_seconds,
+        "count": len(members),
+        "reclaim_bytes": sum((m.size_bytes or 0) for m in losers),
+    }
+
+
+def find_video_dup_groups(session: Session) -> list[dict]:
+    """Cluster videos that are the same content (#871): same artist, duration
+    within tolerance, matching aspect ratio. Returns groups of >1 member. Greedy
+    sweep over duration-sorted rows, anchored to each cluster's first member so the
+    cluster's duration span never exceeds the tolerance (no chain drift)."""
+    rows = session.execute(
+        select(
+            ImageRecord.id, ImageRecord.artist_id, ImageRecord.duration_seconds,
+            ImageRecord.width, ImageRecord.height, ImageRecord.size_bytes,
+        )
+        .where(
+            ImageRecord.mime.like("video/%"),
+            ImageRecord.duration_seconds.is_not(None),
+            ImageRecord.duration_seconds > 0,
+            ImageRecord.artist_id.is_not(None),
+        )
+        .order_by(
+            ImageRecord.artist_id, ImageRecord.duration_seconds, ImageRecord.id
+        )
+    ).all()
+    groups: list[dict] = []
+    cluster: list = []
+    anchor = None
+    for r in rows:
+        if (
+            anchor is not None
+            and r.artist_id == anchor.artist_id
+            and (r.duration_seconds - anchor.duration_seconds)
+            <= _VIDEO_DUP_DURATION_TOL_SECONDS
+            and _aspect_matches(r.width, r.height, anchor.width, anchor.height)
+        ):
+            cluster.append(r)
+        else:
+            if len(cluster) > 1:
+                groups.append(_video_dup_group(cluster))
+            cluster = [r]
+            anchor = r
+    if len(cluster) > 1:
+        groups.append(_video_dup_group(cluster))
+    return groups
+
+
+def _relink_provenance_to_keeper(
+    session: Session, *, loser_id: int, keeper_id: int
+) -> int:
+    """Ensure the keeper has an ImageProvenance row for every post the loser was
+    linked to, so deleting the loser never drops the video off a post. Returns the
+    number of new keeper↔post links added."""
+    rows = session.execute(
+        select(ImageProvenance.post_id, ImageProvenance.source_id)
+        .where(ImageProvenance.image_record_id == loser_id)
+    ).all()
+    added = 0
+    for post_id, source_id in rows:
+        exists = session.execute(
+            select(ImageProvenance.id).where(
+                ImageProvenance.image_record_id == keeper_id,
+                ImageProvenance.post_id == post_id,
+            )
+        ).scalar_one_or_none()
+        if exists is None:
+            session.add(ImageProvenance(
+                image_record_id=keeper_id, post_id=post_id, source_id=source_id,
+            ))
+            session.flush()
+            added += 1
+    return added
+
+
+def dedup_videos(
+    session: Session, *, images_root: Path, dry_run: bool = False
+) -> dict:
+    """Find and (unless dry_run) collapse Tier-1 video duplicates (#871).
+
+    Re-probes NULL-duration videos first so the existing library participates,
+    then clusters by artist + duration + aspect and keeps the highest-res copy per
+    cluster. On apply, each loser's post links are re-pointed to the keeper BEFORE
+    the loser record + file are deleted, so no post loses the video. dry_run shares
+    the same discovery predicate and returns the projection without deleting
+    (rule 93).
+
+    NOTE: tags/curation on a loser are NOT merged onto the keeper — videos rarely
+    carry hand-curation and merging would add FK-juggling risk. Flagged as a
+    follow-up if it ever matters.
+    """
+    backfill_video_durations(session)
+    groups = find_video_dup_groups(session)
+    redundant = sum(len(g["loser_ids"]) for g in groups)
+    reclaim = sum(g["reclaim_bytes"] for g in groups)
+    sample = [
+        {
+            "keeper_id": g["keeper_id"],
+            "redundant": len(g["loser_ids"]),
+            "duration": round(g["duration"], 1),
+        }
+        for g in groups[:50]
+    ]
+    if dry_run:
+        return {
+            "groups": len(groups), "redundant": redundant,
+            "reclaim_bytes": reclaim, "sample": sample,
+        }
+
+    relinked = 0
+    for g in groups:
+        for loser_id in g["loser_ids"]:
+            relinked += _relink_provenance_to_keeper(
+                session, loser_id=loser_id, keeper_id=g["keeper_id"]
+            )
+        session.commit()
+
+    loser_ids = [lid for g in groups for lid in g["loser_ids"]]
+    deleted = delete_images(session, image_ids=loser_ids, images_root=images_root)
+    log.info(
+        "video dedup: %d group(s), %d redundant removed, %d post link(s) re-pointed",
+        len(groups), deleted["images_deleted"], relinked,
+    )
+    return {
+        "groups": len(groups), "redundant": redundant,
+        "reclaim_bytes": reclaim, "deleted": deleted["images_deleted"],
+        "files_deleted": deleted["files_deleted"],
+        "relinked_posts": relinked, "sample": sample,
+    }

backend/app/tasks/admin.py

@@ -122,6 +122,27 @@ def prune_missing_file_records_task(self) -> dict:
     return {"checked": checked, "missing": len(missing_ids), "deleted": deleted}
 
 
+@celery.task(
+    name="backend.app.tasks.admin.dedup_videos_task",
+    bind=True,
+    autoretry_for=(OperationalError, DBAPIError),
+    retry_backoff=15, retry_backoff_max=180, max_retries=1,
+    soft_time_limit=1800, time_limit=2400,  # 30 min / 40 min
+)
+def dedup_videos_task(self, dry_run: bool = False) -> dict:
+    """Tier-1 video dedup (#871): re-probe NULL-duration videos, cluster by
+    artist + duration + aspect, keep the highest-res copy per cluster. dry_run
+    returns the projection (groups/redundant/reclaimable bytes) WITHOUT deleting;
+    apply re-points each loser's post links to the keeper then deletes the
+    redundant records + files. Operator-triggered; the summary lands in
+    task_run.metadata (FC-3i) for the Maintenance card to surface."""
+    SessionLocal = _sync_session_factory()
+    with SessionLocal() as session:
+        return cleanup_service.dedup_videos(
+            session, images_root=IMAGES_ROOT, dry_run=dry_run,
+        )
+
+
 @celery.task(
     name="backend.app.tasks.admin.bulk_delete_images_task",
     bind=True,

frontend/src/components/settings/MaintenancePanel.vue

@@ -17,6 +17,7 @@
     <DbMaintenanceCard class="mt-6" />
     <ArchiveReextractCard class="mt-6" />
     <MissingFileRepairCard class="mt-6" />
+    <VideoDedupCard class="mt-6" />
     <BackupCard class="mt-6" />
     <!-- TagMaintenanceCard moved to Cleanup tab (v26.05.25.7) — it
          operates on the existing library which fits the Cleanup-tab
@@ -37,6 +38,7 @@ import AliasTable from './AliasTable.vue'
 import DbMaintenanceCard from './DbMaintenanceCard.vue'
 import ArchiveReextractCard from './ArchiveReextractCard.vue'
 import MissingFileRepairCard from './MissingFileRepairCard.vue'
+import VideoDedupCard from './VideoDedupCard.vue'
 import BackupCard from './BackupCard.vue'
 import { useSystemActivityStore } from '../../stores/systemActivity.js'
 

frontend/src/components/settings/VideoDedupCard.vue

@@ -0,0 +1,149 @@
+<template>
+  <!-- #871: Tier-1 video dedup — collapse same-artist videos that match on
+       duration + aspect (the same clip re-encoded / pulled from multiple
+       sources). Preview first, then apply (destructive: removes the redundant
+       copies, keeping the highest-resolution one). -->
+  <v-card>
+    <v-card-title>Deduplicate videos</v-card-title>
+    <v-card-text>
+      <p class="text-body-2 mb-3">
+        Finds videos of the same artist that are the same content across
+        re-encodes (matching duration + aspect ratio) and keeps the
+        highest-resolution copy. <strong>Preview</strong> first to see what would
+        be removed; <strong>Apply</strong> re-points each post to the kept copy,
+        then deletes the redundant videos and their files. The first run also
+        computes durations for older videos, so it may take a while.
+      </p>
+
+      <div class="d-flex align-center flex-wrap" style="gap: 12px;">
+        <v-btn
+          color="primary" variant="tonal" rounded="pill"
+          :loading="previewing" :disabled="applying" @click="preview"
+        >
+          <v-icon start>mdi-magnify</v-icon> Preview
+        </v-btn>
+        <v-btn
+          color="error" rounded="pill"
+          :loading="applying"
+          :disabled="previewing || !canApply"
+          @click="confirmOpen = true"
+        >
+          <v-icon start>mdi-content-duplicate</v-icon> Apply
+        </v-btn>
+      </div>
+
+      <v-alert
+        v-if="summary" :type="summaryType" variant="tonal" class="mt-4"
+        density="comfortable"
+      >
+        <span v-if="applied">
+          Removed {{ summary.deleted }} redundant video(s) across
+          {{ summary.groups }} group(s); re-pointed {{ summary.relinked_posts }}
+          post link(s); reclaimed {{ humanBytes(summary.reclaim_bytes) }}.
+        </span>
+        <span v-else-if="summary.redundant > 0">
+          {{ summary.redundant }} redundant video(s) across {{ summary.groups }}
+          group(s) — {{ humanBytes(summary.reclaim_bytes) }} reclaimable. Click
+          <strong>Apply</strong> to remove them (keeping the best copy).
+        </span>
+        <span v-else>No duplicate videos found.</span>
+      </v-alert>
+
+      <QueueStatusBar queue="maintenance" queue-label="Maintenance" />
+    </v-card-text>
+
+    <v-dialog v-model="confirmOpen" max-width="440">
+      <v-card>
+        <v-card-title>Remove duplicate videos?</v-card-title>
+        <v-card-text class="text-body-2">
+          This permanently deletes
+          <strong>{{ summary?.redundant ?? 0 }}</strong> redundant video file(s),
+          keeping the highest-resolution copy in each group. Posts are re-pointed
+          to the kept copy first, so nothing disappears from a post.
+        </v-card-text>
+        <v-card-actions>
+          <v-spacer />
+          <v-btn variant="text" @click="confirmOpen = false">Cancel</v-btn>
+          <v-btn color="error" @click="apply">Remove duplicates</v-btn>
+        </v-card-actions>
+      </v-card>
+    </v-dialog>
+  </v-card>
+</template>
+
+<script setup>
+import { computed, ref } from 'vue'
+
+import { useApi } from '../../composables/useApi.js'
+import { toast } from '../../utils/toast.js'
+import QueueStatusBar from './QueueStatusBar.vue'
+
+const api = useApi()
+const previewing = ref(false)
+const applying = ref(false)
+const confirmOpen = ref(false)
+const summary = ref(null)
+const applied = ref(false)
+
+const canApply = computed(() => !!summary.value && !applied.value && summary.value.redundant > 0)
+const summaryType = computed(() => {
+  if (applied.value) return 'success'
+  return summary.value && summary.value.redundant > 0 ? 'info' : 'success'
+})
+
+function humanBytes (n) {
+  const b = Number(n || 0)
+  if (b >= 1 << 30) return (b / (1 << 30)).toFixed(1) + ' GB'
+  if (b >= 1 << 20) return (b / (1 << 20)).toFixed(1) + ' MB'
+  if (b >= 1 << 10) return (b / (1 << 10)).toFixed(1) + ' KB'
+  return b + ' B'
+}
+
+// Poll the maintenance task result until ready (or give up). The dedup task can
+// re-probe the whole video library on its first run, so allow a generous window.
+async function pollResult (taskId) {
+  for (let i = 0; i < 150; i++) {
+    await new Promise(r => setTimeout(r, 2000))
+    const r = await api.get(`/api/admin/maintenance/task-result/${taskId}`)
+    if (r.ready) {
+      if (!r.successful) throw new Error('the dedup task failed — check the worker logs')
+      return r.result
+    }
+  }
+  throw new Error('timed out waiting for the dedup task — check the task dashboard')
+}
+
+async function run (dryRun) {
+  const { task_id: taskId } = await api.post(
+    '/api/admin/maintenance/dedup-videos', { body: { dry_run: dryRun } },
+  )
+  return pollResult(taskId)
+}
+
+async function preview () {
+  previewing.value = true
+  applied.value = false
+  summary.value = null
+  try {
+    summary.value = await run(true)
+  } catch (e) {
+    toast({ text: e?.body?.detail || e?.message || 'Preview failed', type: 'error' })
+  } finally {
+    previewing.value = false
+  }
+}
+
+async function apply () {
+  confirmOpen.value = false
+  applying.value = true
+  try {
+    summary.value = await run(false)
+    applied.value = true
+    toast({ text: 'Duplicate videos removed', type: 'success' })
+  } catch (e) {
+    toast({ text: e?.body?.detail || e?.message || 'Apply failed', type: 'error' })
+  } finally {
+    applying.value = false
+  }
+}
+</script>

tests/test_cleanup_service.py

@@ -511,3 +511,99 @@ def test_reset_content_tagging_deletes_content_keeps_fandom_series(db_sync, tmp_
     # Only the series image_tag association survived (content ones cascaded).
     remaining = db_sync.execute(select(image_tag.c.tag_id)).scalars().all()
     assert remaining == [s.id]
+
+
+# ---- Tier-1 video dedup (#871) ------------------------------------------
+
+
+def _make_video(db_sync, *, artist, path, sha256, duration, w, h, size=2000):
+    img = ImageRecord(
+        artist_id=artist.id, path=path, sha256=sha256, size_bytes=size,
+        mime="video/mp4", origin="downloaded",
+        width=w, height=h, duration_seconds=duration,
+    )
+    db_sync.add(img)
+    db_sync.flush()
+    return img
+
+
+def test_dedup_videos_collapses_and_relinks(db_sync, tmp_path):
+    """Apply: a same-artist, same-duration, same-aspect video pair collapses to the
+    higher-res keeper, and the loser's post is re-pointed to the keeper first so it
+    doesn't lose the video."""
+    a = _make_artist(db_sync, slug="vd", name="VD")
+    keeper = _make_video(
+        db_sync, artist=a, path=str(tmp_path / "k.mp4"), sha256="k" * 64,
+        duration=120.0, w=1920, h=1080,
+    )
+    loser = _make_video(
+        db_sync, artist=a, path=str(tmp_path / "l.mp4"), sha256="l" * 64,
+        duration=120.3, w=1280, h=720,  # within 1s, same 16:9, smaller
+    )
+    (tmp_path / "k.mp4").write_bytes(b"k")
+    (tmp_path / "l.mp4").write_bytes(b"l")
+    pa = Post(artist_id=a.id, external_post_id="pa")
+    pb = Post(artist_id=a.id, external_post_id="pb")
+    db_sync.add_all([pa, pb])
+    db_sync.flush()
+    db_sync.add(ImageProvenance(image_record_id=keeper.id, post_id=pa.id))
+    db_sync.add(ImageProvenance(image_record_id=loser.id, post_id=pb.id))
+    db_sync.commit()
+    keeper_id, loser_id, pb_id = keeper.id, loser.id, pb.id
+
+    out = cleanup_service.dedup_videos(db_sync, images_root=tmp_path, dry_run=False)
+    assert out["groups"] == 1
+    assert out["deleted"] == 1
+    assert out["relinked_posts"] == 1
+
+    db_sync.expire_all()
+    assert db_sync.get(ImageRecord, loser_id) is None
+    assert db_sync.get(ImageRecord, keeper_id) is not None
+    linked = db_sync.execute(
+        select(ImageProvenance.post_id)
+        .where(ImageProvenance.image_record_id == keeper_id)
+    ).scalars().all()
+    assert pb_id in set(linked)  # keeper inherited the loser's post
+    assert not (tmp_path / "l.mp4").exists()  # redundant file removed
+
+
+def test_dedup_videos_dry_run_keeps_everything(db_sync, tmp_path):
+    """Preview reports the group without deleting (rule 93 shared predicate)."""
+    a = _make_artist(db_sync, slug="vd2", name="VD2")
+    _make_video(
+        db_sync, artist=a, path=str(tmp_path / "k.mp4"), sha256="a" * 64,
+        duration=90.0, w=1920, h=1080,
+    )
+    _make_video(
+        db_sync, artist=a, path=str(tmp_path / "l.mp4"), sha256="b" * 64,
+        duration=90.2, w=1280, h=720,
+    )
+    db_sync.commit()
+
+    out = cleanup_service.dedup_videos(db_sync, images_root=tmp_path, dry_run=True)
+    assert out["groups"] == 1
+    assert out["redundant"] == 1
+    assert "deleted" not in out
+    assert db_sync.execute(
+        select(func.count(ImageRecord.id))
+    ).scalar_one() == 2
+
+
+def test_dedup_videos_distinct_durations_not_grouped(db_sync, tmp_path):
+    """False-merge guard: clearly different durations stay separate."""
+    a = _make_artist(db_sync, slug="vd3", name="VD3")
+    _make_video(
+        db_sync, artist=a, path=str(tmp_path / "x.mp4"), sha256="c" * 64,
+        duration=60.0, w=1920, h=1080,
+    )
+    _make_video(
+        db_sync, artist=a, path=str(tmp_path / "y.mp4"), sha256="d" * 64,
+        duration=200.0, w=1920, h=1080,
+    )
+    db_sync.commit()
+
+    out = cleanup_service.dedup_videos(db_sync, images_root=tmp_path, dry_run=True)
+    assert out["groups"] == 0
+    assert db_sync.execute(
+        select(func.count(ImageRecord.id))
+    ).scalar_one() == 2

tests/test_tasks_admin.py

@@ -42,6 +42,10 @@ def test_bulk_delete_images_task_registered():
     )
 
 
+def test_dedup_videos_task_registered():
+    assert "backend.app.tasks.admin.dedup_videos_task" in celery.tasks
+
+
 # --- delete_artist_cascade_task -------------------------------------