feat(maintenance): retroactive video-dedup action — preview + apply (#871)
Phase 2 of #871: clean up the duplicate videos already in the library (the #859 "same video from multiple sources" clutter). Import-time dedup (Phase 1) only prevents NEW dups; this is the operator-triggered cleanup of existing ones. cleanup_service.dedup_videos(dry_run): - backfill_video_durations: re-probe NULL-duration videos (pre-#871 rows) so the existing library participates; idempotent (only NULL rows), writes a negative sentinel for un-probeable files so they're neither re-probed forever nor matched. - find_video_dup_groups: cluster same-artist videos by duration (±tol) + aspect, anchored per cluster to bound the span (no chain drift); keeper = highest pixel area then bytes. Reuses the importer's _VIDEO_DUP_* tolerances. - apply: re-point each loser's post links to the keeper (so no post loses the video) THEN delete the redundant records + files via delete_images (cascade). dry_run shares the same discovery predicate and returns the projection only (rule 93). Tags on a loser are NOT merged (noted; videos rarely hand-curated). - dedup_videos_task (maintenance queue; summary → task_run.metadata). - POST /maintenance/dedup-videos {dry_run} + GET /maintenance/task-result/<id> so the card shows the dry-run projection before the destructive apply. - VideoDedupCard: Preview → shows groups/redundant/reclaimable, then Apply behind a confirm dialog. Mounted in the Maintenance panel. Tests: dedup collapses + re-links the loser's post to the keeper + removes the file; dry-run deletes nothing; distinct durations aren't grouped; task registered. (Migration 0052 for duration_seconds already shipped with Phase 1.) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -17,6 +17,7 @@
|
||||
<DbMaintenanceCard class="mt-6" />
|
||||
<ArchiveReextractCard class="mt-6" />
|
||||
<MissingFileRepairCard class="mt-6" />
|
||||
<VideoDedupCard class="mt-6" />
|
||||
<BackupCard class="mt-6" />
|
||||
<!-- TagMaintenanceCard moved to Cleanup tab (v26.05.25.7) — it
|
||||
operates on the existing library which fits the Cleanup-tab
|
||||
@@ -37,6 +38,7 @@ import AliasTable from './AliasTable.vue'
|
||||
import DbMaintenanceCard from './DbMaintenanceCard.vue'
|
||||
import ArchiveReextractCard from './ArchiveReextractCard.vue'
|
||||
import MissingFileRepairCard from './MissingFileRepairCard.vue'
|
||||
import VideoDedupCard from './VideoDedupCard.vue'
|
||||
import BackupCard from './BackupCard.vue'
|
||||
import { useSystemActivityStore } from '../../stores/systemActivity.js'
|
||||
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
<template>
|
||||
<!-- #871: Tier-1 video dedup — collapse same-artist videos that match on
|
||||
duration + aspect (the same clip re-encoded / pulled from multiple
|
||||
sources). Preview first, then apply (destructive: removes the redundant
|
||||
copies, keeping the highest-resolution one). -->
|
||||
<v-card>
|
||||
<v-card-title>Deduplicate videos</v-card-title>
|
||||
<v-card-text>
|
||||
<p class="text-body-2 mb-3">
|
||||
Finds videos of the same artist that are the same content across
|
||||
re-encodes (matching duration + aspect ratio) and keeps the
|
||||
highest-resolution copy. <strong>Preview</strong> first to see what would
|
||||
be removed; <strong>Apply</strong> re-points each post to the kept copy,
|
||||
then deletes the redundant videos and their files. The first run also
|
||||
computes durations for older videos, so it may take a while.
|
||||
</p>
|
||||
|
||||
<div class="d-flex align-center flex-wrap" style="gap: 12px;">
|
||||
<v-btn
|
||||
color="primary" variant="tonal" rounded="pill"
|
||||
:loading="previewing" :disabled="applying" @click="preview"
|
||||
>
|
||||
<v-icon start>mdi-magnify</v-icon> Preview
|
||||
</v-btn>
|
||||
<v-btn
|
||||
color="error" rounded="pill"
|
||||
:loading="applying"
|
||||
:disabled="previewing || !canApply"
|
||||
@click="confirmOpen = true"
|
||||
>
|
||||
<v-icon start>mdi-content-duplicate</v-icon> Apply
|
||||
</v-btn>
|
||||
</div>
|
||||
|
||||
<v-alert
|
||||
v-if="summary" :type="summaryType" variant="tonal" class="mt-4"
|
||||
density="comfortable"
|
||||
>
|
||||
<span v-if="applied">
|
||||
Removed {{ summary.deleted }} redundant video(s) across
|
||||
{{ summary.groups }} group(s); re-pointed {{ summary.relinked_posts }}
|
||||
post link(s); reclaimed {{ humanBytes(summary.reclaim_bytes) }}.
|
||||
</span>
|
||||
<span v-else-if="summary.redundant > 0">
|
||||
{{ summary.redundant }} redundant video(s) across {{ summary.groups }}
|
||||
group(s) — {{ humanBytes(summary.reclaim_bytes) }} reclaimable. Click
|
||||
<strong>Apply</strong> to remove them (keeping the best copy).
|
||||
</span>
|
||||
<span v-else>No duplicate videos found.</span>
|
||||
</v-alert>
|
||||
|
||||
<QueueStatusBar queue="maintenance" queue-label="Maintenance" />
|
||||
</v-card-text>
|
||||
|
||||
<v-dialog v-model="confirmOpen" max-width="440">
|
||||
<v-card>
|
||||
<v-card-title>Remove duplicate videos?</v-card-title>
|
||||
<v-card-text class="text-body-2">
|
||||
This permanently deletes
|
||||
<strong>{{ summary?.redundant ?? 0 }}</strong> redundant video file(s),
|
||||
keeping the highest-resolution copy in each group. Posts are re-pointed
|
||||
to the kept copy first, so nothing disappears from a post.
|
||||
</v-card-text>
|
||||
<v-card-actions>
|
||||
<v-spacer />
|
||||
<v-btn variant="text" @click="confirmOpen = false">Cancel</v-btn>
|
||||
<v-btn color="error" @click="apply">Remove duplicates</v-btn>
|
||||
</v-card-actions>
|
||||
</v-card>
|
||||
</v-dialog>
|
||||
</v-card>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed, ref } from 'vue'
|
||||
|
||||
import { useApi } from '../../composables/useApi.js'
|
||||
import { toast } from '../../utils/toast.js'
|
||||
import QueueStatusBar from './QueueStatusBar.vue'
|
||||
|
||||
const api = useApi()
|
||||
const previewing = ref(false)
|
||||
const applying = ref(false)
|
||||
const confirmOpen = ref(false)
|
||||
const summary = ref(null)
|
||||
const applied = ref(false)
|
||||
|
||||
const canApply = computed(() => !!summary.value && !applied.value && summary.value.redundant > 0)
|
||||
const summaryType = computed(() => {
|
||||
if (applied.value) return 'success'
|
||||
return summary.value && summary.value.redundant > 0 ? 'info' : 'success'
|
||||
})
|
||||
|
||||
function humanBytes (n) {
|
||||
const b = Number(n || 0)
|
||||
if (b >= 1 << 30) return (b / (1 << 30)).toFixed(1) + ' GB'
|
||||
if (b >= 1 << 20) return (b / (1 << 20)).toFixed(1) + ' MB'
|
||||
if (b >= 1 << 10) return (b / (1 << 10)).toFixed(1) + ' KB'
|
||||
return b + ' B'
|
||||
}
|
||||
|
||||
// Poll the maintenance task result until ready (or give up). The dedup task can
|
||||
// re-probe the whole video library on its first run, so allow a generous window.
|
||||
async function pollResult (taskId) {
|
||||
for (let i = 0; i < 150; i++) {
|
||||
await new Promise(r => setTimeout(r, 2000))
|
||||
const r = await api.get(`/api/admin/maintenance/task-result/${taskId}`)
|
||||
if (r.ready) {
|
||||
if (!r.successful) throw new Error('the dedup task failed — check the worker logs')
|
||||
return r.result
|
||||
}
|
||||
}
|
||||
throw new Error('timed out waiting for the dedup task — check the task dashboard')
|
||||
}
|
||||
|
||||
async function run (dryRun) {
|
||||
const { task_id: taskId } = await api.post(
|
||||
'/api/admin/maintenance/dedup-videos', { body: { dry_run: dryRun } },
|
||||
)
|
||||
return pollResult(taskId)
|
||||
}
|
||||
|
||||
async function preview () {
|
||||
previewing.value = true
|
||||
applied.value = false
|
||||
summary.value = null
|
||||
try {
|
||||
summary.value = await run(true)
|
||||
} catch (e) {
|
||||
toast({ text: e?.body?.detail || e?.message || 'Preview failed', type: 'error' })
|
||||
} finally {
|
||||
previewing.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function apply () {
|
||||
confirmOpen.value = false
|
||||
applying.value = true
|
||||
try {
|
||||
summary.value = await run(false)
|
||||
applied.value = true
|
||||
toast({ text: 'Duplicate videos removed', type: 'success' })
|
||||
} catch (e) {
|
||||
toast({ text: e?.body?.detail || e?.message || 'Apply failed', type: 'error' })
|
||||
} finally {
|
||||
applying.value = false
|
||||
}
|
||||
}
|
||||
</script>
|
||||
Reference in New Issue
Block a user