feat(maintenance): retroactive video-dedup action — preview + apply (#871)
Phase 2 of #871: clean up the duplicate videos already in the library (the #859 "same video from multiple sources" clutter). Import-time dedup (Phase 1) only prevents NEW dups; this is the operator-triggered cleanup of existing ones. cleanup_service.dedup_videos(dry_run): - backfill_video_durations: re-probe NULL-duration videos (pre-#871 rows) so the existing library participates; idempotent (only NULL rows), writes a negative sentinel for un-probeable files so they're neither re-probed forever nor matched. - find_video_dup_groups: cluster same-artist videos by duration (±tol) + aspect, anchored per cluster to bound the span (no chain drift); keeper = highest pixel area then bytes. Reuses the importer's _VIDEO_DUP_* tolerances. - apply: re-point each loser's post links to the keeper (so no post loses the video) THEN delete the redundant records + files via delete_images (cascade). dry_run shares the same discovery predicate and returns the projection only (rule 93). Tags on a loser are NOT merged (noted; videos rarely hand-curated). - dedup_videos_task (maintenance queue; summary → task_run.metadata). - POST /maintenance/dedup-videos {dry_run} + GET /maintenance/task-result/<id> so the card shows the dry-run projection before the destructive apply. - VideoDedupCard: Preview → shows groups/redundant/reclaimable, then Apply behind a confirm dialog. Mounted in the Maintenance panel. Tests: dedup collapses + re-links the loser's post to the keeper + removes the file; dry-run deletes nothing; distinct durations aren't grouped; task registered. (Migration 0052 for duration_seconds already shipped with Phase 1.) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -511,3 +511,99 @@ def test_reset_content_tagging_deletes_content_keeps_fandom_series(db_sync, tmp_
|
||||
# Only the series image_tag association survived (content ones cascaded).
|
||||
remaining = db_sync.execute(select(image_tag.c.tag_id)).scalars().all()
|
||||
assert remaining == [s.id]
|
||||
|
||||
|
||||
# ---- Tier-1 video dedup (#871) ------------------------------------------
|
||||
|
||||
|
||||
def _make_video(db_sync, *, artist, path, sha256, duration, w, h, size=2000):
|
||||
img = ImageRecord(
|
||||
artist_id=artist.id, path=path, sha256=sha256, size_bytes=size,
|
||||
mime="video/mp4", origin="downloaded",
|
||||
width=w, height=h, duration_seconds=duration,
|
||||
)
|
||||
db_sync.add(img)
|
||||
db_sync.flush()
|
||||
return img
|
||||
|
||||
|
||||
def test_dedup_videos_collapses_and_relinks(db_sync, tmp_path):
|
||||
"""Apply: a same-artist, same-duration, same-aspect video pair collapses to the
|
||||
higher-res keeper, and the loser's post is re-pointed to the keeper first so it
|
||||
doesn't lose the video."""
|
||||
a = _make_artist(db_sync, slug="vd", name="VD")
|
||||
keeper = _make_video(
|
||||
db_sync, artist=a, path=str(tmp_path / "k.mp4"), sha256="k" * 64,
|
||||
duration=120.0, w=1920, h=1080,
|
||||
)
|
||||
loser = _make_video(
|
||||
db_sync, artist=a, path=str(tmp_path / "l.mp4"), sha256="l" * 64,
|
||||
duration=120.3, w=1280, h=720, # within 1s, same 16:9, smaller
|
||||
)
|
||||
(tmp_path / "k.mp4").write_bytes(b"k")
|
||||
(tmp_path / "l.mp4").write_bytes(b"l")
|
||||
pa = Post(artist_id=a.id, external_post_id="pa")
|
||||
pb = Post(artist_id=a.id, external_post_id="pb")
|
||||
db_sync.add_all([pa, pb])
|
||||
db_sync.flush()
|
||||
db_sync.add(ImageProvenance(image_record_id=keeper.id, post_id=pa.id))
|
||||
db_sync.add(ImageProvenance(image_record_id=loser.id, post_id=pb.id))
|
||||
db_sync.commit()
|
||||
keeper_id, loser_id, pb_id = keeper.id, loser.id, pb.id
|
||||
|
||||
out = cleanup_service.dedup_videos(db_sync, images_root=tmp_path, dry_run=False)
|
||||
assert out["groups"] == 1
|
||||
assert out["deleted"] == 1
|
||||
assert out["relinked_posts"] == 1
|
||||
|
||||
db_sync.expire_all()
|
||||
assert db_sync.get(ImageRecord, loser_id) is None
|
||||
assert db_sync.get(ImageRecord, keeper_id) is not None
|
||||
linked = db_sync.execute(
|
||||
select(ImageProvenance.post_id)
|
||||
.where(ImageProvenance.image_record_id == keeper_id)
|
||||
).scalars().all()
|
||||
assert pb_id in set(linked) # keeper inherited the loser's post
|
||||
assert not (tmp_path / "l.mp4").exists() # redundant file removed
|
||||
|
||||
|
||||
def test_dedup_videos_dry_run_keeps_everything(db_sync, tmp_path):
|
||||
"""Preview reports the group without deleting (rule 93 shared predicate)."""
|
||||
a = _make_artist(db_sync, slug="vd2", name="VD2")
|
||||
_make_video(
|
||||
db_sync, artist=a, path=str(tmp_path / "k.mp4"), sha256="a" * 64,
|
||||
duration=90.0, w=1920, h=1080,
|
||||
)
|
||||
_make_video(
|
||||
db_sync, artist=a, path=str(tmp_path / "l.mp4"), sha256="b" * 64,
|
||||
duration=90.2, w=1280, h=720,
|
||||
)
|
||||
db_sync.commit()
|
||||
|
||||
out = cleanup_service.dedup_videos(db_sync, images_root=tmp_path, dry_run=True)
|
||||
assert out["groups"] == 1
|
||||
assert out["redundant"] == 1
|
||||
assert "deleted" not in out
|
||||
assert db_sync.execute(
|
||||
select(func.count(ImageRecord.id))
|
||||
).scalar_one() == 2
|
||||
|
||||
|
||||
def test_dedup_videos_distinct_durations_not_grouped(db_sync, tmp_path):
|
||||
"""False-merge guard: clearly different durations stay separate."""
|
||||
a = _make_artist(db_sync, slug="vd3", name="VD3")
|
||||
_make_video(
|
||||
db_sync, artist=a, path=str(tmp_path / "x.mp4"), sha256="c" * 64,
|
||||
duration=60.0, w=1920, h=1080,
|
||||
)
|
||||
_make_video(
|
||||
db_sync, artist=a, path=str(tmp_path / "y.mp4"), sha256="d" * 64,
|
||||
duration=200.0, w=1920, h=1080,
|
||||
)
|
||||
db_sync.commit()
|
||||
|
||||
out = cleanup_service.dedup_videos(db_sync, images_root=tmp_path, dry_run=True)
|
||||
assert out["groups"] == 0
|
||||
assert db_sync.execute(
|
||||
select(func.count(ImageRecord.id))
|
||||
).scalar_one() == 2
|
||||
|
||||
Reference in New Issue
Block a user