feat(maintenance): retroactive video-dedup action — preview + apply (#871)
CI / lint (push) Successful in 3s
CI / frontend-build (push) Successful in 22s
CI / backend-lint-and-test (push) Successful in 27s
CI / integration (push) Successful in 3m14s

Phase 2 of #871: clean up the duplicate videos already in the library (the #859
"same video from multiple sources" clutter). Import-time dedup (Phase 1) only
prevents NEW dups; this is the operator-triggered cleanup of existing ones.

cleanup_service.dedup_videos(dry_run):
- backfill_video_durations: re-probe NULL-duration videos (pre-#871 rows) so the
  existing library participates; idempotent (only NULL rows), writes a negative
  sentinel for un-probeable files so they're neither re-probed forever nor matched.
- find_video_dup_groups: cluster same-artist videos by duration (±tol) + aspect,
  anchored per cluster to bound the span (no chain drift); keeper = highest pixel
  area then bytes. Reuses the importer's _VIDEO_DUP_* tolerances.
- apply: re-point each loser's post links to the keeper (so no post loses the
  video) THEN delete the redundant records + files via delete_images (cascade).
  dry_run shares the same discovery predicate and returns the projection only
  (rule 93). Tags on a loser are NOT merged (noted; videos rarely hand-curated).

- dedup_videos_task (maintenance queue; summary → task_run.metadata).
- POST /maintenance/dedup-videos {dry_run} + GET /maintenance/task-result/<id> so
  the card shows the dry-run projection before the destructive apply.
- VideoDedupCard: Preview → shows groups/redundant/reclaimable, then Apply behind
  a confirm dialog. Mounted in the Maintenance panel.

Tests: dedup collapses + re-links the loser's post to the keeper + removes the
file; dry-run deletes nothing; distinct durations aren't grouped; task registered.
(Migration 0052 for duration_seconds already shipped with Phase 1.)

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-06-16 08:31:50 -04:00
parent f154603811
commit 41652db20f
7 changed files with 501 additions and 0 deletions
+96
View File
@@ -511,3 +511,99 @@ def test_reset_content_tagging_deletes_content_keeps_fandom_series(db_sync, tmp_
# Only the series image_tag association survived (content ones cascaded).
remaining = db_sync.execute(select(image_tag.c.tag_id)).scalars().all()
assert remaining == [s.id]
# ---- Tier-1 video dedup (#871) ------------------------------------------
def _make_video(db_sync, *, artist, path, sha256, duration, w, h, size=2000):
img = ImageRecord(
artist_id=artist.id, path=path, sha256=sha256, size_bytes=size,
mime="video/mp4", origin="downloaded",
width=w, height=h, duration_seconds=duration,
)
db_sync.add(img)
db_sync.flush()
return img
def test_dedup_videos_collapses_and_relinks(db_sync, tmp_path):
"""Apply: a same-artist, same-duration, same-aspect video pair collapses to the
higher-res keeper, and the loser's post is re-pointed to the keeper first so it
doesn't lose the video."""
a = _make_artist(db_sync, slug="vd", name="VD")
keeper = _make_video(
db_sync, artist=a, path=str(tmp_path / "k.mp4"), sha256="k" * 64,
duration=120.0, w=1920, h=1080,
)
loser = _make_video(
db_sync, artist=a, path=str(tmp_path / "l.mp4"), sha256="l" * 64,
duration=120.3, w=1280, h=720, # within 1s, same 16:9, smaller
)
(tmp_path / "k.mp4").write_bytes(b"k")
(tmp_path / "l.mp4").write_bytes(b"l")
pa = Post(artist_id=a.id, external_post_id="pa")
pb = Post(artist_id=a.id, external_post_id="pb")
db_sync.add_all([pa, pb])
db_sync.flush()
db_sync.add(ImageProvenance(image_record_id=keeper.id, post_id=pa.id))
db_sync.add(ImageProvenance(image_record_id=loser.id, post_id=pb.id))
db_sync.commit()
keeper_id, loser_id, pb_id = keeper.id, loser.id, pb.id
out = cleanup_service.dedup_videos(db_sync, images_root=tmp_path, dry_run=False)
assert out["groups"] == 1
assert out["deleted"] == 1
assert out["relinked_posts"] == 1
db_sync.expire_all()
assert db_sync.get(ImageRecord, loser_id) is None
assert db_sync.get(ImageRecord, keeper_id) is not None
linked = db_sync.execute(
select(ImageProvenance.post_id)
.where(ImageProvenance.image_record_id == keeper_id)
).scalars().all()
assert pb_id in set(linked) # keeper inherited the loser's post
assert not (tmp_path / "l.mp4").exists() # redundant file removed
def test_dedup_videos_dry_run_keeps_everything(db_sync, tmp_path):
"""Preview reports the group without deleting (rule 93 shared predicate)."""
a = _make_artist(db_sync, slug="vd2", name="VD2")
_make_video(
db_sync, artist=a, path=str(tmp_path / "k.mp4"), sha256="a" * 64,
duration=90.0, w=1920, h=1080,
)
_make_video(
db_sync, artist=a, path=str(tmp_path / "l.mp4"), sha256="b" * 64,
duration=90.2, w=1280, h=720,
)
db_sync.commit()
out = cleanup_service.dedup_videos(db_sync, images_root=tmp_path, dry_run=True)
assert out["groups"] == 1
assert out["redundant"] == 1
assert "deleted" not in out
assert db_sync.execute(
select(func.count(ImageRecord.id))
).scalar_one() == 2
def test_dedup_videos_distinct_durations_not_grouped(db_sync, tmp_path):
"""False-merge guard: clearly different durations stay separate."""
a = _make_artist(db_sync, slug="vd3", name="VD3")
_make_video(
db_sync, artist=a, path=str(tmp_path / "x.mp4"), sha256="c" * 64,
duration=60.0, w=1920, h=1080,
)
_make_video(
db_sync, artist=a, path=str(tmp_path / "y.mp4"), sha256="d" * 64,
duration=200.0, w=1920, h=1080,
)
db_sync.commit()
out = cleanup_service.dedup_videos(db_sync, images_root=tmp_path, dry_run=True)
assert out["groups"] == 0
assert db_sync.execute(
select(func.count(ImageRecord.id))
).scalar_one() == 2