Files
bvandeusen 41652db20f
CI / lint (push) Successful in 3s
CI / frontend-build (push) Successful in 22s
CI / backend-lint-and-test (push) Successful in 27s
CI / integration (push) Successful in 3m14s
feat(maintenance): retroactive video-dedup action — preview + apply (#871)
Phase 2 of #871: clean up the duplicate videos already in the library (the #859
"same video from multiple sources" clutter). Import-time dedup (Phase 1) only
prevents NEW dups; this is the operator-triggered cleanup of existing ones.

cleanup_service.dedup_videos(dry_run):
- backfill_video_durations: re-probe NULL-duration videos (pre-#871 rows) so the
  existing library participates; idempotent (only NULL rows), writes a negative
  sentinel for un-probeable files so they're neither re-probed forever nor matched.
- find_video_dup_groups: cluster same-artist videos by duration (±tol) + aspect,
  anchored per cluster to bound the span (no chain drift); keeper = highest pixel
  area then bytes. Reuses the importer's _VIDEO_DUP_* tolerances.
- apply: re-point each loser's post links to the keeper (so no post loses the
  video) THEN delete the redundant records + files via delete_images (cascade).
  dry_run shares the same discovery predicate and returns the projection only
  (rule 93). Tags on a loser are NOT merged (noted; videos rarely hand-curated).

- dedup_videos_task (maintenance queue; summary → task_run.metadata).
- POST /maintenance/dedup-videos {dry_run} + GET /maintenance/task-result/<id> so
  the card shows the dry-run projection before the destructive apply.
- VideoDedupCard: Preview → shows groups/redundant/reclaimable, then Apply behind
  a confirm dialog. Mounted in the Maintenance panel.

Tests: dedup collapses + re-links the loser's post to the keeper + removes the
file; dry-run deletes nothing; distinct durations aren't grouped; task registered.
(Migration 0052 for duration_seconds already shipped with Phase 1.)

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-16 08:31:50 -04:00

237 lines
7.1 KiB
Python

"""FC-3k: admin Celery task integration tests.
task_always_eager + signal handlers from FC-3i populate task_run.
We assert the wrapper passes args through correctly and that
task_run lifecycle status flips as expected.
"""
import pytest
from sqlalchemy import func, select
import backend.app.tasks.admin # noqa: F401 — register tasks
from backend.app.celery_app import celery
from backend.app.models import Artist, ImageRecord, TaskRun
pytestmark = pytest.mark.integration
@pytest.fixture(autouse=True)
def _eager_celery(monkeypatch):
monkeypatch.setattr(celery.conf, "task_always_eager", True)
monkeypatch.setattr(celery.conf, "task_eager_propagates", False)
@pytest.fixture(autouse=True)
def fake_images_root(monkeypatch, tmp_path):
monkeypatch.setattr("backend.app.tasks.admin.IMAGES_ROOT", tmp_path)
# --- registration ----------------------------------------------------
def test_delete_artist_cascade_task_registered():
assert (
"backend.app.tasks.admin.delete_artist_cascade_task"
in celery.tasks
)
def test_bulk_delete_images_task_registered():
assert (
"backend.app.tasks.admin.bulk_delete_images_task"
in celery.tasks
)
def test_dedup_videos_task_registered():
assert "backend.app.tasks.admin.dedup_videos_task" in celery.tasks
# --- delete_artist_cascade_task -------------------------------------
@pytest.mark.asyncio
async def test_delete_artist_cascade_task_removes_artist_and_records_ok(
db_sync, tmp_path,
):
from backend.app.tasks.admin import delete_artist_cascade_task
a = Artist(name="Doomed", slug="doomed")
db_sync.add(a)
db_sync.flush()
for i in range(2):
f = tmp_path / f"d{i}.jpg"
f.write_bytes(b"x")
db_sync.add(ImageRecord(
artist_id=a.id, path=str(f),
sha256=f"{i:064x}", size_bytes=10, mime="image/jpeg",
origin="imported_filesystem",
))
db_sync.commit()
artist_id = a.id
result = delete_artist_cascade_task.delay(artist_id=artist_id).get()
assert result["summary"]["images_deleted"] == 2
surviving = db_sync.execute(
select(func.count(Artist.id)).where(Artist.id == artist_id)
).scalar_one()
assert surviving == 0
# FC-3i task_run lifecycle: task should have an 'ok' row.
status = db_sync.execute(
select(TaskRun.status)
.where(TaskRun.task_name.endswith(".delete_artist_cascade_task"))
.order_by(TaskRun.id.desc()).limit(1)
).scalar_one()
assert status == "ok"
@pytest.mark.asyncio
async def test_delete_artist_cascade_task_records_failure(
db_sync, monkeypatch,
):
from backend.app.tasks.admin import delete_artist_cascade_task
def _boom(*a, **kw):
raise RuntimeError("synthetic cascade fail")
monkeypatch.setattr(
"backend.app.services.cleanup_service.delete_artist_cascade", _boom,
)
with pytest.raises(RuntimeError):
delete_artist_cascade_task.delay(artist_id=1).get()
row = db_sync.execute(
select(TaskRun.status, TaskRun.error_message)
.where(TaskRun.task_name.endswith(".delete_artist_cascade_task"))
.order_by(TaskRun.id.desc()).limit(1)
).one()
assert row.status == "error"
assert "synthetic" in (row.error_message or "")
# --- bulk_delete_images_task ----------------------------------------
@pytest.mark.asyncio
async def test_bulk_delete_images_task_removes_listed_images(
db_sync, tmp_path,
):
from backend.app.tasks.admin import bulk_delete_images_task
a = Artist(name="B", slug="b")
db_sync.add(a)
db_sync.flush()
ids = []
for i in range(3):
f = tmp_path / f"b{i}.jpg"
f.write_bytes(b"x")
img = ImageRecord(
artist_id=a.id, path=str(f),
sha256=f"a{i:063x}", size_bytes=10, mime="image/jpeg",
origin="imported_filesystem",
)
db_sync.add(img)
db_sync.flush()
ids.append(img.id)
db_sync.commit()
result = bulk_delete_images_task.delay(image_ids=ids).get()
assert result["images_deleted"] == 3
surviving = db_sync.execute(
select(func.count(ImageRecord.id))
.where(ImageRecord.id.in_(ids))
).scalar_one()
assert surviving == 0
# --- prune_missing_file_records_task (#859 orphan repair) -----------
def _img(db_sync, artist, *, path, sha):
img = ImageRecord(
artist_id=artist.id, path=str(path),
sha256=sha, size_bytes=10, mime="image/jpeg",
origin="downloaded",
)
db_sync.add(img)
db_sync.flush()
return img.id
@pytest.mark.asyncio
async def test_prune_missing_file_records_deletes_only_orphans(db_sync, tmp_path):
"""Records whose backing file is gone get deleted; records whose file is
present survive. Below the abort sample size, fraction is irrelevant."""
from backend.app.tasks.admin import prune_missing_file_records_task
a = Artist(name="Orph", slug="orph")
db_sync.add(a)
db_sync.flush()
present = []
for i in range(3):
f = tmp_path / f"present{i}.jpg"
f.write_bytes(b"real")
present.append(_img(db_sync, a, path=f, sha=f"{i:064x}"))
missing = [
_img(db_sync, a, path=tmp_path / f"gone{i}.jpg", sha=f"f{i:063x}")
for i in range(2)
]
db_sync.commit()
result = prune_missing_file_records_task.delay().get()
assert result["checked"] == 5
assert result["missing"] == 2
assert result["deleted"] == 2
db_sync.expire_all()
surviving = db_sync.execute(
select(ImageRecord.id).where(ImageRecord.id.in_(present + missing))
).scalars().all()
assert set(surviving) == set(present)
@pytest.mark.asyncio
async def test_prune_missing_file_records_aborts_when_mostly_missing(db_sync, tmp_path):
"""NFS-stall guard: a large sample that's mostly missing means the
filesystem is unhealthy, not that the library evaporated — abort, delete
nothing."""
from backend.app.tasks.admin import prune_missing_file_records_task
a = Artist(name="Stall", slug="stall")
db_sync.add(a)
db_sync.flush()
ids = [
_img(db_sync, a, path=tmp_path / f"phantom{i}.jpg", sha=f"{i:064x}")
for i in range(55) # >= _ORPHAN_MIN_SAMPLE, all missing -> frac 1.0
]
db_sync.commit()
result = prune_missing_file_records_task.delay().get()
assert result["deleted"] == 0
assert "aborted" in result
db_sync.expire_all()
surviving = db_sync.execute(
select(func.count(ImageRecord.id)).where(ImageRecord.id.in_(ids))
).scalar_one()
assert surviving == 55
@pytest.mark.asyncio
async def test_bulk_delete_images_task_idempotent_on_empty_list(db_sync):
from backend.app.tasks.admin import bulk_delete_images_task
result = bulk_delete_images_task.delay(image_ids=[]).get()
assert result["images_deleted"] == 0
@pytest.mark.asyncio
async def test_bulk_delete_images_task_reports_missing_ids(db_sync):
from backend.app.tasks.admin import bulk_delete_images_task
result = bulk_delete_images_task.delay(
image_ids=[9_999_998, 9_999_999],
).get()
assert result["images_deleted"] == 0
assert sorted(result["missing_ids"]) == [9_999_998, 9_999_999]