Files
FabledCurator/tests/test_tasks_admin.py
T
bvandeusen d55e52ae9b
CI / lint (push) Successful in 3s
CI / frontend-build (push) Successful in 20s
CI / backend-lint-and-test (push) Successful in 32s
CI / integration (push) Successful in 3m13s
feat(admin): prune_low_confidence_predictions backfill task + UI (#764)
The one-time backfill that actually shrinks the DB: drops stored
tagger_predictions entries below ml_settings.tagger_store_floor from every
image_record row, and clamps any allowlist min_confidence below the floor up
to it. Keep predicate (confidence >= floor) mirrors Tagger.infer's store gate
so backfilled rows match new imports. Keyset by id ASC, idempotent,
self-resumes on the soft time limit; runs on the maintenance_long lane.

pg_dump copies live data only, so this alone fixes the #739 backup timeout —
the reclaim (VACUUM FULL / pg_repack on image_record) is a separate, optional
disk-return step, brief because post-prune the live data is tiny.

- admin.prune_low_confidence_predictions_task + POST /api/admin/maintenance/prune-predictions
- PrunePredictionsCard in the Maintenance panel (shows the current floor)
- tests: registration + prune-keeps->=floor/drops-<floor + allowlist clamp

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-10 13:57:39 -04:00

221 lines
7.1 KiB
Python

"""FC-3k: admin Celery task integration tests.
task_always_eager + signal handlers from FC-3i populate task_run.
We assert the wrapper passes args through correctly and that
task_run lifecycle status flips as expected.
"""
import pytest
from sqlalchemy import func, select
import backend.app.tasks.admin # noqa: F401 — register tasks
from backend.app.celery_app import celery
from backend.app.models import Artist, ImageRecord, TaskRun
pytestmark = pytest.mark.integration
@pytest.fixture(autouse=True)
def _eager_celery(monkeypatch):
monkeypatch.setattr(celery.conf, "task_always_eager", True)
monkeypatch.setattr(celery.conf, "task_eager_propagates", False)
@pytest.fixture(autouse=True)
def fake_images_root(monkeypatch, tmp_path):
monkeypatch.setattr("backend.app.tasks.admin.IMAGES_ROOT", tmp_path)
# --- registration ----------------------------------------------------
def test_delete_artist_cascade_task_registered():
assert (
"backend.app.tasks.admin.delete_artist_cascade_task"
in celery.tasks
)
def test_bulk_delete_images_task_registered():
assert (
"backend.app.tasks.admin.bulk_delete_images_task"
in celery.tasks
)
def test_prune_low_confidence_predictions_task_registered():
assert (
"backend.app.tasks.admin.prune_low_confidence_predictions_task"
in celery.tasks
)
@pytest.mark.asyncio
async def test_prune_low_confidence_predictions(db_sync, tmp_path):
# #764: drop stored tagger predictions below the store floor (default
# 0.70) and clamp allowlist thresholds up to it.
from backend.app.models import Tag, TagAllowlist, TagKind
from backend.app.tasks.admin import prune_low_confidence_predictions_task
f0 = tmp_path / "p0.jpg"
f0.write_bytes(b"x")
img0 = ImageRecord(
path=str(f0), sha256=f"{0:064x}", size_bytes=10, mime="image/jpeg",
origin="imported_filesystem",
tagger_predictions={
"keep_high": {"category": "general", "confidence": 0.92},
"keep_edge": {"category": "general", "confidence": 0.70},
"drop_mid": {"category": "general", "confidence": 0.40},
"drop_tiny": {"category": "general", "confidence": 0.06},
},
)
db_sync.add(img0)
f1 = tmp_path / "p1.jpg"
f1.write_bytes(b"x")
img1 = ImageRecord(
path=str(f1), sha256=f"{1:064x}", size_bytes=10, mime="image/jpeg",
origin="imported_filesystem",
tagger_predictions={"only": {"category": "general", "confidence": 0.99}},
)
db_sync.add(img1)
tag = Tag(name="lowthr-tag", kind=TagKind.general)
db_sync.add(tag)
db_sync.flush()
db_sync.add(TagAllowlist(tag_id=tag.id, min_confidence=0.30))
db_sync.commit()
img0_id, img1_id, tag_id = img0.id, img1.id, tag.id
result = prune_low_confidence_predictions_task.delay().get()
assert result["floor"] == pytest.approx(0.70)
assert result["pruned"] == 1 # only img0 had sub-floor entries
assert result["allowlist_clamped"] == 1
db_sync.expire_all()
p0 = db_sync.execute(
select(ImageRecord.tagger_predictions).where(ImageRecord.id == img0_id)
).scalar_one()
assert set(p0) == {"keep_high", "keep_edge"} # >=0.70 kept, <0.70 dropped
p1 = db_sync.execute(
select(ImageRecord.tagger_predictions).where(ImageRecord.id == img1_id)
).scalar_one()
assert set(p1) == {"only"} # already clean — untouched
clamped = db_sync.execute(
select(TagAllowlist.min_confidence).where(TagAllowlist.tag_id == tag_id)
).scalar_one()
assert clamped == pytest.approx(0.70)
# --- delete_artist_cascade_task -------------------------------------
@pytest.mark.asyncio
async def test_delete_artist_cascade_task_removes_artist_and_records_ok(
db_sync, tmp_path,
):
from backend.app.tasks.admin import delete_artist_cascade_task
a = Artist(name="Doomed", slug="doomed")
db_sync.add(a)
db_sync.flush()
for i in range(2):
f = tmp_path / f"d{i}.jpg"
f.write_bytes(b"x")
db_sync.add(ImageRecord(
artist_id=a.id, path=str(f),
sha256=f"{i:064x}", size_bytes=10, mime="image/jpeg",
origin="imported_filesystem",
))
db_sync.commit()
artist_id = a.id
result = delete_artist_cascade_task.delay(artist_id=artist_id).get()
assert result["summary"]["images_deleted"] == 2
surviving = db_sync.execute(
select(func.count(Artist.id)).where(Artist.id == artist_id)
).scalar_one()
assert surviving == 0
# FC-3i task_run lifecycle: task should have an 'ok' row.
status = db_sync.execute(
select(TaskRun.status)
.where(TaskRun.task_name.endswith(".delete_artist_cascade_task"))
.order_by(TaskRun.id.desc()).limit(1)
).scalar_one()
assert status == "ok"
@pytest.mark.asyncio
async def test_delete_artist_cascade_task_records_failure(
db_sync, monkeypatch,
):
from backend.app.tasks.admin import delete_artist_cascade_task
def _boom(*a, **kw):
raise RuntimeError("synthetic cascade fail")
monkeypatch.setattr(
"backend.app.services.cleanup_service.delete_artist_cascade", _boom,
)
with pytest.raises(RuntimeError):
delete_artist_cascade_task.delay(artist_id=1).get()
row = db_sync.execute(
select(TaskRun.status, TaskRun.error_message)
.where(TaskRun.task_name.endswith(".delete_artist_cascade_task"))
.order_by(TaskRun.id.desc()).limit(1)
).one()
assert row.status == "error"
assert "synthetic" in (row.error_message or "")
# --- bulk_delete_images_task ----------------------------------------
@pytest.mark.asyncio
async def test_bulk_delete_images_task_removes_listed_images(
db_sync, tmp_path,
):
from backend.app.tasks.admin import bulk_delete_images_task
a = Artist(name="B", slug="b")
db_sync.add(a)
db_sync.flush()
ids = []
for i in range(3):
f = tmp_path / f"b{i}.jpg"
f.write_bytes(b"x")
img = ImageRecord(
artist_id=a.id, path=str(f),
sha256=f"a{i:063x}", size_bytes=10, mime="image/jpeg",
origin="imported_filesystem",
)
db_sync.add(img)
db_sync.flush()
ids.append(img.id)
db_sync.commit()
result = bulk_delete_images_task.delay(image_ids=ids).get()
assert result["images_deleted"] == 3
surviving = db_sync.execute(
select(func.count(ImageRecord.id))
.where(ImageRecord.id.in_(ids))
).scalar_one()
assert surviving == 0
@pytest.mark.asyncio
async def test_bulk_delete_images_task_idempotent_on_empty_list(db_sync):
from backend.app.tasks.admin import bulk_delete_images_task
result = bulk_delete_images_task.delay(image_ids=[]).get()
assert result["images_deleted"] == 0
@pytest.mark.asyncio
async def test_bulk_delete_images_task_reports_missing_ids(db_sync):
from backend.app.tasks.admin import bulk_delete_images_task
result = bulk_delete_images_task.delay(
image_ids=[9_999_998, 9_999_999],
).get()
assert result["images_deleted"] == 0
assert sorted(result["missing_ids"]) == [9_999_998, 9_999_999]