feat(admin): prune_low_confidence_predictions backfill task + UI (#764)
The one-time backfill that actually shrinks the DB: drops stored tagger_predictions entries below ml_settings.tagger_store_floor from every image_record row, and clamps any allowlist min_confidence below the floor up to it. Keep predicate (confidence >= floor) mirrors Tagger.infer's store gate so backfilled rows match new imports. Keyset by id ASC, idempotent, self-resumes on the soft time limit; runs on the maintenance_long lane. pg_dump copies live data only, so this alone fixes the #739 backup timeout — the reclaim (VACUUM FULL / pg_repack on image_record) is a separate, optional disk-return step, brief because post-prune the live data is tiny. - admin.prune_low_confidence_predictions_task + POST /api/admin/maintenance/prune-predictions - PrunePredictionsCard in the Maintenance panel (shows the current floor) - tests: registration + prune-keeps->=floor/drops-<floor + allowlist clamp Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -42,6 +42,68 @@ def test_bulk_delete_images_task_registered():
|
||||
)
|
||||
|
||||
|
||||
def test_prune_low_confidence_predictions_task_registered():
|
||||
assert (
|
||||
"backend.app.tasks.admin.prune_low_confidence_predictions_task"
|
||||
in celery.tasks
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prune_low_confidence_predictions(db_sync, tmp_path):
|
||||
# #764: drop stored tagger predictions below the store floor (default
|
||||
# 0.70) and clamp allowlist thresholds up to it.
|
||||
from backend.app.models import Tag, TagAllowlist, TagKind
|
||||
from backend.app.tasks.admin import prune_low_confidence_predictions_task
|
||||
|
||||
f0 = tmp_path / "p0.jpg"
|
||||
f0.write_bytes(b"x")
|
||||
img0 = ImageRecord(
|
||||
path=str(f0), sha256=f"{0:064x}", size_bytes=10, mime="image/jpeg",
|
||||
origin="imported_filesystem",
|
||||
tagger_predictions={
|
||||
"keep_high": {"category": "general", "confidence": 0.92},
|
||||
"keep_edge": {"category": "general", "confidence": 0.70},
|
||||
"drop_mid": {"category": "general", "confidence": 0.40},
|
||||
"drop_tiny": {"category": "general", "confidence": 0.06},
|
||||
},
|
||||
)
|
||||
db_sync.add(img0)
|
||||
f1 = tmp_path / "p1.jpg"
|
||||
f1.write_bytes(b"x")
|
||||
img1 = ImageRecord(
|
||||
path=str(f1), sha256=f"{1:064x}", size_bytes=10, mime="image/jpeg",
|
||||
origin="imported_filesystem",
|
||||
tagger_predictions={"only": {"category": "general", "confidence": 0.99}},
|
||||
)
|
||||
db_sync.add(img1)
|
||||
tag = Tag(name="lowthr-tag", kind=TagKind.general)
|
||||
db_sync.add(tag)
|
||||
db_sync.flush()
|
||||
db_sync.add(TagAllowlist(tag_id=tag.id, min_confidence=0.30))
|
||||
db_sync.commit()
|
||||
img0_id, img1_id, tag_id = img0.id, img1.id, tag.id
|
||||
|
||||
result = prune_low_confidence_predictions_task.delay().get()
|
||||
assert result["floor"] == pytest.approx(0.70)
|
||||
assert result["pruned"] == 1 # only img0 had sub-floor entries
|
||||
assert result["allowlist_clamped"] == 1
|
||||
|
||||
db_sync.expire_all()
|
||||
p0 = db_sync.execute(
|
||||
select(ImageRecord.tagger_predictions).where(ImageRecord.id == img0_id)
|
||||
).scalar_one()
|
||||
assert set(p0) == {"keep_high", "keep_edge"} # >=0.70 kept, <0.70 dropped
|
||||
p1 = db_sync.execute(
|
||||
select(ImageRecord.tagger_predictions).where(ImageRecord.id == img1_id)
|
||||
).scalar_one()
|
||||
assert set(p1) == {"only"} # already clean — untouched
|
||||
clamped = db_sync.execute(
|
||||
select(TagAllowlist.min_confidence).where(TagAllowlist.tag_id == tag_id)
|
||||
).scalar_one()
|
||||
assert clamped == pytest.approx(0.70)
|
||||
|
||||
|
||||
# --- delete_artist_cascade_task -------------------------------------
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user