feat(ml): drop image_record.tagger_predictions — image_prediction is sole store (#768 step 3)
Read cutover verified in prod (suggestions + allowlist read image_prediction; backfill complete at 908k rows / 51k images). Removes the old JSON column and everything that fed it: - ImageRecord.tagger_predictions column removed; migration 0046 DROPs it. tagger_model_version kept as the "tagged / current?" signal the backfill sweep reads (needs-tagging check switched to tagger_model_version IS NULL). - tag_and_embed no longer dual-writes the JSON — image_prediction is the only write path. - importer re-import reset drops the JSON line (image_prediction rows are already deleted on re-import). - Retired the one-time #768 backfill task + the #764 prune task, their admin endpoints, and their Maintenance cards (Backfill/PrunePredictionsCard). - Tests seed/assert via image_prediction; stale column refs removed. Disk reclaim is NOT automatic: DROP COLUMN is a catalog change. Run `VACUUM FULL image_record` off-hours afterward to return the ~100 GB to the OS so DB backups go small (#739). image_prediction (~90 MB) stays in pg_dump — it's the source of truth now. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -42,75 +42,6 @@ def test_bulk_delete_images_task_registered():
|
||||
)
|
||||
|
||||
|
||||
def test_prune_low_confidence_predictions_task_registered():
|
||||
assert (
|
||||
"backend.app.tasks.admin.prune_low_confidence_predictions_task"
|
||||
in celery.tasks
|
||||
)
|
||||
|
||||
|
||||
def test_backfill_image_predictions_task_registered():
|
||||
assert (
|
||||
"backend.app.tasks.admin.backfill_image_predictions_task"
|
||||
in celery.tasks
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prune_low_confidence_predictions(db_sync, tmp_path):
|
||||
# #764: drop stored tagger predictions below the store floor (default
|
||||
# 0.70) and clamp allowlist thresholds up to it.
|
||||
from backend.app.models import Tag, TagAllowlist, TagKind
|
||||
from backend.app.tasks.admin import prune_low_confidence_predictions_task
|
||||
|
||||
f0 = tmp_path / "p0.jpg"
|
||||
f0.write_bytes(b"x")
|
||||
img0 = ImageRecord(
|
||||
path=str(f0), sha256=f"{0:064x}", size_bytes=10, mime="image/jpeg",
|
||||
origin="imported_filesystem",
|
||||
tagger_predictions={
|
||||
"keep_high": {"category": "general", "confidence": 0.92},
|
||||
"keep_edge": {"category": "general", "confidence": 0.70},
|
||||
"drop_mid": {"category": "general", "confidence": 0.40},
|
||||
"drop_tiny": {"category": "general", "confidence": 0.06},
|
||||
},
|
||||
)
|
||||
db_sync.add(img0)
|
||||
f1 = tmp_path / "p1.jpg"
|
||||
f1.write_bytes(b"x")
|
||||
img1 = ImageRecord(
|
||||
path=str(f1), sha256=f"{1:064x}", size_bytes=10, mime="image/jpeg",
|
||||
origin="imported_filesystem",
|
||||
tagger_predictions={"only": {"category": "general", "confidence": 0.99}},
|
||||
)
|
||||
db_sync.add(img1)
|
||||
tag = Tag(name="lowthr-tag", kind=TagKind.general)
|
||||
db_sync.add(tag)
|
||||
db_sync.flush()
|
||||
db_sync.add(TagAllowlist(tag_id=tag.id, min_confidence=0.30))
|
||||
db_sync.commit()
|
||||
img0_id, img1_id, tag_id = img0.id, img1.id, tag.id
|
||||
|
||||
result = prune_low_confidence_predictions_task.delay().get()
|
||||
assert result["floor"] == pytest.approx(0.70)
|
||||
assert result["pruned"] == 1 # only img0 had sub-floor entries
|
||||
assert result["allowlist_clamped"] == 1
|
||||
|
||||
db_sync.expire_all()
|
||||
p0 = db_sync.execute(
|
||||
select(ImageRecord.tagger_predictions).where(ImageRecord.id == img0_id)
|
||||
).scalar_one()
|
||||
assert set(p0) == {"keep_high", "keep_edge"} # >=0.70 kept, <0.70 dropped
|
||||
p1 = db_sync.execute(
|
||||
select(ImageRecord.tagger_predictions).where(ImageRecord.id == img1_id)
|
||||
).scalar_one()
|
||||
assert set(p1) == {"only"} # already clean — untouched
|
||||
clamped = db_sync.execute(
|
||||
select(TagAllowlist.min_confidence).where(TagAllowlist.tag_id == tag_id)
|
||||
).scalar_one()
|
||||
assert clamped == pytest.approx(0.70)
|
||||
|
||||
|
||||
# --- delete_artist_cascade_task -------------------------------------
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user