feat(ml): drop image_record.tagger_predictions — image_prediction is sole store (#768 step 3)

Read cutover verified in prod (suggestions + allowlist read image_prediction;
backfill complete at 908k rows / 51k images). Removes the old JSON column and
everything that fed it:

- ImageRecord.tagger_predictions column removed; migration 0046 DROPs it.
  tagger_model_version kept as the "tagged / current?" signal the backfill
  sweep reads (needs-tagging check switched to tagger_model_version IS NULL).
- tag_and_embed no longer dual-writes the JSON — image_prediction is the only
  write path.
- importer re-import reset drops the JSON line (image_prediction rows are
  already deleted on re-import).
- Retired the one-time #768 backfill task + the #764 prune task, their admin
  endpoints, and their Maintenance cards (Backfill/PrunePredictionsCard).
- Tests seed/assert via image_prediction; stale column refs removed.

Disk reclaim is NOT automatic: DROP COLUMN is a catalog change. Run
`VACUUM FULL image_record` off-hours afterward to return the ~100 GB to the OS
so DB backups go small (#739). image_prediction (~90 MB) stays in pg_dump — it's
the source of truth now.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

tests/test_migration_0003.py

@@ -24,8 +24,10 @@ def test_new_tables_registered():
 
 def test_image_record_columns_renamed():
     cols = {c.name for c in ImageRecord.__table__.columns}
-    assert "tagger_predictions" in cols
+    # tagger_predictions (the renamed wd14_predictions) was later dropped in
+    # migration 0046 — predictions live in image_prediction now (#768).
     assert "tagger_model_version" in cols
+    assert "tagger_predictions" not in cols
     assert "wd14_predictions" not in cols
     assert "wd14_model_version" not in cols
 

tests/test_phash_dedup.py

@@ -11,6 +11,7 @@ from PIL import Image
 from sqlalchemy import func, select
 
 from backend.app.models import (
+    ImagePrediction,
     ImageProvenance,
     ImageRecord,
     ImportSettings,
@@ -118,7 +119,11 @@ def test_smaller_existing_is_superseded(importer, import_layout):
             image_record_id=eid, tag_id=tag.id, source="manual"
         )
     )
-    old.tagger_predictions = {"x": 1}
+    importer.session.add(
+        ImagePrediction(
+            image_record_id=eid, raw_name="x", category="general", score=0.9
+        )
+    )
     old.siglip_embedding = [0.0] * 1152
     old.integrity_status = "ok"
     importer.session.commit()
@@ -136,7 +141,11 @@ def test_smaller_existing_is_superseded(importer, import_layout):
     assert row.path != old_path
     assert row.phash is not None
     assert row.integrity_status == "unknown"
-    assert row.tagger_predictions is None
+    # #768: re-import clears the normalized predictions too
+    assert importer.session.execute(
+        select(func.count()).select_from(ImagePrediction)
+        .where(ImagePrediction.image_record_id == eid)
+    ).scalar_one() == 0
     assert row.siglip_embedding is None
     linked = importer.session.execute(
         select(image_tag.c.tag_id).where(

tests/test_tag_merge.py

@@ -324,9 +324,7 @@ async def test_protective_alias_uses_tag_kind(db):
     # The protective alias category is the tag's KIND — the tagger maps each name
     # to exactly one category and a tag's kind is set from it, so kind already IS
     # the tagger's category. The merge no longer scans image_record's predictions
-    # to rediscover it. Even with a (contrived) differing prediction category
-    # present, the merge writes a single (name, kind) alias.
-    from backend.app.models import ImageRecord
+    # to rediscover it — it writes a single (name, kind) alias from the tag kind.
     from backend.app.models.tag_alias import TagAlias
 
     svc = TagService(db)
@@ -335,10 +333,6 @@ async def test_protective_alias_uses_tag_kind(db):
     img = await _img(db)
     # mark source machine-known so keep_as_alias is True
     await svc.add_to_image(img, a.id, source="ml_auto")
-    r1 = await db.get(ImageRecord, img)
-    r1.tagger_predictions = {
-        "predname": {"category": "copyright", "confidence": 0.8}
-    }
     await db.flush()
     result = await svc.merge(a.id, b.id)
     assert result.alias_created is True
@@ -381,7 +375,6 @@ async def test_alias_fallback_to_kind_when_no_predictions(db):
 
 @pytest.mark.asyncio
 async def test_alias_create_does_not_clobber_existing(db):
-    from backend.app.models import ImageRecord
     from backend.app.models.tag_alias import TagAlias
 
     svc = TagService(db)
@@ -397,10 +390,6 @@ async def test_alias_create_does_not_clobber_existing(db):
     )
     img = await _img(db)
     await svc.add_to_image(img, a.id, source="ml_auto")
-    r = await db.get(ImageRecord, img)
-    r.tagger_predictions = {
-        "dupalias": {"category": "general", "confidence": 0.9}
-    }
     await db.flush()
     await svc.merge(a.id, b.id)
     cid = await db.scalar(

tests/test_tasks_admin.py

@@ -42,75 +42,6 @@ def test_bulk_delete_images_task_registered():
     )
 
 
-def test_prune_low_confidence_predictions_task_registered():
-    assert (
-        "backend.app.tasks.admin.prune_low_confidence_predictions_task"
-        in celery.tasks
-    )
-
-
-def test_backfill_image_predictions_task_registered():
-    assert (
-        "backend.app.tasks.admin.backfill_image_predictions_task"
-        in celery.tasks
-    )
-
-
-@pytest.mark.asyncio
-async def test_prune_low_confidence_predictions(db_sync, tmp_path):
-    # #764: drop stored tagger predictions below the store floor (default
-    # 0.70) and clamp allowlist thresholds up to it.
-    from backend.app.models import Tag, TagAllowlist, TagKind
-    from backend.app.tasks.admin import prune_low_confidence_predictions_task
-
-    f0 = tmp_path / "p0.jpg"
-    f0.write_bytes(b"x")
-    img0 = ImageRecord(
-        path=str(f0), sha256=f"{0:064x}", size_bytes=10, mime="image/jpeg",
-        origin="imported_filesystem",
-        tagger_predictions={
-            "keep_high": {"category": "general", "confidence": 0.92},
-            "keep_edge": {"category": "general", "confidence": 0.70},
-            "drop_mid": {"category": "general", "confidence": 0.40},
-            "drop_tiny": {"category": "general", "confidence": 0.06},
-        },
-    )
-    db_sync.add(img0)
-    f1 = tmp_path / "p1.jpg"
-    f1.write_bytes(b"x")
-    img1 = ImageRecord(
-        path=str(f1), sha256=f"{1:064x}", size_bytes=10, mime="image/jpeg",
-        origin="imported_filesystem",
-        tagger_predictions={"only": {"category": "general", "confidence": 0.99}},
-    )
-    db_sync.add(img1)
-    tag = Tag(name="lowthr-tag", kind=TagKind.general)
-    db_sync.add(tag)
-    db_sync.flush()
-    db_sync.add(TagAllowlist(tag_id=tag.id, min_confidence=0.30))
-    db_sync.commit()
-    img0_id, img1_id, tag_id = img0.id, img1.id, tag.id
-
-    result = prune_low_confidence_predictions_task.delay().get()
-    assert result["floor"] == pytest.approx(0.70)
-    assert result["pruned"] == 1            # only img0 had sub-floor entries
-    assert result["allowlist_clamped"] == 1
-
-    db_sync.expire_all()
-    p0 = db_sync.execute(
-        select(ImageRecord.tagger_predictions).where(ImageRecord.id == img0_id)
-    ).scalar_one()
-    assert set(p0) == {"keep_high", "keep_edge"}   # >=0.70 kept, <0.70 dropped
-    p1 = db_sync.execute(
-        select(ImageRecord.tagger_predictions).where(ImageRecord.id == img1_id)
-    ).scalar_one()
-    assert set(p1) == {"only"}                      # already clean — untouched
-    clamped = db_sync.execute(
-        select(TagAllowlist.min_confidence).where(TagAllowlist.tag_id == tag_id)
-    ).scalar_one()
-    assert clamped == pytest.approx(0.70)
-
-
 # --- delete_artist_cascade_task -------------------------------------
 
 

tests/test_tasks_ml.py

@@ -44,7 +44,7 @@ async def test_backfill_enqueues_missing(db, monkeypatch):
         path="/images/n.jpg", sha256="n" * 64, size_bytes=1,
         mime="image/jpeg", width=1, height=1,
         origin="imported_filesystem", integrity_status="unknown",
-        tagger_predictions=None, siglip_embedding=None,
+        siglip_embedding=None,
     )
     db.add(img)
     await db.commit()