feat(ml): DB-backed tagger_store_floor (default 0.70), the ingest confidence floor

Promotes the prediction store-floor from the TAGGER_STORE_FLOOR env (default
0.05) to a DB-backed, Settings-UI-tunable ml_settings column (default 0.70).
Storing every tag down to 0.05 from a ~10k-tag tagger is what grew
image_record's TOAST to ~100 GB; the suggestion path already filters at 0.70
and the centroid/learned path covers lower-confidence preferred tags, so the
sub-0.70 tail is redundant. Foundation for plan-task #764 (backfill + reclaim
land next; this only changes the write gate for NEW imports).

- ml_settings.tagger_store_floor (migration 0044, default 0.70)
- tagger.Tagger.infer(store_floor=...); ml task passes settings.tagger_store_floor
- ML admin GET/PATCH expose it; PATCH rejects a category suggestion threshold
  below the floor (nothing below the floor is stored, so the gap surfaces
  nothing) — server backstop for the UI slider clamp
- Settings → ML: store-floor slider + caption; category sliders min-bound to it

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

alembic/versions/0044_ml_settings_tagger_store_floor.py

@@ -0,0 +1,37 @@
+"""ml_settings.tagger_store_floor
+
+The ingest confidence floor below which tagger predictions are not stored,
+promoted from the TAGGER_STORE_FLOOR env var to a DB-backed, UI-tunable
+setting. Default 0.70 (was an env default of 0.05): the suggestion path
+already filters at 0.70 and the centroid/learned path covers low-confidence
+preferred tags, so the sub-0.70 tail was redundant weight — it had grown
+image_record's TOAST to ~100 GB. See plan-task #764.
+
+Revision ID: 0044
+Revises: 0043
+Create Date: 2026-06-10
+
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0044"
+down_revision: Union[str, None] = "0043"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "ml_settings",
+        sa.Column(
+            "tagger_store_floor", sa.Float(),
+            nullable=False, server_default="0.7",
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("ml_settings", "tagger_store_floor")

backend/app/api/ml_admin.py

@@ -13,6 +13,7 @@ _EDITABLE = (
     "suggestion_threshold_general",
     "centroid_similarity_threshold",
     "min_reference_images",
+    "tagger_store_floor",
 )
 
 
@@ -30,6 +31,7 @@ async def get_settings():
             "suggestion_threshold_general": s.suggestion_threshold_general,
             "centroid_similarity_threshold": s.centroid_similarity_threshold,
             "min_reference_images": s.min_reference_images,
+            "tagger_store_floor": s.tagger_store_floor,
             "tagger_model_version": s.tagger_model_version,
             "embedder_model_version": s.embedder_model_version,
         }
@@ -47,13 +49,45 @@ async def patch_settings():
         s = (
             await session.execute(select(MLSettings).where(MLSettings.id == 1))
         ).scalar_one()
+
+        # Merge the patch over current values, then validate the result as a
+        # whole — the store-floor invariant couples three fields, so they
+        # can't be checked one at a time.
+        proposed = {f: getattr(s, f) for f in _EDITABLE}
         for field in _EDITABLE:
             if field in body:
-                setattr(s, field, body[field])
+                proposed[field] = body[field]
+
+        err = _validate(proposed)
+        if err is not None:
+            return jsonify({"error": err}), 400
+
+        for field in _EDITABLE:
+            setattr(s, field, proposed[field])
         await session.commit()
     return await get_settings()
 
 
+def _validate(p: dict) -> str | None:
+    """Returns an error string if the proposed settings are invalid, else None.
+
+    Invariant (plan-task #764): the per-category suggestion thresholds can't
+    drop below tagger_store_floor — nothing below the floor is stored, so a
+    lower threshold would silently surface nothing in that gap. The UI clamps
+    the sliders to the floor; this is the server-side backstop.
+    """
+    floor = p["tagger_store_floor"]
+    if not (0.0 <= floor <= 1.0):
+        return "tagger_store_floor must be between 0 and 1"
+    for cat in ("character", "general"):
+        if p[f"suggestion_threshold_{cat}"] < floor:
+            return (
+                f"suggestion_threshold_{cat} cannot be below tagger_store_floor "
+                f"({floor}) — predictions below the floor are not stored"
+            )
+    return None
+
+
 @ml_admin_bp.route("/backfill", methods=["POST"])
 async def trigger_backfill():
     from ..tasks.ml import backfill

backend/app/models/ml_settings.py

@@ -28,6 +28,15 @@ class MLSettings(Base):
     centroid_similarity_threshold: Mapped[float] = mapped_column(
         Float, nullable=False, default=0.55
     )
+    # Ingest floor: tagger predictions below this confidence are not stored
+    # (tagger.Tagger.infer). Default 0.70 — the suggestion path already
+    # filters at 0.70 and the centroid/learned path covers low-confidence
+    # preferred tags, so the sub-0.70 tail is redundant weight (it had
+    # bloated image_record's TOAST to ~100 GB; plan-task #764). Operator-
+    # tunable via Settings → ML; must stay ≤ the suggestion thresholds.
+    tagger_store_floor: Mapped[float] = mapped_column(
+        Float, nullable=False, default=0.70
+    )
     min_reference_images: Mapped[int] = mapped_column(
         Integer, nullable=False, default=5
     )

backend/app/services/ml/tagger.py

@@ -33,8 +33,13 @@ _MODEL_DIR = Path(os.environ.get("ML_MODEL_DIR", "/models")) / "camie"
 _MODEL_FILE = f"{MODEL_NAME}.onnx"
 _METADATA_FILE = f"{MODEL_NAME}-metadata.json"
 
-# Below this confidence, predictions aren't stored (keeps the JSON compact).
-STORE_FLOOR = float(os.environ.get("TAGGER_STORE_FLOOR", "0.05"))
+# Ingest floor below which predictions aren't stored (keeps the JSON compact).
+# DEFAULT/fallback only — the live value is DB-backed
+# (ml_settings.tagger_store_floor) and passed into infer() per call by the ml
+# task. 0.70: the suggestion path already filters there and the centroid path
+# covers lower-confidence preferred tags, so the sub-0.70 tail is redundant
+# (it had bloated image_record's TOAST to ~100 GB; plan-task #764).
+DEFAULT_STORE_FLOOR = 0.70
 
 # The categories FC-2b surfaces in the UI. Others (meta/rating/year) are
 # still stored but the suggestion service filters them out.
@@ -145,10 +150,13 @@ class Tagger:
         arr = arr.transpose(2, 0, 1)                        # HWC -> CHW
         return arr[np.newaxis, :, :, :]                     # NCHW
 
-    def infer(self, image_path: Path) -> dict[str, TagPrediction]:
+    def infer(
+        self, image_path: Path, *, store_floor: float = DEFAULT_STORE_FLOOR,
+    ) -> dict[str, TagPrediction]:
         """Run Camie v2 on one image. Returns {name: TagPrediction} with
-        confidence >= STORE_FLOOR (across all categories — the suggestion
-        service does category filtering later).
+        confidence >= store_floor (across all categories — the suggestion
+        service does category filtering later). store_floor is the DB-backed
+        ml_settings.tagger_store_floor, passed in by the ml task.
 
         v2 emits multiple outputs; we use the refined predictions
         (output[1] per onnx_inference.py). Sigmoid is applied to raw
@@ -167,7 +175,7 @@ class Tagger:
         cats = self._tag_categories
         for idx, score in enumerate(probs):
             conf = float(score)
-            if conf < STORE_FLOOR:
+            if conf < store_floor:
                 continue
             if idx >= len(names):
                 # Output longer than metadata declared — shouldn't happen but

backend/app/tasks/ml.py

@@ -127,7 +127,10 @@ def tag_and_embed(self, image_id: int) -> dict:
                 phase = "video_infer"
                 import numpy as np
 
-                preds = _maxpool_predictions([tagger.infer(f) for f in frames])
+                preds = _maxpool_predictions(
+                    [tagger.infer(f, store_floor=settings.tagger_store_floor)
+                     for f in frames]
+                )
                 embedding = np.mean(
                     [embedder.infer(f) for f in frames], axis=0
                 ).astype("float32")
@@ -136,7 +139,7 @@ def tag_and_embed(self, image_id: int) -> dict:
             else:
                 phase = "tag"
                 t0 = time.monotonic()
-                raw = tagger.infer(src)
+                raw = tagger.infer(src, store_floor=settings.tagger_store_floor)
                 log.info(
                     "tag_and_embed tagged in %.1fs (%d tags): %s",
                     time.monotonic() - t0, len(raw), ctx,

frontend/src/components/settings/MLThresholdSliders.vue

@@ -6,9 +6,28 @@
         <v-col cols="12">
           <v-slider
             v-model="local[f.key]" :label="f.label"
+            :min="f.floorMin ? local.tagger_store_floor : 0" max="1" step="0.05"
+            thumb-label hide-details
+            color="accent" @end="save"
+          />
+        </v-col>
+      </v-row>
+
+      <v-divider class="my-4" />
+
+      <v-row>
+        <v-col cols="12">
+          <v-slider
+            v-model="local.tagger_store_floor" label="Tagger store floor"
             min="0" max="1" step="0.05" thumb-label hide-details
             color="accent" @end="save"
           />
+          <div class="text-caption fc-muted mt-1">
+            Tagger predictions below this confidence aren't stored — raising it
+            keeps the image library lean. Suggestions can't be shown below the
+            floor; lower-confidence tags you actually want still surface through
+            the learned centroid path.
+          </div>
         </v-col>
       </v-row>
     </v-card-text>
@@ -24,17 +43,25 @@ import { useMLStore } from '../../stores/ml.js'
 const store = useMLStore()
 // 'artist' (FC-2d-vii-c) and 'copyright' (2026-06-01) retired as
 // suggestion categories; their threshold rows are gone.
+// floorMin: the per-category suggestion thresholds can't drop below the
+// tagger store floor (nothing below the floor is stored to surface).
 const fields = [
-  { key: 'suggestion_threshold_character', label: 'Character' },
-  { key: 'suggestion_threshold_general', label: 'General' },
+  { key: 'suggestion_threshold_character', label: 'Character', floorMin: true },
+  { key: 'suggestion_threshold_general', label: 'General', floorMin: true },
   { key: 'centroid_similarity_threshold', label: 'Centroid similarity' }
 ]
 const local = reactive({})
 watch(() => store.settings, (s) => { if (s) Object.assign(local, s) }, { immediate: true })
 
 async function save() {
+  // Mirror the server invariant: keep the category thresholds at or above the
+  // store floor so a raised floor doesn't leave a threshold stranded below it.
+  const floor = local.tagger_store_floor
+  local.suggestion_threshold_character = Math.max(local.suggestion_threshold_character, floor)
+  local.suggestion_threshold_general = Math.max(local.suggestion_threshold_general, floor)
   const patch = {}
   for (const f of fields) patch[f.key] = local[f.key]
+  patch.tagger_store_floor = local.tagger_store_floor
   try { await store.patchSettings(patch) }
   catch (e) { toast({ text: e.message, type: 'error' }) }
 }

tests/test_api_ml_admin.py

@@ -34,6 +34,28 @@ async def test_get_and_patch_settings(client):
     assert (await resp.get_json())["suggestion_threshold_general"] == pytest.approx(0.90)
 
 
+@pytest.mark.asyncio
+async def test_tagger_store_floor_default_and_patch(client):
+    body = await (await client.get("/api/ml/settings")).get_json()
+    assert body["tagger_store_floor"] == pytest.approx(0.70)
+
+    resp = await client.patch("/api/ml/settings", json={"tagger_store_floor": 0.6})
+    assert resp.status_code == 200
+    assert (await resp.get_json())["tagger_store_floor"] == pytest.approx(0.6)
+
+
+@pytest.mark.asyncio
+async def test_suggestion_threshold_below_store_floor_rejected(client):
+    # Invariant (#764): a category threshold can't sit below the store floor —
+    # nothing below the floor is stored, so the gap would surface nothing.
+    # Floor defaults to 0.70; pushing general down to 0.50 must 400.
+    resp = await client.patch(
+        "/api/ml/settings", json={"suggestion_threshold_general": 0.50}
+    )
+    assert resp.status_code == 400
+    assert "tagger_store_floor" in (await resp.get_json())["error"]
+
+
 @pytest.mark.asyncio
 async def test_backfill_and_recompute_trigger(client):
     r1 = await client.post("/api/ml/backfill")

tests/test_ml_tagger.py

@@ -1,14 +1,14 @@
 """Tagger unit tests. The ONNX model isn't available in CI (it's a 1GB
-download into /models), so these test the pure-logic surface: STORE_FLOOR
-constant, SURFACED_CATEGORIES set, TagPrediction dataclass, and the
-load()-missing-file error path. Full inference is exercised by the local
-integration suite against a real /models volume.
+download into /models), so these test the pure-logic surface:
+DEFAULT_STORE_FLOOR constant, SURFACED_CATEGORIES set, TagPrediction
+dataclass, and the load()-missing-file error path. Full inference is
+exercised by the local integration suite against a real /models volume.
 """
 
 import pytest
 
 from backend.app.services.ml.tagger import (
-    STORE_FLOOR,
+    DEFAULT_STORE_FLOOR,
     SURFACED_CATEGORIES,
     Tagger,
     TagPrediction,
@@ -26,8 +26,12 @@ def test_surfaced_categories():
     assert "copyright" not in SURFACED_CATEGORIES
 
 
-def test_store_floor_is_low():
-    assert 0 < STORE_FLOOR < 0.2
+def test_default_store_floor():
+    # Raised 0.05 → 0.70 (plan-task #764): the suggestion path filters at
+    # 0.70 and the centroid path covers lower-confidence preferred tags, so
+    # storing the sub-0.70 tail was redundant (100 GB of TOAST). The live
+    # value is DB-backed (ml_settings.tagger_store_floor); this is the default.
+    assert DEFAULT_STORE_FLOOR == 0.70
 
 
 def test_tag_prediction_dataclass():