feat(ml): operator model swap — GPU re-embed + embedder as a setting (#1190)

Make the SigLIP embedder an operator choice (drop-in to SigLIP 2:
google/siglip2-so400m-patch16-512 is a verified 1152-d model at 512px → no
schema change, better small-cue fidelity). A swap = set model + re-embed +
retrain, all operator-driven; the GPU agent does the re-embed so it's fast.

- settings: embedder_model_name is now a setting (migration 0065) alongside the
  existing embedder_model_version; both editable + validated (non-empty) in the
  ml admin API. The server embedder loads by HF name (AutoImageProcessor/Model,
  model-agnostic), preferring the pre-downloaded local dir for the default so
  existing deploys don't re-download; rebuilds on a name change.
- agent: new 'embed' job = whole-image SigLIP embedding (mean-pool video frames)
  under the lease-announced model → POST /jobs/submit_embedding writes
  image_record.siglip_embedding + siglip_model_version. The lease now announces
  the model FROM THE SETTING (not a constant).
- re-embed routing: enqueue_gpu_backfill('embed') selects unembedded + stale-
  version images; 'siglip' now re-embeds concept crops whose version != current
  (so a swap re-triggers crops, not just the never-embedded back-catalogue). The
  CPU ml-worker backfill no longer re-embeds on a version mismatch (it can't
  churn the library at 512px) — the GPU agent owns version re-embeds. Daily
  'embed' + 'siglip' beats self-heal.
- scoring: score_image only bags embeddings in the CURRENT model's space (whole-
  image gated by siglip_model_version, concept regions by embedding_version) so a
  mid-swap stale vector isn't scored by new-space heads; legacy NULL = current.
- UI: GpuAgentCard "Embedding model (advanced)" — edit name/version, Save, and
  "Re-embed library (GPU)" (queues embed + siglip); points at SigLIP 2.

Tests: lease announces model + submit_embedding round-trip; enqueue 'embed'
selects stale/unembedded; stale-version excluded from scoring; embedder model
settable + empty rejected; siglip gate updated to current-version concept.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa

tests/test_api_gpu.py

@@ -69,6 +69,39 @@ async def test_lease_submit_round_trip(client, db):
     assert len(regs) == 1 and len(list(regs[0].ccip_embedding)) == 768
 
 
+@pytest.mark.asyncio
+async def test_lease_announces_embed_model_then_submit_embedding(client, db):
+    # Whole-image GPU re-embed (#1190): the lease announces the embedder model so
+    # the agent loads the right one, and submit_embedding writes it back onto
+    # image_record with its version stamp.
+    img = await _img(db, "b" * 64)
+    await GpuJobService(db).enqueue(img.id, "embed")
+    await db.commit()
+
+    token = (await (await client.post("/api/gpu/token/rotate")).get_json())["token"]
+    hdr = {"Authorization": f"Bearer {token}"}
+
+    leased = await client.post(
+        "/api/gpu/jobs/lease", json={"agent_id": "a1", "batch_size": 5}, headers=hdr,
+    )
+    j = (await leased.get_json())["jobs"][0]
+    assert j["task"] == "embed"
+    assert j["embed_model_name"] and j["embed_version"]   # server-announced model
+
+    submitted = await client.post("/api/gpu/jobs/submit_embedding", json={
+        "agent_id": "a1", "job_id": j["job_id"],
+        "embedding": [0.2] * 1152, "embedding_version": "siglip2-test-v9",
+    }, headers=hdr)
+    assert submitted.status_code == 200
+
+    st = await (await client.get("/api/gpu/status")).get_json()
+    assert st["done"] == 1 and st["leased"] == 0
+
+    await db.refresh(img)
+    assert img.siglip_model_version == "siglip2-test-v9"
+    assert img.siglip_embedding is not None and len(list(img.siglip_embedding)) == 1152
+
+
 @pytest.mark.asyncio
 async def test_submit_with_stale_lease_is_409(client, db):
     img = await _img(db, "b" * 64)

tests/test_api_ml_admin.py

@@ -34,6 +34,26 @@ async def test_get_and_patch_settings(client):
     assert (await resp.get_json())["suggestion_threshold_general"] == pytest.approx(0.90)
 
 
+@pytest.mark.asyncio
+async def test_embedder_model_settable_and_empty_rejected(client):
+    # #1190: the embedder model name + version are operator-settable (a swap),
+    # and neither may be blanked.
+    body = await (await client.get("/api/ml/settings")).get_json()
+    assert body["embedder_model_name"] == "google/siglip-so400m-patch14-384"
+
+    ok = await client.patch("/api/ml/settings", json={
+        "embedder_model_name": "google/siglip2-so400m-patch16-512",
+        "embedder_model_version": "siglip2-so400m-patch16-512",
+    })
+    assert ok.status_code == 200
+    out = await ok.get_json()
+    assert out["embedder_model_name"] == "google/siglip2-so400m-patch16-512"
+    assert out["embedder_model_version"] == "siglip2-so400m-patch16-512"
+
+    bad = await client.patch("/api/ml/settings", json={"embedder_model_name": "  "})
+    assert bad.status_code == 400
+
+
 @pytest.mark.asyncio
 async def test_tagger_store_floor_default_and_patch(client):
     body = await (await client.get("/api/ml/settings")).get_json()

tests/test_gpu_jobs.py

@@ -25,13 +25,17 @@ async def test_enqueue_siglip_backfill_gates_on_concept_region(db):
     # 'siglip' backfill enqueues images that lack a concept region (the
     # back-catalogue) and skips ones that already have one — and never double-
     # enqueues an image that already has a pending siglip job.
+    from backend.app.models import MLSettings
     from backend.app.tasks.ml import enqueue_gpu_backfill
 
+    cur = (await db.execute(
+        select(MLSettings.embedder_model_version).where(MLSettings.id == 1)
+    )).scalar_one()
     need = await _img(db, "e1" * 32)          # no concept region → wants one
-    have = await _img(db, "e2" * 32)          # already embedded → skip
+    have = await _img(db, "e2" * 32)          # concept @ current version → skip
     db.add(ImageRegion(
         image_record_id=have.id, kind="concept", rx=0.0, ry=0.0, rw=1.0, rh=1.0,
-        siglip_embedding=[0.0] * 1152, embedding_version="siglip-test",
+        siglip_embedding=[0.0] * 1152, embedding_version=cur,
     ))
     await db.commit()
 
@@ -57,6 +61,36 @@ async def test_enqueue_siglip_backfill_gates_on_concept_region(db):
     assert n_for_need == 1
 
 
+@pytest.mark.asyncio
+async def test_enqueue_embed_backfill_selects_stale_and_unembedded(db):
+    # Whole-image GPU re-embed (#1190): enqueue images with no embedding or one
+    # stamped under a DIFFERENT model version (an operator swap); skip ones
+    # already at the current version.
+    from backend.app.models import MLSettings
+    from backend.app.tasks.ml import enqueue_gpu_backfill
+
+    cur = (await db.execute(
+        select(MLSettings.embedder_model_version).where(MLSettings.id == 1)
+    )).scalar_one()
+    current = await _img(db, "f1" * 32)
+    current.siglip_embedding = [0.0] * 1152
+    current.siglip_model_version = cur                 # up to date → skip
+    stale = await _img(db, "f2" * 32)
+    stale.siglip_embedding = [0.0] * 1152
+    stale.siglip_model_version = "old-embedder-v0"      # wrong space → re-embed
+    unembedded = await _img(db, "f3" * 32)             # never embedded → embed
+    await db.commit()
+
+    assert enqueue_gpu_backfill("embed") >= 2
+    queued = {
+        j.image_record_id for j in (
+            await db.execute(select(GpuJob).where(GpuJob.task == "embed"))
+        ).scalars()
+    }
+    assert stale.id in queued and unembedded.id in queued
+    assert current.id not in queued
+
+
 @pytest.mark.asyncio
 async def test_enqueue_dedupes_same_pair(db):
     img = await _img(db, "a" * 64)

tests/test_ml_suggestions.py

@@ -145,6 +145,20 @@ async def test_concept_region_surfaces_via_max_over_bag(db):
     assert any(s.canonical_tag_id == tag.id and s.score > 0.7 for s in general)
 
 
+@pytest.mark.asyncio
+async def test_stale_embedding_version_excluded_from_scoring(db):
+    # Mid model-swap (#1190): an image still carrying an OLD-version whole-image
+    # embedding must NOT be scored by heads trained in the new model's space —
+    # even though the vector aligns with the head, it's the wrong coordinate
+    # system, so nothing surfaces until it's re-embedded.
+    tag = await TagService(db).find_or_create("glasses", TagKind.general)
+    img = await _img(db, "c1" * 32, _emb(0))
+    img.siglip_model_version = "some-old-model-v0"      # != current embedder
+    await _head(db, tag.id, slot=0, suggest_threshold=0.5)
+    await db.commit()
+    assert not (await SuggestionService(db).for_image(img.id)).by_category.get("general")
+
+
 @pytest.mark.asyncio
 async def test_rejected_tag_surfaced_flagged_then_reversible(db):
     # A dismissed suggestion is NOT dropped: it stays flagged rejected so the