feat(b3): ml-worker becomes optional — embed-only role, decoupled GPU coordination, cpu-embed switch

The ml-worker's ONLY processing role is now the CPU whole-image embed fallback
(tag_and_embed renamed embed_image — Camie tagging was retired #1189 and the
name kept implying otherwise; videos were already handled agent-style: frame
sampling + mean-pool). Detection/cropping/CCIP stay GPU-agent-only, and their
completion is judged per-pipeline: ccip by gpu_job rows, siglip by concept
regions at the current model version — never by image_record.siglip_embedding.
A CPU embed therefore can NEVER close crop work for the agent (regression test
pins this; only the whole-image 'embed' job, the same artifact, is satisfied).

Making removal actually safe (operator will drop the container):
- GPU-queue coordination (enqueue_gpu_backfill, recover_orphaned_gpu_jobs,
  reprocess_gpu_jobs) moved verbatim to tasks/gpu_queue.py on the maintenance
  quick lane — it lived on the 'ml' queue only by module colocation, which made
  the ml-worker a hard dependency of the whole agent pipeline.
- New ml_settings.cpu_embed_enabled (migration 0074, default ON so agent-less
  installs keep working): OFF stops the four import hooks queueing embed work
  nothing will consume and no-ops the manual backfill; switch lives on the
  renamed 'CPU embedding backfill' card.
- NB heads training / auto-apply still run on the ml image (sklearn) — a stack
  that removes the container gives those up too.

Deploy note: in-flight messages under the old task names are dropped by the
new workers; the 60s orphan sweep + hourly backfill re-fire under the new
names immediately.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM

tests/test_api_gpu.py

@@ -127,7 +127,7 @@ async def test_backfill_enqueues_then_is_idempotent(db):
     await _img(db, "c" * 64)
     await _img(db, "d" * 64)
     await db.commit()
-    from backend.app.tasks.ml import enqueue_gpu_backfill
+    from backend.app.tasks.gpu_queue import enqueue_gpu_backfill
 
     n = enqueue_gpu_backfill("ccip")     # sync task, own session
     assert n >= 2
@@ -260,3 +260,33 @@ async def test_errors_endpoint_reports_triage_view(client, db):
     assert item["reason_class"] == "truncated_or_corrupt"
     assert item["triage_status"] is None
     assert item["image_url"].startswith("/images/")
+
+
+@pytest.mark.asyncio
+async def test_cpu_embed_never_blocks_gpu_crop_backfills(db):
+    """B3 invariant (operator 2026-07-02): ccip (detect + character) and
+    siglip (concept crops) completion is judged per-pipeline — gpu_job rows and
+    image_region state — never inferred from image_record.siglip_embedding. So
+    an image the CPU fallback already embedded still gets both crop jobs; only
+    the whole-image 'embed' job (the SAME artifact the CPU path produces) is
+    satisfied by it."""
+    from backend.app.models import MLSettings
+    from backend.app.tasks.gpu_queue import enqueue_gpu_backfill
+
+    img = await _img(db, "7" * 64)
+    cur = (await db.execute(
+        select(MLSettings.embedder_model_version).where(MLSettings.id == 1)
+    )).scalar_one()
+    # As if the CPU fallback already embedded it under the current model.
+    img.siglip_embedding = [0.1] * 1152
+    img.siglip_model_version = cur
+    await db.commit()
+
+    assert enqueue_gpu_backfill("ccip") == 1      # crops still open
+    assert enqueue_gpu_backfill("siglip") == 1    # concept crops still open
+    assert enqueue_gpu_backfill("embed") == 0     # same artifact — already done
+
+    tasks = set((await db.execute(
+        select(GpuJob.task).where(GpuJob.image_record_id == img.id)
+    )).scalars().all())
+    assert tasks == {"ccip", "siglip"}

tests/test_download_service.py

@@ -922,7 +922,7 @@ async def test_download_enqueues_thumbnail_and_ml_per_attached_image(
         lambda image_id: thumb_calls.append(image_id),
     )
     monkeypatch.setattr(
-        ml_mod.tag_and_embed, "delay",
+        ml_mod.embed_image, "delay",
         lambda image_id: ml_calls.append(image_id),
     )
 

tests/test_external_worker.py

@@ -125,7 +125,7 @@ def test_refetch_same_link_keeps_canonical_file(db_sync, tmp_path, monkeypatch):
 
     import backend.app.tasks.ml as ml_mod
     import backend.app.tasks.thumbnail as thumb_mod
-    monkeypatch.setattr(ml_mod.tag_and_embed, "delay", lambda i: None)
+    monkeypatch.setattr(ml_mod.embed_image, "delay", lambda i: None)
     monkeypatch.setattr(thumb_mod.generate_thumbnail, "delay", lambda i: None)
 
     out = ext.fetch_external_link(link_id)
@@ -234,7 +234,7 @@ def test_downloaded_archive_gets_provenance_and_tagging(db_sync, tmp_path, monke
     tagged, thumbed = [], []
     import backend.app.tasks.ml as ml_mod
     import backend.app.tasks.thumbnail as thumb_mod
-    monkeypatch.setattr(ml_mod.tag_and_embed, "delay", lambda i: tagged.append(i))
+    monkeypatch.setattr(ml_mod.embed_image, "delay", lambda i: tagged.append(i))
     monkeypatch.setattr(thumb_mod.generate_thumbnail, "delay", lambda i: thumbed.append(i))
 
     out = ext.fetch_external_link(link_id)

tests/test_gpu_jobs.py

@@ -30,7 +30,7 @@ async def test_enqueue_siglip_backfill_gates_on_concept_region(db):
     # back-catalogue) and skips ones that already have one — and never double-
     # enqueues an image that already has a pending siglip job.
     from backend.app.models import MLSettings
-    from backend.app.tasks.ml import enqueue_gpu_backfill
+    from backend.app.tasks.gpu_queue import enqueue_gpu_backfill
 
     cur = (await db.execute(
         select(MLSettings.embedder_model_version).where(MLSettings.id == 1)
@@ -71,7 +71,7 @@ async def test_enqueue_embed_backfill_selects_stale_and_unembedded(db):
     # stamped under a DIFFERENT model version (an operator swap); skip ones
     # already at the current version.
     from backend.app.models import MLSettings
-    from backend.app.tasks.ml import enqueue_gpu_backfill
+    from backend.app.tasks.gpu_queue import enqueue_gpu_backfill
 
     cur = (await db.execute(
         select(MLSettings.embedder_model_version).where(MLSettings.id == 1)
@@ -99,7 +99,7 @@ async def test_enqueue_embed_backfill_selects_stale_and_unembedded(db):
 async def test_reprocess_resets_done_jobs_to_pending(db):
     # Re-process (#1202): done/error jobs of a task go back to pending so the
     # agent re-runs the whole library under the current pipeline.
-    from backend.app.tasks.ml import reprocess_gpu_jobs
+    from backend.app.tasks.gpu_queue import reprocess_gpu_jobs
 
     img = await _img(db, "r1" * 32)
     job = await GpuJobService(db).enqueue(img.id, "ccip")
@@ -274,7 +274,7 @@ async def test_backfill_skips_errored_images(db):
     # An errored job is a TOMBSTONE for its (image, task): no backfill variant
     # re-enqueues it — retry is deliberate-only (/retry_errors). Pre-fix, the
     # hourly ccip run minted a fresh doomed job per bad file forever.
-    from backend.app.tasks.ml import enqueue_gpu_backfill
+    from backend.app.tasks.gpu_queue import enqueue_gpu_backfill
 
     img = await _img(db, "f1" * 32)
     svc = GpuJobService(db)
@@ -294,7 +294,7 @@ async def test_backfill_prunes_moot_error_tombstones(db):
     # Loop-era duplicates: several error rows for one (image, task), all made
     # moot by a later done row. The backfill's dedupe pass removes them, and
     # the done row still blocks re-enqueue.
-    from backend.app.tasks.ml import enqueue_gpu_backfill
+    from backend.app.tasks.gpu_queue import enqueue_gpu_backfill
 
     img = await _img(db, "f2" * 32)
     for i in range(3):

tests/test_maintenance.py

@@ -310,7 +310,7 @@ def test_recover_stalled_task_runs_skips_fresh_running(db_sync):
 
 
 def test_recover_stalled_task_runs_ml_queue_uses_longer_threshold(db_sync):
-    """ml-queue tasks (tag_and_embed video branch) legitimately run
+    """ml-queue tasks (embed_image video branch) legitimately run
     past the default 5-min threshold. The sweep must NOT flag an
     ml-queue task that's only been running 10 min — the override
     threshold (25 min via QUEUE_STUCK_THRESHOLD_MINUTES) protects

tests/test_reextract_archives.py

@@ -46,7 +46,7 @@ def test_reextract_links_archive_members_to_post(db_sync, tmp_path, monkeypatch)
 
     # No broker in this path — the post-import enqueue is best-effort anyway.
     monkeypatch.setattr(thumb_mod.generate_thumbnail, "delay", lambda *a, **k: None)
-    monkeypatch.setattr(ml_mod.tag_and_embed, "delay", lambda *a, **k: None)
+    monkeypatch.setattr(ml_mod.embed_image, "delay", lambda *a, **k: None)
 
     images_root = tmp_path / "images"
     images_root.mkdir()
@@ -116,7 +116,7 @@ def test_reextract_timebox_resumes_from_cursor(db_sync, tmp_path, monkeypatch):
     from backend.app.tasks import thumbnail as thumb_mod
 
     monkeypatch.setattr(thumb_mod.generate_thumbnail, "delay", lambda *a, **k: None)
-    monkeypatch.setattr(ml_mod.tag_and_embed, "delay", lambda *a, **k: None)
+    monkeypatch.setattr(ml_mod.embed_image, "delay", lambda *a, **k: None)
 
     images_root = tmp_path / "images"
     images_root.mkdir()

tests/test_tasks_ml.py

@@ -1,4 +1,4 @@
-"""tag_and_embed (embed-only) / backfill task tests. The pure _is_video helper
+"""embed_image (embed-only) / backfill task tests. The pure _is_video helper
 is a unit test; the DB-touching backfill query is an integration test with
 monkeypatched dispatch."""
 
@@ -23,7 +23,7 @@ async def test_backfill_enqueues_missing(db, monkeypatch):
 
     calls = []
     monkeypatch.setattr(
-        ml_tasks.tag_and_embed, "delay", lambda image_id: calls.append(image_id)
+        ml_tasks.embed_image, "delay", lambda image_id: calls.append(image_id)
     )
 
     img = ImageRecord(
@@ -38,3 +38,35 @@ async def test_backfill_enqueues_missing(db, monkeypatch):
     count = ml_tasks.backfill()
     assert count >= 1
     assert img.id in calls
+
+
+@pytest.mark.integration
+@pytest.mark.asyncio
+async def test_backfill_respects_cpu_embed_toggle(db, monkeypatch):
+    """B3: with cpu_embed_enabled off (agent-equipped stack, no ml-worker),
+    the CPU backfill is a no-op — the GPU 'embed' backfill owns whole-image
+    embeds there. Same gate the import hooks consult before dispatching."""
+    from sqlalchemy import update
+
+    from backend.app.models import ImageRecord, MLSettings
+    from backend.app.tasks import ml as ml_tasks
+
+    calls = []
+    monkeypatch.setattr(
+        ml_tasks.embed_image, "delay", lambda image_id: calls.append(image_id)
+    )
+    db.add(ImageRecord(
+        path="/images/o.jpg", sha256="o" * 64, size_bytes=1,
+        mime="image/jpeg", width=1, height=1,
+        origin="imported_filesystem", integrity_status="unknown",
+        siglip_embedding=None,
+    ))
+    await db.execute(
+        update(MLSettings).where(MLSettings.id == 1)
+        .values(cpu_embed_enabled=False)
+    )
+    await db.commit()
+
+    assert ml_tasks.cpu_embed_enabled() is False
+    assert ml_tasks.backfill() == 0
+    assert calls == []