feat(heads): earned auto-apply — sweep mechanism, off by default (#114 auto-apply A)

Graduated heads can now apply their tag without a human — gated so it's safe:
- FIRING GATE: a head fires only when the master switch (head_auto_apply_enabled,
  default OFF) is on AND it has >= head_auto_apply_min_positives (default 30)
  clean labels. A precise-looking but under-supported low-N head can't spray tags.
- auto_apply_sweep (heads.py): streams every embedded image in chunks, scores
  against the eligible heads (numpy, no sklearn), applies each head's tag where
  score >= its auto_apply_threshold and the tag isn't already applied/rejected,
  with source='head_auto' (distinguishable + reversible). dry_run counts only.
- HeadAutoApplyRun (migration 0059) tracks each sweep / preview; apply_head_tags
  task (ml queue) + scheduled_apply_head_tags daily beat (no-op unless enabled)
  + recovery sweep + retention(20).
- API: POST /api/heads/auto-apply {dry_run} (202 / 409 running / 400 disabled),
  GET /api/heads/auto-apply (recent runs + per-concept report). Settings
  head_auto_apply_enabled + min_positives via /api/ml/settings.

Tests: sweep applies above threshold, dry-run writes nothing, skips under-
supported + ungraduated heads; API disabled/dry-run/conflict guards.

NEXT (slice 2): the observability the operator asked for — per-concept misfire
(auto-applied-then-removed) + under-fire tracking, time-series snapshots, and a
reporting API to tune. Slice 3: the UI (enable, preview, trends).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa

backend/app/services/ml/heads.py

@@ -26,12 +26,14 @@ from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import Session
 
 from ...models import (
+    HeadAutoApplyRun,
     HeadTrainingRun,
     ImageRecord,
     MLSettings,
     Tag,
     TagHead,
     TagKind,
+    TagSuggestionRejection,
 )
 from ...models.tag import image_tag
 from .tag_eval import (
@@ -328,3 +330,138 @@ async def _settings_async(session: AsyncSession) -> MLSettings:
     return (
         await session.execute(select(MLSettings).where(MLSettings.id == 1))
     ).scalar_one()
+
+
+# --- Earned auto-apply (sync, ml worker) ---------------------------------
+# A graduated head can apply its tag to images it scores above the head's
+# auto_apply_threshold, without a human. Gated by a master switch + a support
+# floor so a precise-looking but under-supported head can't spray tags.
+
+_AUTO_APPLY_CHUNK = 5000
+
+
+class HeadAutoApplyAlreadyRunning(Exception):
+    """Raised when an auto-apply sweep is already in flight."""
+
+
+class HeadAutoApplyDisabled(Exception):
+    """Raised when a real (non-dry-run) sweep is requested but the master
+    switch (head_auto_apply_enabled) is off."""
+
+
+def start_head_auto_apply_run(session: Session, params: dict[str, Any]) -> int:
+    """Create a HeadAutoApplyRun + dispatch the ml-queue sweep. dry_run previews
+    (writes nothing); a real sweep needs the master switch on. One run at a time."""
+    dry_run = bool((params or {}).get("dry_run", False))
+    existing = session.execute(
+        select(HeadAutoApplyRun.id).where(HeadAutoApplyRun.status == "running")
+    ).scalar_one_or_none()
+    if existing is not None:
+        raise HeadAutoApplyAlreadyRunning(existing)
+    if not dry_run and not _settings(session).head_auto_apply_enabled:
+        raise HeadAutoApplyDisabled()
+    run = HeadAutoApplyRun(
+        dry_run=dry_run, params={"dry_run": dry_run}, status="running",
+        last_progress_at=datetime.now(UTC),
+    )
+    session.add(run)
+    session.flush()
+    run_id = run.id
+    from ...tasks.ml import apply_head_tags as _task
+    _task.delay(run_id)
+    return run_id
+
+
+def _auto_apply_heads(session: Session, embedding_version: str, min_pos: int):
+    """Eligible heads to fire: graduated (auto_apply_threshold set), enough
+    support, current embedding. Returns the row list (tag_id/name/weights/...)."""
+    return session.execute(
+        select(
+            TagHead.tag_id, Tag.name, TagHead.weights, TagHead.bias,
+            TagHead.auto_apply_threshold,
+        )
+        .join(Tag, Tag.id == TagHead.tag_id)
+        .where(TagHead.embedding_version == embedding_version)
+        .where(TagHead.auto_apply_threshold.is_not(None))
+        .where(TagHead.n_pos >= min_pos)
+    ).all()
+
+
+def auto_apply_sweep(
+    session: Session, run: HeadAutoApplyRun, dry_run: bool
+) -> dict[str, Any]:
+    """Score every embedded image against the eligible heads and apply (or, for
+    dry_run, just count) each head's tag where score >= its auto_apply_threshold
+    and the tag isn't already applied or rejected on that image. Streams
+    embeddings in chunks; commits per chunk on a real run. Returns
+    {n_applied, concepts:[{tag_id,name,applied,scanned,threshold}]}."""
+    import numpy as np
+    from sqlalchemy.dialects.postgresql import insert as pg_insert
+
+    settings = _settings(session)
+    rows = _auto_apply_heads(
+        session, settings.embedder_model_version,
+        settings.head_auto_apply_min_positives,
+    )
+    if not rows:
+        return {"n_applied": 0, "concepts": []}
+
+    W = np.vstack([np.asarray(r.weights, dtype=np.float32) for r in rows])
+    b = np.asarray([r.bias for r in rows], dtype=np.float32)
+    thr = np.asarray([r.auto_apply_threshold for r in rows], dtype=np.float32)
+    tag_ids = [r.tag_id for r in rows]
+    names = [r.name for r in rows]
+
+    # Skip images that already carry, or have rejected, each tag.
+    skip = {tid: set() for tid in tag_ids}
+    for tid in tag_ids:
+        for (iid,) in session.execute(
+            select(image_tag.c.image_record_id).where(image_tag.c.tag_id == tid)
+        ):
+            skip[tid].add(iid)
+        for (iid,) in session.execute(
+            select(TagSuggestionRejection.image_record_id).where(
+                TagSuggestionRejection.tag_id == tid
+            )
+        ):
+            skip[tid].add(iid)
+
+    applied = [0] * len(rows)
+    scanned = 0
+    all_ids = list(session.execute(
+        select(ImageRecord.id).where(ImageRecord.siglip_embedding.is_not(None))
+    ).scalars())
+    for start in range(0, len(all_ids), _AUTO_APPLY_CHUNK):
+        chunk = all_ids[start:start + _AUTO_APPLY_CHUNK]
+        emb = _load_embeddings(session, chunk)
+        cids = [i for i in chunk if i in emb]
+        if not cids:
+            continue
+        Xn = _l2norm(np.vstack([emb[i] for i in cids]).astype(np.float32), np)
+        probs = 1.0 / (1.0 + np.exp(-(Xn @ W.T + b)))   # (N, H)
+        scanned += len(cids)
+        for h in range(len(rows)):
+            tid = tag_ids[h]
+            for idx in np.where(probs[:, h] >= thr[h])[0]:
+                iid = cids[int(idx)]
+                if iid in skip[tid]:
+                    continue
+                skip[tid].add(iid)
+                applied[h] += 1
+                if not dry_run:
+                    session.execute(
+                        pg_insert(image_tag)
+                        .values(image_record_id=iid, tag_id=tid, source="head_auto")
+                        .on_conflict_do_nothing()
+                    )
+        if not dry_run:
+            session.commit()
+        run.last_progress_at = datetime.now(UTC)
+        session.commit()
+
+    concepts = [
+        {"tag_id": tag_ids[h], "name": names[h], "applied": applied[h],
+         "scanned": scanned, "threshold": float(thr[h])}
+        for h in range(len(rows))
+    ]
+    return {"n_applied": sum(applied), "concepts": concepts}