feat(heads): production per-concept heads — train + score backend (#114 A)

The eval (#1130) proved the frozen-embedding + trained-head spine; this lands
its production form (the first of three slices that make heads the suggestion
source, replacing Camie + centroid).

- tag_head: one logistic-regression head per general/character concept with
  enough labelled positives. Weights (pgvector), honest CV-derived suggest
  threshold + earned-auto-apply point, and per-concept quality metrics.
- head_training_run: persisted batch lifecycle (mirrors tag_eval_run) so the
  admin card shows live + historical status across navigation.
- services/ml/heads.py: TRAIN (sync, ml worker, reuses tag_eval's proven data
  loaders + metric math so production heads match measured eval numbers) and
  SCORE (async, API worker — numpy via pgvector, no scikit-learn): score one
  image's embedding against all heads → the rail's suggestions, cached on
  (count, max trained_at) so a retrain invalidates without per-request loads.
- tasks.ml.train_heads (ml queue, commits per head so a kill leaves progress)
  + recover_stalled_head_training_runs sweep + retention(20) + 5-min beat
  (rule 89).
- api/heads.py: POST /api/heads/train (one run at a time, 409 guard) + GET
  /api/heads (count, graduated, last-trained, running, per-concept table,
  recent runs).
- ml_settings: head_min_positives + head_auto_apply_precision, tunable via
  /api/ml/settings.

Scoring isn't wired into the rail yet (slice C) and the admin UI is slice B —
this slice makes training + scoring exist and CI-verifiable. 'precision' column
stored as precision_cv (SQL reserved word). Migration 0058.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa

backend/app/api/__init__.py

@@ -25,6 +25,7 @@ def all_blueprints() -> list[Blueprint]:
     from .downloads import downloads_bp
     from .extension import extension_bp
     from .gallery import gallery_bp
+    from .heads import heads_bp
     from .import_admin import import_admin_bp
     from .ml_admin import ml_admin_bp
     from .platforms import platforms_bp
@@ -58,6 +59,7 @@ def all_blueprints() -> list[Blueprint]:
         allowlist_bp,
         aliases_bp,
         tag_eval_bp,
+        heads_bp,
         ml_admin_bp,
         thumbnails_bp,
         sources_bp,

backend/app/api/heads.py

@@ -0,0 +1,118 @@
+"""Heads API (#114): train + inspect the per-concept heads that power
+suggestions (replacing Camie + centroid).
+
+POST /api/heads/train   — (re)train all eligible heads (one run at a time).
+GET  /api/heads         — status: head count, last-trained, running run, the
+                          per-concept head table (strength + auto-apply ready),
+                          and recent training runs. The card rehydrates from
+                          here so status survives navigation.
+"""
+
+from quart import Blueprint, jsonify, request
+from sqlalchemy import desc, func, select
+
+from ..extensions import get_session
+from ..models import HeadTrainingRun, Tag, TagHead
+from ..services.ml.heads import HeadTrainingAlreadyRunning, start_head_training_run
+
+heads_bp = Blueprint("heads", __name__, url_prefix="/api/heads")
+
+
+def _serialize_run(run: HeadTrainingRun) -> dict:
+    return {
+        "id": run.id,
+        "params": run.params,
+        "status": run.status,
+        "started_at": run.started_at.isoformat() if run.started_at else None,
+        "finished_at": run.finished_at.isoformat() if run.finished_at else None,
+        "n_trained": run.n_trained,
+        "n_skipped": run.n_skipped,
+        "error": run.error,
+    }
+
+
+@heads_bp.route("/train", methods=["POST"])
+async def train():
+    body = await request.get_json(silent=True) or {}
+    params = body.get("params") or body or {}
+    async with get_session() as session:
+        try:
+            run_id = await session.run_sync(
+                lambda s: start_head_training_run(s, params)
+            )
+        except HeadTrainingAlreadyRunning as running:
+            return jsonify({
+                "error": "training_already_running",
+                "running_id": int(running.args[0]),
+            }), 409
+        await session.commit()
+    return jsonify({"run_id": run_id, "status": "running"}), 202
+
+
+@heads_bp.route("", methods=["GET"])
+async def status():
+    async with get_session() as session:
+        count, last_trained = (
+            await session.execute(
+                select(func.count(), func.max(TagHead.trained_at))
+            )
+        ).one()
+        graduated = (
+            await session.execute(
+                select(func.count()).where(
+                    TagHead.auto_apply_threshold.is_not(None)
+                )
+            )
+        ).scalar_one()
+        running = (
+            await session.execute(
+                select(HeadTrainingRun.id)
+                .where(HeadTrainingRun.status == "running")
+                .order_by(HeadTrainingRun.id.desc())
+                .limit(1)
+            )
+        ).scalar_one_or_none()
+        runs = (
+            await session.execute(
+                select(HeadTrainingRun)
+                .order_by(HeadTrainingRun.id.desc())
+                .limit(10)
+            )
+        ).scalars().all()
+        # The per-concept table: strongest first, capped for the admin card.
+        head_rows = (
+            await session.execute(
+                select(
+                    TagHead.tag_id, Tag.name, Tag.kind,
+                    TagHead.n_pos, TagHead.n_neg, TagHead.ap,
+                    TagHead.precision_cv, TagHead.recall,
+                    TagHead.auto_apply_threshold, TagHead.trained_at,
+                )
+                .join(Tag, Tag.id == TagHead.tag_id)
+                .order_by(desc(TagHead.ap))
+                .limit(500)
+            )
+        ).all()
+        heads = [
+            {
+                "tag_id": r.tag_id,
+                "name": r.name,
+                "category": r.kind.value if hasattr(r.kind, "value") else str(r.kind),
+                "n_pos": r.n_pos,
+                "n_neg": r.n_neg,
+                "ap": r.ap,
+                "precision": r.precision_cv,
+                "recall": r.recall,
+                "auto_apply": r.auto_apply_threshold is not None,
+                "trained_at": r.trained_at.isoformat() if r.trained_at else None,
+            }
+            for r in head_rows
+        ]
+    return jsonify({
+        "head_count": count,
+        "graduated_count": graduated,
+        "last_trained_at": last_trained.isoformat() if last_trained else None,
+        "running_id": running,
+        "runs": [_serialize_run(r) for r in runs],
+        "heads": heads,
+    })

backend/app/api/ml_admin.py

@@ -17,6 +17,8 @@ _EDITABLE = (
     "video_frame_interval_seconds",
     "video_max_frames",
     "video_min_tag_frames",
+    "head_min_positives",
+    "head_auto_apply_precision",
 )
 
 
@@ -40,6 +42,8 @@ async def get_settings():
             "video_min_tag_frames": s.video_min_tag_frames,
             "tagger_model_version": s.tagger_model_version,
             "embedder_model_version": s.embedder_model_version,
+            "head_min_positives": s.head_min_positives,
+            "head_auto_apply_precision": s.head_auto_apply_precision,
         }
     )
 
@@ -100,6 +104,11 @@ def _validate(p: dict) -> str | None:
         return "video_min_tag_frames must be >= 1"
     if p["video_min_tag_frames"] > p["video_max_frames"]:
         return "video_min_tag_frames cannot exceed video_max_frames"
+    # Head training (#114).
+    if int(p["head_min_positives"]) < 1:
+        return "head_min_positives must be >= 1"
+    if not (0.5 <= float(p["head_auto_apply_precision"]) <= 0.999):
+        return "head_auto_apply_precision must be between 0.5 and 0.999"
     return None