feat(heads): production per-concept heads — train + score backend (#114 A)

The eval (#1130) proved the frozen-embedding + trained-head spine; this lands
its production form (the first of three slices that make heads the suggestion
source, replacing Camie + centroid).

- tag_head: one logistic-regression head per general/character concept with
  enough labelled positives. Weights (pgvector), honest CV-derived suggest
  threshold + earned-auto-apply point, and per-concept quality metrics.
- head_training_run: persisted batch lifecycle (mirrors tag_eval_run) so the
  admin card shows live + historical status across navigation.
- services/ml/heads.py: TRAIN (sync, ml worker, reuses tag_eval's proven data
  loaders + metric math so production heads match measured eval numbers) and
  SCORE (async, API worker — numpy via pgvector, no scikit-learn): score one
  image's embedding against all heads → the rail's suggestions, cached on
  (count, max trained_at) so a retrain invalidates without per-request loads.
- tasks.ml.train_heads (ml queue, commits per head so a kill leaves progress)
  + recover_stalled_head_training_runs sweep + retention(20) + 5-min beat
  (rule 89).
- api/heads.py: POST /api/heads/train (one run at a time, 409 guard) + GET
  /api/heads (count, graduated, last-trained, running, per-concept table,
  recent runs).
- ml_settings: head_min_positives + head_auto_apply_precision, tunable via
  /api/ml/settings.

Scoring isn't wired into the rail yet (slice C) and the admin UI is slice B —
this slice makes training + scoring exist and CI-verifiable. 'precision' column
stored as precision_cv (SQL reserved word). Migration 0058.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa

backend/app/tasks/maintenance.py

@@ -17,6 +17,7 @@ from ..models import (
     ImportBatch,
     ImportSettings,
     ImportTask,
+    HeadTrainingRun,
     LibraryAuditRun,
     Source,
     TagEvalRun,
@@ -97,6 +98,9 @@ LIBRARY_AUDIT_STALL_THRESHOLD_MINUTES = 135
 # tag-eval (#1130) has a 30-min soft limit; flag a run with no progress past 40.
 TAG_EVAL_STALL_THRESHOLD_MINUTES = 40
 TAG_EVAL_KEEP_RUNS = 20
+# head training (#114) has a 60-min soft limit; flag no-progress past 75.
+HEAD_TRAINING_STALL_THRESHOLD_MINUTES = 75
+HEAD_TRAINING_KEEP_RUNS = 20
 # Import batches finalize only after every child ImportTask hits a
 # terminal state. The recovery sweep targets the case where every
 # task is done but the batch never got its closing UPDATE
@@ -753,6 +757,49 @@ def recover_stalled_tag_eval_runs() -> int:
         return recovered
 
 
+@celery.task(name="backend.app.tasks.maintenance.recover_stalled_head_training_runs")
+def recover_stalled_head_training_runs() -> int:
+    """Flip HeadTrainingRun rows stuck in 'running' past the stall threshold to
+    'error', and prune old runs to the last HEAD_TRAINING_KEEP_RUNS (retention,
+    rule 89). Runs every 5 min on the maintenance lane; no-op when idle."""
+    SessionLocal = _sync_session_factory()
+    now = datetime.now(UTC)
+    cutoff = now - timedelta(minutes=HEAD_TRAINING_STALL_THRESHOLD_MINUTES)
+    with SessionLocal() as session:
+        result = session.execute(
+            update(HeadTrainingRun)
+            .where(HeadTrainingRun.status == "running")
+            .where(
+                func.coalesce(
+                    HeadTrainingRun.last_progress_at, HeadTrainingRun.started_at
+                )
+                < cutoff
+            )
+            .values(
+                status="error", finished_at=now,
+                error=(
+                    f"stranded by recovery sweep (no progress for "
+                    f"{HEAD_TRAINING_STALL_THRESHOLD_MINUTES} min)"
+                ),
+            )
+        )
+        keep = session.execute(
+            select(HeadTrainingRun.id).order_by(HeadTrainingRun.id.desc())
+            .limit(HEAD_TRAINING_KEEP_RUNS)
+        ).scalars().all()
+        if keep:
+            session.execute(
+                delete(HeadTrainingRun).where(HeadTrainingRun.id.not_in(keep))
+            )
+        session.commit()
+        recovered = result.rowcount or 0
+        if recovered:
+            log.info(
+                "recover_stalled_head_training_runs: recovered %d rows", recovered
+            )
+        return recovered
+
+
 @celery.task(name="backend.app.tasks.maintenance.recover_stalled_import_batches")
 def recover_stalled_import_batches() -> int:
     """Finalize ImportBatch rows stuck in running past the hard limit

backend/app/tasks/ml.py

@@ -583,3 +583,49 @@ def tag_eval_run(self, run_id: int) -> str:
         run.finished_at = datetime.now(UTC)
         session.commit()
         return "ready"
+
+
+@celery.task(
+    name="backend.app.tasks.ml.train_heads",
+    bind=True,
+    # Trains a logistic-regression head per eligible concept over stored SigLIP
+    # embeddings — minutes for a full library. Runs on the ml queue (only that
+    # worker has scikit-learn). Commits per head so a kill leaves progress.
+    soft_time_limit=3600, time_limit=3900,
+)
+def train_heads(self, run_id: int) -> str:
+    """(Re)train all eligible concept heads into tag_head, tracked by the
+    HeadTrainingRun row so the admin card shows live + historical status."""
+    from datetime import UTC, datetime
+
+    from ..models import HeadTrainingRun
+    from ..services.ml.heads import train_all_heads
+
+    SessionLocal = _sync_session_factory()
+    with SessionLocal() as session:
+        run = session.get(HeadTrainingRun, run_id)
+        if run is None:
+            return "missing"
+        run.last_progress_at = datetime.now(UTC)
+        session.commit()
+        try:
+            result = train_all_heads(session, run.params, run)
+        except SoftTimeLimitExceeded:
+            run.status = "error"
+            run.error = "timed out"
+            run.finished_at = datetime.now(UTC)
+            session.commit()
+            raise
+        except Exception as exc:
+            log.exception("train_heads %d failed", run_id)
+            run.status = "error"
+            run.error = str(exc)
+            run.finished_at = datetime.now(UTC)
+            session.commit()
+            return "error"
+        run.n_trained = result["n_trained"]
+        run.n_skipped = result["n_skipped"]
+        run.status = "ready"
+        run.finished_at = datetime.now(UTC)
+        session.commit()
+        return "ready"