feat(ml): tag-eval backend — head-vs-centroid learning-curve eval (persisted)

Slice 1 of milestone #114 (tagging v2). Proves the frozen-embedding + trained-
head spine on the operator's own data, reusing the SigLIP embeddings already
stored on image_record — no re-embedding, no GPU.

Per concept: train a logistic-regression HEAD (positives + negatives = explicit
rejections + sampled unlabeled) vs the old single-CENTROID baseline; report
cross-validated precision/recall/AP for both, a LEARNING CURVE (AP/F1 as tagged
positives grow 10→30→100→300), and example image ids (head-would-suggest /
head-doubts-positive) to eyeball.

Persisted so the report SURVIVES navigation (operator-flagged): the run + full
report live in a new tag_eval_run row (mirrors library_audit_run); the admin
card will rehydrate from GET on mount, not transient state.

- models.TagEvalRun + migration 0056; runs on the ml queue (only worker with
  numpy/sklearn) — numpy/sklearn lazy-imported so the API can still enqueue.
- services/ml/tag_eval (compute + start helper, one-running guard), tasks.ml
  .tag_eval_run, api/tag-eval (POST create, GET history light / detail w/ report).
- recover_stalled_tag_eval_runs sweep + retention (keep last 20) + 5-min beat
  (rule 89). scikit-learn added to requirements-ml.
- tests: param normalization + the rehydrate read-path + create/conflict.

Frontend admin card (trigger + render persisted report) follows next.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

backend/app/tasks/maintenance.py

@@ -19,6 +19,7 @@ from ..models import (
     ImportTask,
     LibraryAuditRun,
     Source,
+    TagEvalRun,
     TaskRun,
 )
 from ..utils.phash import compute_phash
@@ -93,6 +94,9 @@ BACKUP_DB_STALL_THRESHOLD_MINUTES = 40
 # Library audit: scan_library_for_rule has time_limit=7500s (2h5m).
 # 2h15m gives a 10-min buffer.
 LIBRARY_AUDIT_STALL_THRESHOLD_MINUTES = 135
+# tag-eval (#1130) has a 30-min soft limit; flag a run with no progress past 40.
+TAG_EVAL_STALL_THRESHOLD_MINUTES = 40
+TAG_EVAL_KEEP_RUNS = 20
 # Import batches finalize only after every child ImportTask hits a
 # terminal state. The recovery sweep targets the case where every
 # task is done but the batch never got its closing UPDATE
@@ -709,6 +713,46 @@ def recover_stalled_library_audit_runs() -> int:
         return recovered
 
 
+@celery.task(name="backend.app.tasks.maintenance.recover_stalled_tag_eval_runs")
+def recover_stalled_tag_eval_runs() -> int:
+    """Flip TagEvalRun rows stuck in 'running' past the stall threshold to
+    'error', and prune old runs to the last TAG_EVAL_KEEP_RUNS (retention,
+    rule 89). Runs every 5 min on the maintenance lane; no-op when idle."""
+    SessionLocal = _sync_session_factory()
+    now = datetime.now(UTC)
+    cutoff = now - timedelta(minutes=TAG_EVAL_STALL_THRESHOLD_MINUTES)
+    with SessionLocal() as session:
+        result = session.execute(
+            update(TagEvalRun)
+            .where(TagEvalRun.status == "running")
+            .where(
+                func.coalesce(TagEvalRun.last_progress_at, TagEvalRun.started_at)
+                < cutoff
+            )
+            .values(
+                status="error", finished_at=now,
+                error=(
+                    f"stranded by recovery sweep (no progress for "
+                    f"{TAG_EVAL_STALL_THRESHOLD_MINUTES} min)"
+                ),
+            )
+        )
+        # Retention: keep only the most recent N runs.
+        keep = session.execute(
+            select(TagEvalRun.id).order_by(TagEvalRun.id.desc())
+            .limit(TAG_EVAL_KEEP_RUNS)
+        ).scalars().all()
+        if keep:
+            session.execute(
+                delete(TagEvalRun).where(TagEvalRun.id.not_in(keep))
+            )
+        session.commit()
+        recovered = result.rowcount or 0
+        if recovered:
+            log.info("recover_stalled_tag_eval_runs: recovered %d rows", recovered)
+        return recovered
+
+
 @celery.task(name="backend.app.tasks.maintenance.recover_stalled_import_batches")
 def recover_stalled_import_batches() -> int:
     """Finalize ImportBatch rows stuck in running past the hard limit

backend/app/tasks/ml.py

@@ -538,3 +538,48 @@ def recompute_centroids(self) -> int:
     for tid in drifted:
         recompute_centroid.delay(tid)
     return len(drifted)
+
+
+@celery.task(
+    name="backend.app.tasks.ml.tag_eval_run",
+    bind=True,
+    # The head-vs-centroid eval (#1130) loads embeddings + fits sklearn heads
+    # for several concepts — minutes, not seconds. Runs on the ml queue because
+    # only that worker has numpy/scikit-learn.
+    soft_time_limit=1800, time_limit=2100,
+)
+def tag_eval_run(self, run_id: int) -> str:
+    """Compute the eval report into the persisted TagEvalRun row so it survives
+    navigation (the admin card rehydrates from the row, not transient state)."""
+    from datetime import UTC, datetime
+
+    from ..models import TagEvalRun
+    from ..services.ml.tag_eval import run_eval
+
+    SessionLocal = _sync_session_factory()
+    with SessionLocal() as session:
+        run = session.get(TagEvalRun, run_id)
+        if run is None:
+            return "missing"
+        run.last_progress_at = datetime.now(UTC)
+        session.commit()
+        try:
+            report = run_eval(session, run.params)
+        except SoftTimeLimitExceeded:
+            run.status = "error"
+            run.error = "timed out"
+            run.finished_at = datetime.now(UTC)
+            session.commit()
+            raise
+        except Exception as exc:
+            log.exception("tag_eval_run %d failed", run_id)
+            run.status = "error"
+            run.error = str(exc)
+            run.finished_at = datetime.now(UTC)
+            session.commit()
+            return "error"
+        run.report = report
+        run.status = "ready"
+        run.finished_at = datetime.now(UTC)
+        session.commit()
+        return "ready"