feat(heads): earned auto-apply — sweep mechanism, off by default (#114 auto-apply A)

Graduated heads can now apply their tag without a human — gated so it's safe:
- FIRING GATE: a head fires only when the master switch (head_auto_apply_enabled,
  default OFF) is on AND it has >= head_auto_apply_min_positives (default 30)
  clean labels. A precise-looking but under-supported low-N head can't spray tags.
- auto_apply_sweep (heads.py): streams every embedded image in chunks, scores
  against the eligible heads (numpy, no sklearn), applies each head's tag where
  score >= its auto_apply_threshold and the tag isn't already applied/rejected,
  with source='head_auto' (distinguishable + reversible). dry_run counts only.
- HeadAutoApplyRun (migration 0059) tracks each sweep / preview; apply_head_tags
  task (ml queue) + scheduled_apply_head_tags daily beat (no-op unless enabled)
  + recovery sweep + retention(20).
- API: POST /api/heads/auto-apply {dry_run} (202 / 409 running / 400 disabled),
  GET /api/heads/auto-apply (recent runs + per-concept report). Settings
  head_auto_apply_enabled + min_positives via /api/ml/settings.

Tests: sweep applies above threshold, dry-run writes nothing, skips under-
supported + ungraduated heads; API disabled/dry-run/conflict guards.

NEXT (slice 2): the observability the operator asked for — per-concept misfire
(auto-applied-then-removed) + under-fire tracking, time-series snapshots, and a
reporting API to tune. Slice 3: the UI (enable, preview, trends).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa

backend/app/tasks/maintenance.py

@@ -13,6 +13,7 @@ from ..celery_app import celery
 from ..models import (
     BackupRun,
     DownloadEvent,
+    HeadAutoApplyRun,
     HeadTrainingRun,
     ImageRecord,
     ImportBatch,
@@ -101,6 +102,9 @@ TAG_EVAL_KEEP_RUNS = 20
 # head training (#114) has a 60-min soft limit; flag no-progress past 75.
 HEAD_TRAINING_STALL_THRESHOLD_MINUTES = 75
 HEAD_TRAINING_KEEP_RUNS = 20
+# head auto-apply (#114) shares the 60-min soft limit; flag past 75.
+HEAD_AUTO_APPLY_STALL_THRESHOLD_MINUTES = 75
+HEAD_AUTO_APPLY_KEEP_RUNS = 20
 # Import batches finalize only after every child ImportTask hits a
 # terminal state. The recovery sweep targets the case where every
 # task is done but the batch never got its closing UPDATE
@@ -800,6 +804,48 @@ def recover_stalled_head_training_runs() -> int:
         return recovered
 
 
+@celery.task(name="backend.app.tasks.maintenance.recover_stalled_head_auto_apply_runs")
+def recover_stalled_head_auto_apply_runs() -> int:
+    """Flip stalled HeadAutoApplyRun 'running' rows to 'error' + prune to the
+    last HEAD_AUTO_APPLY_KEEP_RUNS (retention, rule 89). 5-min maintenance lane."""
+    SessionLocal = _sync_session_factory()
+    now = datetime.now(UTC)
+    cutoff = now - timedelta(minutes=HEAD_AUTO_APPLY_STALL_THRESHOLD_MINUTES)
+    with SessionLocal() as session:
+        result = session.execute(
+            update(HeadAutoApplyRun)
+            .where(HeadAutoApplyRun.status == "running")
+            .where(
+                func.coalesce(
+                    HeadAutoApplyRun.last_progress_at, HeadAutoApplyRun.started_at
+                )
+                < cutoff
+            )
+            .values(
+                status="error", finished_at=now,
+                error=(
+                    f"stranded by recovery sweep (no progress for "
+                    f"{HEAD_AUTO_APPLY_STALL_THRESHOLD_MINUTES} min)"
+                ),
+            )
+        )
+        keep = session.execute(
+            select(HeadAutoApplyRun.id).order_by(HeadAutoApplyRun.id.desc())
+            .limit(HEAD_AUTO_APPLY_KEEP_RUNS)
+        ).scalars().all()
+        if keep:
+            session.execute(
+                delete(HeadAutoApplyRun).where(HeadAutoApplyRun.id.not_in(keep))
+            )
+        session.commit()
+        recovered = result.rowcount or 0
+        if recovered:
+            log.info(
+                "recover_stalled_head_auto_apply_runs: recovered %d rows", recovered
+            )
+        return recovered
+
+
 @celery.task(name="backend.app.tasks.maintenance.recover_stalled_import_batches")
 def recover_stalled_import_batches() -> int:
     """Finalize ImportBatch rows stuck in running past the hard limit

backend/app/tasks/ml.py

@@ -659,3 +659,82 @@ def scheduled_train_heads() -> str:
         run_id = run.id
     train_heads.delay(run_id)
     return "dispatched"
+
+
+@celery.task(
+    name="backend.app.tasks.ml.apply_head_tags",
+    bind=True,
+    # Scores the whole library against the graduated heads and applies their
+    # tags (or, dry_run, just counts). Streams embeddings in chunks; numpy only,
+    # but ml queue keeps it off the API workers. Commits per chunk.
+    soft_time_limit=3600, time_limit=3900,
+)
+def apply_head_tags(self, run_id: int) -> str:
+    """Run an earned-auto-apply sweep into the persisted HeadAutoApplyRun row."""
+    from datetime import UTC, datetime
+
+    from ..models import HeadAutoApplyRun
+    from ..services.ml.heads import auto_apply_sweep
+
+    SessionLocal = _sync_session_factory()
+    with SessionLocal() as session:
+        run = session.get(HeadAutoApplyRun, run_id)
+        if run is None:
+            return "missing"
+        run.last_progress_at = datetime.now(UTC)
+        session.commit()
+        try:
+            result = auto_apply_sweep(session, run, run.dry_run)
+        except SoftTimeLimitExceeded:
+            run.status = "error"
+            run.error = "timed out"
+            run.finished_at = datetime.now(UTC)
+            session.commit()
+            raise
+        except Exception as exc:
+            log.exception("apply_head_tags %d failed", run_id)
+            run.status = "error"
+            run.error = str(exc)
+            run.finished_at = datetime.now(UTC)
+            session.commit()
+            return "error"
+        run.n_applied = result["n_applied"]
+        run.report = {"concepts": result["concepts"]}
+        run.status = "ready"
+        run.finished_at = datetime.now(UTC)
+        session.commit()
+        return "ready"
+
+
+@celery.task(name="backend.app.tasks.ml.scheduled_apply_head_tags")
+def scheduled_apply_head_tags() -> str:
+    """Daily passive auto-apply sweep (#114) — only when the master switch is on.
+    Skips if a sweep is already in flight. Creates + COMMITS the run before
+    dispatching so the worker always finds it."""
+    from datetime import UTC, datetime
+
+    from sqlalchemy import select as sa_select
+
+    from ..models import HeadAutoApplyRun, MLSettings
+
+    SessionLocal = _sync_session_factory()
+    with SessionLocal() as session:
+        enabled = session.execute(
+            sa_select(MLSettings.head_auto_apply_enabled).where(MLSettings.id == 1)
+        ).scalar_one_or_none()
+        if not enabled:
+            return "disabled"
+        running = session.execute(
+            sa_select(HeadAutoApplyRun.id).where(HeadAutoApplyRun.status == "running")
+        ).scalar_one_or_none()
+        if running is not None:
+            return "already running"
+        run = HeadAutoApplyRun(
+            dry_run=False, params={"dry_run": False, "source": "scheduled"},
+            status="running", last_progress_at=datetime.now(UTC),
+        )
+        session.add(run)
+        session.commit()
+        run_id = run.id
+    apply_head_tags.delay(run_id)
+    return "dispatched"