backend/app/services/ml/tag_eval.py

@@ -23,7 +23,7 @@ from typing import Any
 from sqlalchemy import func, select
 from sqlalchemy.orm import Session
 
-from ...models import ImageRecord, Tag, TagEvalRun, TagSuggestionRejection
+from ...models import ImageRecord, Tag, TagEvalRun, TagKind, TagSuggestionRejection
 from ...models.tag import image_tag
 
 log = logging.getLogger(__name__)
@@ -68,8 +68,7 @@ class EvalAlreadyRunning(Exception):
 
 def _normalize_params(params: dict[str, Any] | None) -> dict[str, Any]:
     params = params or {}
-    concepts = params.get("concepts") or DEFAULT_CONCEPTS
-    concepts = [str(c).strip() for c in concepts if str(c).strip()]
+    concepts = [str(c).strip() for c in (params.get("concepts") or []) if str(c).strip()]
     try:
         neg_ratio = max(1, int(params.get("neg_ratio", DEFAULT_NEG_RATIO)))
     except (TypeError, ValueError):
@@ -78,16 +77,45 @@ def _normalize_params(params: dict[str, Any] | None) -> dict[str, Any]:
         cv_folds = max(2, int(params.get("cv_folds", DEFAULT_CV_FOLDS)))
     except (TypeError, ValueError):
         cv_folds = DEFAULT_CV_FOLDS
+    try:
+        auto_top_n = min(max(int(params.get("auto_top_n", 0) or 0), 0), 200)
+    except (TypeError, ValueError):
+        auto_top_n = 0
+    try:
+        precision_target = min(max(float(params.get("precision_target", 0.97)), 0.5), 0.999)
+    except (TypeError, ValueError):
+        precision_target = 0.97
+    # No explicit concepts and auto-discovery off → fall back to the hand list.
+    if not concepts and not auto_top_n:
+        concepts = list(DEFAULT_CONCEPTS)
     curve = params.get("curve_points") or DEFAULT_CURVE_POINTS
     curve = sorted({int(n) for n in curve if int(n) > 0})
     return {
         "concepts": concepts,
         "neg_ratio": neg_ratio,
         "cv_folds": cv_folds,
+        "auto_top_n": auto_top_n,
+        "precision_target": round(precision_target, 4),
         "curve_points": curve,
     }
 
 
+def _top_general_concepts(session: Session, n: int, min_count: int) -> list[str]:
+    """The n most-tagged general (concept) tags with >= min_count images — a fast
+    server-side way to broaden the eval beyond the hand-picked list (counts all
+    sources; source-aware filtering is a separate concern)."""
+    rows = session.execute(
+        select(Tag.name)
+        .join(image_tag, image_tag.c.tag_id == Tag.id)
+        .where(Tag.kind == TagKind.general)
+        .group_by(Tag.id)
+        .having(func.count(image_tag.c.image_record_id) >= min_count)
+        .order_by(func.count(image_tag.c.image_record_id).desc())
+        .limit(n)
+    ).all()
+    return [r[0] for r in rows]
+
+
 def _resolve_tag_id(session: Session, name: str) -> int | None:
     """Case-insensitive tag-name match; if several share a name, take the one
     applied to the most images (the one the operator actually uses)."""
@@ -155,6 +183,16 @@ def run_eval(session: Session, params: dict[str, Any]) -> dict[str, Any]:
     import numpy as np
 
     cfg = _normalize_params(params)
+    # Auto-discovery: union the explicit concepts with the top-N most-tagged
+    # general tags (server-side, fast) so the eval can broaden itself.
+    concepts = list(cfg["concepts"])
+    if cfg["auto_top_n"]:
+        seen = {c.lower() for c in concepts}
+        for name in _top_general_concepts(session, cfg["auto_top_n"], MIN_POSITIVES):
+            if name.lower() not in seen:
+                concepts.append(name)
+                seen.add(name.lower())
+    cfg["concepts"] = concepts
     concepts_out = []
     for name in cfg["concepts"]:
         try:
@@ -198,7 +236,7 @@ def _eval_concept(session: Session, name: str, cfg: dict, np) -> dict[str, Any]:
     y = np.array([1] * len(pos) + [0] * len(neg))
     Xn = _l2norm(X, np)
 
-    head = _eval_head(Xn, y, cfg["cv_folds"], np)
+    head = _eval_head(Xn, y, cfg["cv_folds"], cfg["precision_target"], np)
     centroid = _eval_centroid(Xn, y, cfg["cv_folds"], np)
     curve = _learning_curve(Xn, y, cfg["curve_points"], neg_ratio, np)
     examples = _examples(session, Xn, y, ids, np)
@@ -241,7 +279,7 @@ def _safe_folds(y, folds, np) -> int:
     return max(2, min(folds, minority))
 
 
-def _eval_head(Xn, y, folds, np) -> dict[str, float]:
+def _eval_head(Xn, y, folds, target, np) -> dict[str, float]:
     from sklearn.linear_model import LogisticRegression
     from sklearn.model_selection import StratifiedKFold, cross_val_predict
 
@@ -249,7 +287,31 @@ def _eval_head(Xn, y, folds, np) -> dict[str, float]:
     cv = StratifiedKFold(n_splits=_safe_folds(y, folds, np), shuffle=True,
                          random_state=0)
     probs = cross_val_predict(clf, Xn, y, cv=cv, method="predict_proba")[:, 1]
-    return _metrics_from_scores(y, probs, np)
+    m = _metrics_from_scores(y, probs, np)
+    m["auto_apply"] = _auto_apply_point(y, probs, target, np)
+    return m
+
+
+def _auto_apply_point(y, scores, target, np) -> dict | None:
+    """The auto-apply operating point: the threshold that yields the MOST recall
+    while holding precision >= target. This answers 'could this concept fire
+    without a human, and how much would it catch?' Returns None if no threshold
+    reaches the precision target (concept not auto-apply-ready)."""
+    from sklearn.metrics import precision_recall_curve
+
+    prec, rec, thr = precision_recall_curve(y, scores)
+    best = None  # (threshold, precision, recall) maximizing recall s.t. prec>=target
+    for i in range(len(thr)):  # thr[i] corresponds to prec[i], rec[i]
+        if prec[i] >= target and (best is None or rec[i] > best[2]):
+            best = (float(thr[i]), float(prec[i]), float(rec[i]))
+    if best is None:
+        return None
+    return {
+        "target": round(float(target), 4),
+        "threshold": round(best[0], 4),
+        "precision": round(best[1], 4),
+        "recall": round(best[2], 4),
+    }
 
 
 def _eval_centroid(Xn, y, folds, np) -> dict[str, float]:

frontend/src/components/settings/TagEvalCard.vue

@@ -19,6 +19,19 @@
       :disabled="running"
     />
 
+    <div class="d-flex mb-3" style="gap: 12px;">
+      <v-text-field
+        v-model.number="autoTopN" label="+ auto-add top-N concepts"
+        type="number" min="0" max="200" density="compact" hide-details
+        :disabled="running" style="max-width: 220px;"
+      />
+      <v-text-field
+        v-model.number="precisionTarget" label="Auto-apply precision target"
+        type="number" min="0.5" max="0.999" step="0.01" density="compact" hide-details
+        :disabled="running" style="max-width: 220px;"
+      />
+    </div>
+
     <v-btn
       v-if="!running"
       color="accent" variant="flat" rounded="pill"
@@ -78,6 +91,16 @@
             (head − centroid)
           </div>
 
+          <div class="text-caption mb-2">
+            <span class="fc-muted">Auto-apply:</span>
+            <template v-if="c.head.auto_apply">
+              <span class="fc-up">ready</span> — at P≥{{ c.head.auto_apply.target }}
+              catches recall <strong>{{ c.head.auto_apply.recall }}</strong>
+              (thr {{ c.head.auto_apply.threshold }})
+            </template>
+            <span v-else class="fc-down">not reachable at P≥{{ report.params.precision_target }}</span>
+          </div>
+
           <div v-if="c.curve && c.curve.length" class="fc-curve">
             <span class="fc-muted text-caption">Learning curve (AP @ N positives):</span>
             <span v-for="p in c.curve" :key="p.n_pos" class="fc-curve__pt">
@@ -145,6 +168,8 @@ const store = useTagEvalStore()
 const modal = useModalStore()
 const run = ref(null)
 const conceptsText = ref(DEFAULT_CONCEPTS)
+const autoTopN = ref(0)
+const precisionTarget = ref(0.97)
 const busy = ref(false)
 let pollTimer = null
 
@@ -186,7 +211,11 @@ async function onStart() {
   busy.value = true
   try {
     const concepts = conceptsText.value.split(',').map(s => s.trim()).filter(Boolean)
-    const res = await store.start({ concepts })
+    const res = await store.start({
+      concepts,
+      auto_top_n: Number(autoTopN.value) || 0,
+      precision_target: Number(precisionTarget.value) || 0.97,
+    })
     run.value = await store.getRun(res.run_id)
     startPoll(res.run_id)
   } catch (e) {