diff --git a/backend/app/services/ml/tag_eval.py b/backend/app/services/ml/tag_eval.py index 2a1cbb5..24039c6 100644 --- a/backend/app/services/ml/tag_eval.py +++ b/backend/app/services/ml/tag_eval.py @@ -23,7 +23,7 @@ from typing import Any from sqlalchemy import func, select from sqlalchemy.orm import Session -from ...models import ImageRecord, Tag, TagEvalRun, TagSuggestionRejection +from ...models import ImageRecord, Tag, TagEvalRun, TagKind, TagSuggestionRejection from ...models.tag import image_tag log = logging.getLogger(__name__) @@ -68,8 +68,7 @@ class EvalAlreadyRunning(Exception): def _normalize_params(params: dict[str, Any] | None) -> dict[str, Any]: params = params or {} - concepts = params.get("concepts") or DEFAULT_CONCEPTS - concepts = [str(c).strip() for c in concepts if str(c).strip()] + concepts = [str(c).strip() for c in (params.get("concepts") or []) if str(c).strip()] try: neg_ratio = max(1, int(params.get("neg_ratio", DEFAULT_NEG_RATIO))) except (TypeError, ValueError): @@ -78,16 +77,45 @@ def _normalize_params(params: dict[str, Any] | None) -> dict[str, Any]: cv_folds = max(2, int(params.get("cv_folds", DEFAULT_CV_FOLDS))) except (TypeError, ValueError): cv_folds = DEFAULT_CV_FOLDS + try: + auto_top_n = min(max(int(params.get("auto_top_n", 0) or 0), 0), 200) + except (TypeError, ValueError): + auto_top_n = 0 + try: + precision_target = min(max(float(params.get("precision_target", 0.97)), 0.5), 0.999) + except (TypeError, ValueError): + precision_target = 0.97 + # No explicit concepts and auto-discovery off → fall back to the hand list. + if not concepts and not auto_top_n: + concepts = list(DEFAULT_CONCEPTS) curve = params.get("curve_points") or DEFAULT_CURVE_POINTS curve = sorted({int(n) for n in curve if int(n) > 0}) return { "concepts": concepts, "neg_ratio": neg_ratio, "cv_folds": cv_folds, + "auto_top_n": auto_top_n, + "precision_target": round(precision_target, 4), "curve_points": curve, } +def _top_general_concepts(session: Session, n: int, min_count: int) -> list[str]: + """The n most-tagged general (concept) tags with >= min_count images — a fast + server-side way to broaden the eval beyond the hand-picked list (counts all + sources; source-aware filtering is a separate concern).""" + rows = session.execute( + select(Tag.name) + .join(image_tag, image_tag.c.tag_id == Tag.id) + .where(Tag.kind == TagKind.general) + .group_by(Tag.id) + .having(func.count(image_tag.c.image_record_id) >= min_count) + .order_by(func.count(image_tag.c.image_record_id).desc()) + .limit(n) + ).all() + return [r[0] for r in rows] + + def _resolve_tag_id(session: Session, name: str) -> int | None: """Case-insensitive tag-name match; if several share a name, take the one applied to the most images (the one the operator actually uses).""" @@ -155,6 +183,16 @@ def run_eval(session: Session, params: dict[str, Any]) -> dict[str, Any]: import numpy as np cfg = _normalize_params(params) + # Auto-discovery: union the explicit concepts with the top-N most-tagged + # general tags (server-side, fast) so the eval can broaden itself. + concepts = list(cfg["concepts"]) + if cfg["auto_top_n"]: + seen = {c.lower() for c in concepts} + for name in _top_general_concepts(session, cfg["auto_top_n"], MIN_POSITIVES): + if name.lower() not in seen: + concepts.append(name) + seen.add(name.lower()) + cfg["concepts"] = concepts concepts_out = [] for name in cfg["concepts"]: try: @@ -198,7 +236,7 @@ def _eval_concept(session: Session, name: str, cfg: dict, np) -> dict[str, Any]: y = np.array([1] * len(pos) + [0] * len(neg)) Xn = _l2norm(X, np) - head = _eval_head(Xn, y, cfg["cv_folds"], np) + head = _eval_head(Xn, y, cfg["cv_folds"], cfg["precision_target"], np) centroid = _eval_centroid(Xn, y, cfg["cv_folds"], np) curve = _learning_curve(Xn, y, cfg["curve_points"], neg_ratio, np) examples = _examples(session, Xn, y, ids, np) @@ -241,7 +279,7 @@ def _safe_folds(y, folds, np) -> int: return max(2, min(folds, minority)) -def _eval_head(Xn, y, folds, np) -> dict[str, float]: +def _eval_head(Xn, y, folds, target, np) -> dict[str, float]: from sklearn.linear_model import LogisticRegression from sklearn.model_selection import StratifiedKFold, cross_val_predict @@ -249,7 +287,31 @@ def _eval_head(Xn, y, folds, np) -> dict[str, float]: cv = StratifiedKFold(n_splits=_safe_folds(y, folds, np), shuffle=True, random_state=0) probs = cross_val_predict(clf, Xn, y, cv=cv, method="predict_proba")[:, 1] - return _metrics_from_scores(y, probs, np) + m = _metrics_from_scores(y, probs, np) + m["auto_apply"] = _auto_apply_point(y, probs, target, np) + return m + + +def _auto_apply_point(y, scores, target, np) -> dict | None: + """The auto-apply operating point: the threshold that yields the MOST recall + while holding precision >= target. This answers 'could this concept fire + without a human, and how much would it catch?' Returns None if no threshold + reaches the precision target (concept not auto-apply-ready).""" + from sklearn.metrics import precision_recall_curve + + prec, rec, thr = precision_recall_curve(y, scores) + best = None # (threshold, precision, recall) maximizing recall s.t. prec>=target + for i in range(len(thr)): # thr[i] corresponds to prec[i], rec[i] + if prec[i] >= target and (best is None or rec[i] > best[2]): + best = (float(thr[i]), float(prec[i]), float(rec[i])) + if best is None: + return None + return { + "target": round(float(target), 4), + "threshold": round(best[0], 4), + "precision": round(best[1], 4), + "recall": round(best[2], 4), + } def _eval_centroid(Xn, y, folds, np) -> dict[str, float]: diff --git a/frontend/src/components/settings/TagEvalCard.vue b/frontend/src/components/settings/TagEvalCard.vue index e427012..8c4957d 100644 --- a/frontend/src/components/settings/TagEvalCard.vue +++ b/frontend/src/components/settings/TagEvalCard.vue @@ -19,6 +19,19 @@ :disabled="running" /> +