From 4fd8790c85132c604350a218e0517f26110b9f7b Mon Sep 17 00:00:00 2001
From: Bryan Van Deusen <bryan.vandeusen@fabledsword.com>
Date: Sun, 28 Jun 2026 01:06:04 -0400
Subject: [PATCH] fix(tag-eval): don't re-suggest already-rejected items every
 run
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

"head would suggest" drew from the whole negative pool, which INCLUDES the
images the operator rejected. A rejected near-miss (e.g. an orc under "goblin")
is a hard negative that still scores high, so it kept resurfacing as a fresh
suggestion every run (operator-flagged: "same items keep appearing"). Exclude
already-rejected ids from the suggest list — once you've said no, it's gone.
(head doubts = lowest-scoring positives is unchanged; genuinely-hard true
positives legitimately recur there.)

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 backend/app/services/ml/tag_eval.py | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/backend/app/services/ml/tag_eval.py b/backend/app/services/ml/tag_eval.py
index 24039c6..bfb7ff5 100644
--- a/backend/app/services/ml/tag_eval.py
+++ b/backend/app/services/ml/tag_eval.py
@@ -239,7 +239,7 @@ def _eval_concept(session: Session, name: str, cfg: dict, np) -> dict[str, Any]:
     head = _eval_head(Xn, y, cfg["cv_folds"], cfg["precision_target"], np)
     centroid = _eval_centroid(Xn, y, cfg["cv_folds"], np)
     curve = _learning_curve(Xn, y, cfg["curve_points"], neg_ratio, np)
-    examples = _examples(session, Xn, y, ids, np)
+    examples = _examples(session, Xn, y, ids, np, set(rejected))
 
     return {
         "name": name, "tag_id": tag_id,
@@ -358,13 +358,13 @@ def _learning_curve(Xn, y, points, neg_ratio, np) -> list[dict[str, float]]:
     return out
 
 
-def _examples(session, Xn, y, ids, np) -> dict[str, list[dict]]:
-    """Train on all data, then surface: top-scoring UNLABELED-ish (highest among
-    the negative pool = what the head would newly suggest) and lowest-scoring
-    POSITIVES (where the head disagrees with the operator's tag — likely the
-    most informative to review). Resolves thumbnail urls so the stored report
-    renders without per-id lookups (survives navigation as a self-contained
-    artifact)."""
+def _examples(session, Xn, y, ids, np, rejected_set) -> dict[str, list[dict]]:
+    """Train on all data, then surface: top-scoring negatives the operator has
+    NOT already rejected (= fresh suggestions) and lowest-scoring POSITIVES
+    (where the head disagrees with the operator's tag). Excluding already-
+    rejected ids stops an adjudicated near-miss — a hard negative that still
+    scores high — from resurfacing in 'would suggest' on every run. Resolves
+    thumbnail urls so the stored report renders without per-id lookups."""
     from sklearn.linear_model import LogisticRegression
 
     clf = LogisticRegression(max_iter=1000, class_weight="balanced")
@@ -372,7 +372,14 @@ def _examples(session, Xn, y, ids, np) -> dict[str, list[dict]]:
     s = clf.predict_proba(Xn)[:, 1]
     neg_idx = np.where(y == 0)[0]
     pos_idx = np.where(y == 1)[0]
-    top_neg = [int(ids[i]) for i in neg_idx[np.argsort(s[neg_idx])[::-1][:_EXAMPLES_K]]]
+    top_neg = []
+    for i in neg_idx[np.argsort(s[neg_idx])[::-1]]:  # high score → low
+        rid = int(ids[i])
+        if rid in rejected_set:
+            continue  # already told the head 'no' — don't re-suggest it
+        top_neg.append(rid)
+        if len(top_neg) >= _EXAMPLES_K:
+            break
     low_pos = [int(ids[i]) for i in pos_idx[np.argsort(s[pos_idx])[:_EXAMPLES_K]]]
     thumbs = _resolve_thumbs(session, top_neg + low_pos)
     return {