fix(tag-eval): don't re-suggest already-rejected items every run
CI / lint (push) Successful in 3s
CI / frontend-build (push) Successful in 20s
CI / backend-lint-and-test (push) Successful in 26s
CI / integration (push) Successful in 3m21s

"head would suggest" drew from the whole negative pool, which INCLUDES the
images the operator rejected. A rejected near-miss (e.g. an orc under "goblin")
is a hard negative that still scores high, so it kept resurfacing as a fresh
suggestion every run (operator-flagged: "same items keep appearing"). Exclude
already-rejected ids from the suggest list — once you've said no, it's gone.
(head doubts = lowest-scoring positives is unchanged; genuinely-hard true
positives legitimately recur there.)

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-28 01:06:04 -04:00
parent 5143f4c34f
commit 4fd8790c85
+16 -9
View File
@@ -239,7 +239,7 @@ def _eval_concept(session: Session, name: str, cfg: dict, np) -> dict[str, Any]:
head = _eval_head(Xn, y, cfg["cv_folds"], cfg["precision_target"], np)
centroid = _eval_centroid(Xn, y, cfg["cv_folds"], np)
curve = _learning_curve(Xn, y, cfg["curve_points"], neg_ratio, np)
examples = _examples(session, Xn, y, ids, np)
examples = _examples(session, Xn, y, ids, np, set(rejected))
return {
"name": name, "tag_id": tag_id,
@@ -358,13 +358,13 @@ def _learning_curve(Xn, y, points, neg_ratio, np) -> list[dict[str, float]]:
return out
def _examples(session, Xn, y, ids, np) -> dict[str, list[dict]]:
"""Train on all data, then surface: top-scoring UNLABELED-ish (highest among
the negative pool = what the head would newly suggest) and lowest-scoring
POSITIVES (where the head disagrees with the operator's tag — likely the
most informative to review). Resolves thumbnail urls so the stored report
renders without per-id lookups (survives navigation as a self-contained
artifact)."""
def _examples(session, Xn, y, ids, np, rejected_set) -> dict[str, list[dict]]:
"""Train on all data, then surface: top-scoring negatives the operator has
NOT already rejected (= fresh suggestions) and lowest-scoring POSITIVES
(where the head disagrees with the operator's tag). Excluding already-
rejected ids stops an adjudicated near-miss — a hard negative that still
scores high — from resurfacing in 'would suggest' on every run. Resolves
thumbnail urls so the stored report renders without per-id lookups."""
from sklearn.linear_model import LogisticRegression
clf = LogisticRegression(max_iter=1000, class_weight="balanced")
@@ -372,7 +372,14 @@ def _examples(session, Xn, y, ids, np) -> dict[str, list[dict]]:
s = clf.predict_proba(Xn)[:, 1]
neg_idx = np.where(y == 0)[0]
pos_idx = np.where(y == 1)[0]
top_neg = [int(ids[i]) for i in neg_idx[np.argsort(s[neg_idx])[::-1][:_EXAMPLES_K]]]
top_neg = []
for i in neg_idx[np.argsort(s[neg_idx])[::-1]]: # high score → low
rid = int(ids[i])
if rid in rejected_set:
continue # already told the head 'no' — don't re-suggest it
top_neg.append(rid)
if len(top_neg) >= _EXAMPLES_K:
break
low_pos = [int(ids[i]) for i in pos_idx[np.argsort(s[pos_idx])[:_EXAMPLES_K]]]
thumbs = _resolve_thumbs(session, top_neg + low_pos)
return {