fix(tag-eval): don't re-suggest already-rejected items every run
"head would suggest" drew from the whole negative pool, which INCLUDES the images the operator rejected. A rejected near-miss (e.g. an orc under "goblin") is a hard negative that still scores high, so it kept resurfacing as a fresh suggestion every run (operator-flagged: "same items keep appearing"). Exclude already-rejected ids from the suggest list — once you've said no, it's gone. (head doubts = lowest-scoring positives is unchanged; genuinely-hard true positives legitimately recur there.) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -239,7 +239,7 @@ def _eval_concept(session: Session, name: str, cfg: dict, np) -> dict[str, Any]:
|
||||
head = _eval_head(Xn, y, cfg["cv_folds"], cfg["precision_target"], np)
|
||||
centroid = _eval_centroid(Xn, y, cfg["cv_folds"], np)
|
||||
curve = _learning_curve(Xn, y, cfg["curve_points"], neg_ratio, np)
|
||||
examples = _examples(session, Xn, y, ids, np)
|
||||
examples = _examples(session, Xn, y, ids, np, set(rejected))
|
||||
|
||||
return {
|
||||
"name": name, "tag_id": tag_id,
|
||||
@@ -358,13 +358,13 @@ def _learning_curve(Xn, y, points, neg_ratio, np) -> list[dict[str, float]]:
|
||||
return out
|
||||
|
||||
|
||||
def _examples(session, Xn, y, ids, np) -> dict[str, list[dict]]:
|
||||
"""Train on all data, then surface: top-scoring UNLABELED-ish (highest among
|
||||
the negative pool = what the head would newly suggest) and lowest-scoring
|
||||
POSITIVES (where the head disagrees with the operator's tag — likely the
|
||||
most informative to review). Resolves thumbnail urls so the stored report
|
||||
renders without per-id lookups (survives navigation as a self-contained
|
||||
artifact)."""
|
||||
def _examples(session, Xn, y, ids, np, rejected_set) -> dict[str, list[dict]]:
|
||||
"""Train on all data, then surface: top-scoring negatives the operator has
|
||||
NOT already rejected (= fresh suggestions) and lowest-scoring POSITIVES
|
||||
(where the head disagrees with the operator's tag). Excluding already-
|
||||
rejected ids stops an adjudicated near-miss — a hard negative that still
|
||||
scores high — from resurfacing in 'would suggest' on every run. Resolves
|
||||
thumbnail urls so the stored report renders without per-id lookups."""
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
|
||||
clf = LogisticRegression(max_iter=1000, class_weight="balanced")
|
||||
@@ -372,7 +372,14 @@ def _examples(session, Xn, y, ids, np) -> dict[str, list[dict]]:
|
||||
s = clf.predict_proba(Xn)[:, 1]
|
||||
neg_idx = np.where(y == 0)[0]
|
||||
pos_idx = np.where(y == 1)[0]
|
||||
top_neg = [int(ids[i]) for i in neg_idx[np.argsort(s[neg_idx])[::-1][:_EXAMPLES_K]]]
|
||||
top_neg = []
|
||||
for i in neg_idx[np.argsort(s[neg_idx])[::-1]]: # high score → low
|
||||
rid = int(ids[i])
|
||||
if rid in rejected_set:
|
||||
continue # already told the head 'no' — don't re-suggest it
|
||||
top_neg.append(rid)
|
||||
if len(top_neg) >= _EXAMPLES_K:
|
||||
break
|
||||
low_pos = [int(ids[i]) for i in pos_idx[np.argsort(s[pos_idx])[:_EXAMPLES_K]]]
|
||||
thumbs = _resolve_thumbs(session, top_neg + low_pos)
|
||||
return {
|
||||
|
||||
Reference in New Issue
Block a user