feat(settings): tag-eval admin card — trigger + persisted report (survives nav)
Frontend for #1130. A maintenance tile in Settings → Tagging: - Editable concept list + "Run eval" → POST /api/tag-eval (one running at a time). - Rehydrates on mount via the persisted run (getRun by latest id) and polls while running — so the report SURVIVES navigation (operator-flagged); the task runs backend-side regardless and the card reconnects to its row. - Renders the saved report: per-concept head-vs-centroid metrics table (AP/F1/ precision/recall) with Δ AP, the learning curve (AP @ N positives), and thumbnail galleries (head-would-suggest / head-doubts-positive) for eyeballing. Backend: _examples now stores thumbnail_urls (not just ids) so the report is a self-contained artifact that renders without per-id lookups on reload. No new top-level surface — slots into the existing maintenance area. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -201,7 +201,7 @@ def _eval_concept(session: Session, name: str, cfg: dict, np) -> dict[str, Any]:
|
||||
head = _eval_head(Xn, y, cfg["cv_folds"], np)
|
||||
centroid = _eval_centroid(Xn, y, cfg["cv_folds"], np)
|
||||
curve = _learning_curve(Xn, y, cfg["curve_points"], neg_ratio, np)
|
||||
examples = _examples(Xn, y, ids, np)
|
||||
examples = _examples(session, Xn, y, ids, np)
|
||||
|
||||
return {
|
||||
"name": name, "tag_id": tag_id,
|
||||
@@ -296,11 +296,13 @@ def _learning_curve(Xn, y, points, neg_ratio, np) -> list[dict[str, float]]:
|
||||
return out
|
||||
|
||||
|
||||
def _examples(Xn, y, ids, np) -> dict[str, list[int]]:
|
||||
def _examples(session, Xn, y, ids, np) -> dict[str, list[dict]]:
|
||||
"""Train on all data, then surface: top-scoring UNLABELED-ish (highest among
|
||||
the negative pool = what the head would newly suggest) and lowest-scoring
|
||||
POSITIVES (where the head disagrees with the operator's tag — likely the
|
||||
most informative to review)."""
|
||||
most informative to review). Resolves thumbnail urls so the stored report
|
||||
renders without per-id lookups (survives navigation as a self-contained
|
||||
artifact)."""
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
|
||||
clf = LogisticRegression(max_iter=1000, class_weight="balanced")
|
||||
@@ -308,9 +310,26 @@ def _examples(Xn, y, ids, np) -> dict[str, list[int]]:
|
||||
s = clf.predict_proba(Xn)[:, 1]
|
||||
neg_idx = np.where(y == 0)[0]
|
||||
pos_idx = np.where(y == 1)[0]
|
||||
top_neg = neg_idx[np.argsort(s[neg_idx])[::-1][:_EXAMPLES_K]]
|
||||
low_pos = pos_idx[np.argsort(s[pos_idx])[:_EXAMPLES_K]]
|
||||
top_neg = [int(ids[i]) for i in neg_idx[np.argsort(s[neg_idx])[::-1][:_EXAMPLES_K]]]
|
||||
low_pos = [int(ids[i]) for i in pos_idx[np.argsort(s[pos_idx])[:_EXAMPLES_K]]]
|
||||
thumbs = _resolve_thumbs(session, top_neg + low_pos)
|
||||
return {
|
||||
"head_would_suggest": [int(ids[i]) for i in top_neg],
|
||||
"head_doubts_positive": [int(ids[i]) for i in low_pos],
|
||||
"head_would_suggest": [thumbs[i] for i in top_neg if i in thumbs],
|
||||
"head_doubts_positive": [thumbs[i] for i in low_pos if i in thumbs],
|
||||
}
|
||||
|
||||
|
||||
def _resolve_thumbs(session, ids: list[int]) -> dict[int, dict]:
|
||||
from ..gallery_service import thumbnail_url
|
||||
|
||||
out: dict[int, dict] = {}
|
||||
if not ids:
|
||||
return out
|
||||
for rid, tp, sha, mime in session.execute(
|
||||
select(
|
||||
ImageRecord.id, ImageRecord.thumbnail_path,
|
||||
ImageRecord.sha256, ImageRecord.mime,
|
||||
).where(ImageRecord.id.in_(ids))
|
||||
).all():
|
||||
out[rid] = {"id": rid, "thumbnail_url": thumbnail_url(tp, sha, mime)}
|
||||
return out
|
||||
|
||||
Reference in New Issue
Block a user