Merge pull request 'tag-eval: "keep" records a confirmation so doubts stop resurfacing' (#141) from dev into main

alembic/versions/0057_tag_positive_confirmation.py

@@ -0,0 +1,40 @@
+"""tag_positive_confirmation: operator-affirmed correct positives (#1130)
+
+Mirror of tag_suggestion_rejection. "Keep" on a doubted positive records here so
+the eval's doubts list stops resurfacing confirmed-correct images every run.
+
+Revision ID: 0057
+Revises: 0056
+Create Date: 2026-06-28
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0057"
+down_revision: Union[str, None] = "0056"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "tag_positive_confirmation",
+        sa.Column(
+            "image_record_id", sa.Integer(),
+            sa.ForeignKey("image_record.id", ondelete="CASCADE"), primary_key=True,
+        ),
+        sa.Column(
+            "tag_id", sa.Integer(),
+            sa.ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True, index=True,
+        ),
+        sa.Column(
+            "confirmed_at", sa.DateTime(timezone=True), nullable=False,
+            server_default=sa.func.now(),
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("tag_positive_confirmation")

backend/app/api/tags.py

@@ -2,10 +2,11 @@
 
 from quart import Blueprint, jsonify, request
 from sqlalchemy import exists, select
+from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.exc import IntegrityError
 
 from ..extensions import get_session
-from ..models import Tag, TagKind
+from ..models import Tag, TagKind, TagPositiveConfirmation
 from ..models.tag_allowlist import TagAllowlist
 from ..services.bulk_tag_service import BulkTagService
 from ..services.ml.aliases import AliasService
@@ -183,6 +184,20 @@ async def remove_tag_from_image(image_id: int, tag_id: int):
     return "", 204
 
 
+@tags_bp.route("/images/<int:image_id>/tags/<int:tag_id>/confirm", methods=["POST"])
+async def confirm_tag_on_image(image_id: int, tag_id: int):
+    """Operator affirmed an applied tag is correct ("keep" on a doubted positive).
+    Idempotent; recorded so the eval's doubts list stops resurfacing it (#1130)."""
+    async with get_session() as session:
+        await session.execute(
+            pg_insert(TagPositiveConfirmation)
+            .values(image_record_id=image_id, tag_id=tag_id)
+            .on_conflict_do_nothing(index_elements=["image_record_id", "tag_id"])
+        )
+        await session.commit()
+    return "", 204
+
+
 @tags_bp.route("/tags/<int:tag_id>", methods=["GET"])
 async def get_tag(tag_id: int):
     """Resolve a single tag (used by the gallery to label its active

backend/app/models/__init__.py

@@ -30,6 +30,7 @@ from .tag import Tag, TagKind, image_tag
 from .tag_alias import TagAlias
 from .tag_allowlist import TagAllowlist
 from .tag_eval_run import TagEvalRun
+from .tag_positive_confirmation import TagPositiveConfirmation
 from .tag_reference_embedding import TagReferenceEmbedding
 from .tag_suggestion_rejection import TagSuggestionRejection
 from .task_run import TaskRun
@@ -67,6 +68,7 @@ __all__ = [
     "TagAlias",
     "TagAllowlist",
     "TagEvalRun",
+    "TagPositiveConfirmation",
     "TagReferenceEmbedding",
     "TagSuggestionRejection",
     "TaskRun",

backend/app/models/tag_positive_confirmation.py

@@ -0,0 +1,28 @@
+"""TagPositiveConfirmation — operator affirmed an applied tag is correct.
+
+The mirror of TagSuggestionRejection (#1130). When the operator "keeps" a
+positive the head doubts (low-scoring), record it so the eval's doubts list
+stops resurfacing the same confirmed-correct images every run. Does not change
+training (it's already a positive) — purely a "I've reviewed this" marker.
+"""
+
+from datetime import datetime
+
+from sqlalchemy import DateTime, ForeignKey, func
+from sqlalchemy.orm import Mapped, mapped_column
+
+from .base import Base
+
+
+class TagPositiveConfirmation(Base):
+    __tablename__ = "tag_positive_confirmation"
+
+    image_record_id: Mapped[int] = mapped_column(
+        ForeignKey("image_record.id", ondelete="CASCADE"), primary_key=True
+    )
+    tag_id: Mapped[int] = mapped_column(
+        ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True, index=True
+    )
+    confirmed_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )

backend/app/services/ml/tag_eval.py

@@ -23,7 +23,14 @@ from typing import Any
 from sqlalchemy import func, select
 from sqlalchemy.orm import Session
 
-from ...models import ImageRecord, Tag, TagEvalRun, TagKind, TagSuggestionRejection
+from ...models import (
+    ImageRecord,
+    Tag,
+    TagEvalRun,
+    TagKind,
+    TagPositiveConfirmation,
+    TagSuggestionRejection,
+)
 from ...models.tag import image_tag
 
 log = logging.getLogger(__name__)
@@ -146,6 +153,17 @@ def _rejected_ids(session: Session, tag_id: int) -> list[int]:
     ]
 
 
+def _confirmed_ids(session: Session, tag_id: int) -> set[int]:
+    """Positives the operator explicitly affirmed ('keep') — excluded from the
+    doubts list so confirmed-correct images don't resurface every run."""
+    return {
+        r[0] for r in session.execute(
+            select(TagPositiveConfirmation.image_record_id)
+            .where(TagPositiveConfirmation.tag_id == tag_id)
+        ).all()
+    }
+
+
 def _sample_unlabeled(session: Session, exclude: set[int], limit: int) -> list[int]:
     """Random image ids (with an embedding) NOT carrying the tag. Concepts are
     sparse, so an untagged image is almost always a true negative."""
@@ -239,7 +257,8 @@ def _eval_concept(session: Session, name: str, cfg: dict, np) -> dict[str, Any]:
     head = _eval_head(Xn, y, cfg["cv_folds"], cfg["precision_target"], np)
     centroid = _eval_centroid(Xn, y, cfg["cv_folds"], np)
     curve = _learning_curve(Xn, y, cfg["curve_points"], neg_ratio, np)
-    examples = _examples(session, Xn, y, ids, np, set(rejected))
+    confirmed = _confirmed_ids(session, tag_id)
+    examples = _examples(session, Xn, y, ids, np, set(rejected), confirmed)
 
     return {
         "name": name, "tag_id": tag_id,
@@ -358,13 +377,13 @@ def _learning_curve(Xn, y, points, neg_ratio, np) -> list[dict[str, float]]:
     return out
 
 
-def _examples(session, Xn, y, ids, np, rejected_set) -> dict[str, list[dict]]:
+def _examples(session, Xn, y, ids, np, rejected_set, confirmed_set) -> dict[str, list[dict]]:
     """Train on all data, then surface: top-scoring negatives the operator has
-    NOT already rejected (= fresh suggestions) and lowest-scoring POSITIVES
-    (where the head disagrees with the operator's tag). Excluding already-
-    rejected ids stops an adjudicated near-miss — a hard negative that still
-    scores high — from resurfacing in 'would suggest' on every run. Resolves
-    thumbnail urls so the stored report renders without per-id lookups."""
+    NOT already rejected (= fresh suggestions) and lowest-scoring POSITIVES the
+    operator has NOT already confirmed (= unreviewed doubts). Excluding rejected
+    ids stops an adjudicated near-miss from resurfacing in 'would suggest';
+    excluding confirmed ids stops a 'kept' correct positive from resurfacing in
+    'head doubts' every run. Resolves thumbnail urls for a self-contained report."""
     from sklearn.linear_model import LogisticRegression
 
     clf = LogisticRegression(max_iter=1000, class_weight="balanced")
@@ -380,7 +399,14 @@ def _examples(session, Xn, y, ids, np, rejected_set) -> dict[str, list[dict]]:
         top_neg.append(rid)
         if len(top_neg) >= _EXAMPLES_K:
             break
-    low_pos = [int(ids[i]) for i in pos_idx[np.argsort(s[pos_idx])[:_EXAMPLES_K]]]
+    low_pos = []
+    for i in pos_idx[np.argsort(s[pos_idx])]:  # low score → high
+        rid = int(ids[i])
+        if rid in confirmed_set:
+            continue  # already kept/confirmed — don't re-doubt it
+        low_pos.append(rid)
+        if len(low_pos) >= _EXAMPLES_K:
+            break
     thumbs = _resolve_thumbs(session, top_neg + low_pos)
     return {
         "head_would_suggest": [thumbs[i] for i in top_neg if i in thumbs],

frontend/src/components/settings/TagEvalCard.vue

@@ -247,7 +247,7 @@ async function act(c, it, dir, verdict) {
   if (dir === 'suggest' && verdict === 'yes') { call = store.applyTag(it.id, c.tag_id); label = 'tagged' }
   else if (dir === 'suggest' && verdict === 'no') { call = store.rejectTag(it.id, c.tag_id); label = 'rejected' }
   else if (dir === 'doubts' && verdict === 'no') { call = store.removeTag(it.id, c.tag_id); label = 'removed' }
-  else { acted.value[key] = 'kept'; return }   // doubt + yes = keep, no write
+  else { call = store.confirmTag(it.id, c.tag_id); label = 'kept' }   // doubt + yes = keep (confirm)
   try {
     await call
     acted.value[key] = label

frontend/src/stores/tagEval.js

@@ -47,5 +47,11 @@ export const useTagEvalStore = defineStore('tagEval', () => {
       { body: { tag_id: tagId } })
   }
 
-  return { start, getRun, latest, applyTag, rejectTag, removeTag }
+  // "Keep" — affirm a doubted positive is correct. Records a confirmation so it
+  // stops resurfacing in the doubts list (it stays a positive either way).
+  async function confirmTag(imageId, tagId) {
+    return await api.post(`/api/images/${imageId}/tags/${tagId}/confirm`)
+  }
+
+  return { start, getRun, latest, applyTag, rejectTag, removeTag, confirmTag }
 })