From 2d44a26bdf5536161637c9fffaf4a66d13604d71 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 6 Jul 2026 18:28:25 -0400 Subject: [PATCH] feat(ml): auto-applied tags don't train a head unless confirmed (milestone 139) Makes auto-apply truly "soft" for heads: _ids_with_tag (head positives) and _eligible_tag_ids (graduation count) now count human-applied + operator-confirmed tags only, via a shared _AUTO_SOURCES (head_auto/ccip_auto/ml_auto) exclusion. Unconfirmed auto-applied tags no longer train the head that judges them, so a misfire can't reinforce itself and the retraction sweep can actually drop it. Confirming a tag (TagPositiveConfirmation) promotes it to a positive AND protects it from retraction. sklearn-free tests. CCIP reference exclusion is the companion piece, next. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM --- backend/app/services/ml/heads.py | 16 ++++-- backend/app/services/ml/training_data.py | 29 +++++++++- tests/test_head_positive_sources.py | 69 ++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 6 deletions(-) create mode 100644 tests/test_head_positive_sources.py diff --git a/backend/app/services/ml/heads.py b/backend/app/services/ml/heads.py index c01fd7f..592a471 100644 --- a/backend/app/services/ml/heads.py +++ b/backend/app/services/ml/heads.py @@ -22,7 +22,7 @@ import logging from datetime import UTC, datetime from typing import Any -from sqlalchemy import delete, func, select +from sqlalchemy import delete, exists, func, select from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import Session @@ -40,6 +40,7 @@ from ...models import ( ) from ...models.tag import image_tag from .training_data import ( + _AUTO_SOURCES, _auto_apply_point, _hygiene_excluded_ids, _ids_with_tag, @@ -138,13 +139,20 @@ def _embedder_version(session: Session) -> str: def _eligible_tag_ids(session: Session, min_pos: int) -> list[int]: - """Concept tags (general/character) with >= min_pos labelled images — the - set that gets a head. Counts all sources; source-aware filtering (#1133) is - a separate, optional refinement.""" + """Concept tags (general/character) with >= min_pos POSITIVE images — the set + that gets a head. Counts human-applied + operator-confirmed tags only; + unconfirmed auto-applied predictions do NOT count toward eligibility (they + don't train the head — milestone 139), so a concept can't graduate on its own + guesses.""" + confirmed = exists().where( + TagPositiveConfirmation.image_record_id == image_tag.c.image_record_id, + TagPositiveConfirmation.tag_id == image_tag.c.tag_id, + ) rows = session.execute( select(Tag.id) .join(image_tag, image_tag.c.tag_id == Tag.id) .where(Tag.kind.in_(_HEAD_KINDS)) + .where(image_tag.c.source.not_in(_AUTO_SOURCES) | confirmed) .group_by(Tag.id) .having(func.count(image_tag.c.image_record_id) >= min_pos) ).all() diff --git a/backend/app/services/ml/training_data.py b/backend/app/services/ml/training_data.py index 5e2963a..6c9802a 100644 --- a/backend/app/services/ml/training_data.py +++ b/backend/app/services/ml/training_data.py @@ -17,9 +17,20 @@ from typing import Any from sqlalchemy import func, select from sqlalchemy.orm import Session -from ...models import ImageRecord, Tag, TagSuggestionRejection +from ...models import ( + ImageRecord, + Tag, + TagPositiveConfirmation, + TagSuggestionRejection, +) from ...models.tag import image_tag +# Auto-apply sources whose tags are PROVISIONAL: they never train a head (or seed +# a CCIP reference) unless the operator confirms them (milestone 139). Keeping +# auto-applied predictions out of training is what makes them "soft" — a misfire +# can't reinforce itself, so the retraction sweep can actually drop it. +_AUTO_SOURCES = ("head_auto", "ccip_auto", "ml_auto") + def _hygiene_excluded_ids(session: Session) -> set[int]: """Ids of images carrying ANY system tag (wip / banner / editor @@ -45,9 +56,23 @@ def _hygiene_excluded_ids(session: Session) -> set[int]: def _ids_with_tag(session: Session, tag_id: int) -> list[int]: + """Image ids that count as POSITIVES for this tag's head: human-applied + (manual / accepted) tags PLUS any auto-applied tag the operator explicitly + confirmed (TagPositiveConfirmation). Unconfirmed auto-applied tags are + EXCLUDED — they are provisional and must not train the head that judges + them (milestone 139).""" + confirmed = ( + select(TagPositiveConfirmation.image_record_id) + .where(TagPositiveConfirmation.tag_id == tag_id) + ) return [ r[0] for r in session.execute( - select(image_tag.c.image_record_id).where(image_tag.c.tag_id == tag_id) + select(image_tag.c.image_record_id) + .where(image_tag.c.tag_id == tag_id) + .where( + image_tag.c.source.not_in(_AUTO_SOURCES) + | image_tag.c.image_record_id.in_(confirmed) + ) ).all() ] diff --git a/tests/test_head_positive_sources.py b/tests/test_head_positive_sources.py new file mode 100644 index 0000000..c26039e --- /dev/null +++ b/tests/test_head_positive_sources.py @@ -0,0 +1,69 @@ +"""Soft auto-apply (milestone 139): unconfirmed auto-applied tags do NOT train a +head. _ids_with_tag (positives) + _eligible_tag_ids (graduation count) count +human-applied + operator-confirmed tags only. Sklearn-free, so tested via +db_sync.""" +import pytest + +from backend.app.models import ImageRecord, Tag, TagKind, TagPositiveConfirmation +from backend.app.models.tag import image_tag +from backend.app.services.ml.heads import _eligible_tag_ids +from backend.app.services.ml.training_data import _ids_with_tag + +pytestmark = pytest.mark.integration + + +def _img(db, sha: str) -> ImageRecord: + img = ImageRecord( + path=f"/images/{sha}.jpg", sha256=sha, size_bytes=1, mime="image/jpeg", + width=1, height=1, origin="imported_filesystem", + integrity_status="unknown", + ) + db.add(img) + db.flush() + return img + + +def _tag(db, name: str) -> Tag: + t = Tag(name=name, kind=TagKind.general) + db.add(t) + db.flush() + return t + + +def _apply(db, image_id: int, tag_id: int, source: str) -> None: + db.execute(image_tag.insert().values( + image_record_id=image_id, tag_id=tag_id, source=source, + )) + + +def test_positives_exclude_unconfirmed_auto(db_sync): + tag = _tag(db_sync, "glasses") + man = _img(db_sync, "a" * 64) + auto = _img(db_sync, "b" * 64) + conf = _img(db_sync, "c" * 64) + acc = _img(db_sync, "d" * 64) + _apply(db_sync, man.id, tag.id, "manual") + _apply(db_sync, auto.id, tag.id, "head_auto") # unconfirmed → excluded + _apply(db_sync, conf.id, tag.id, "head_auto") # confirmed → included + _apply(db_sync, acc.id, tag.id, "ml_accepted") + db_sync.add(TagPositiveConfirmation(image_record_id=conf.id, tag_id=tag.id)) + db_sync.commit() + + pos = set(_ids_with_tag(db_sync, tag.id)) + assert pos == {man.id, conf.id, acc.id} + assert auto.id not in pos + + +def test_eligibility_counts_positives_only(db_sync): + # A concept whose only tags are unconfirmed auto-applies does NOT graduate. + tag = _tag(db_sync, "autotag") + for i in range(3): + _apply(db_sync, _img(db_sync, f"e{i}" * 32).id, tag.id, "head_auto") + db_sync.commit() + assert tag.id not in _eligible_tag_ids(db_sync, min_pos=2) + + # Two human positives → now eligible at min_pos=2. + for i in range(2): + _apply(db_sync, _img(db_sync, f"h{i}" * 32).id, tag.id, "manual") + db_sync.commit() + assert tag.id in _eligible_tag_ids(db_sync, min_pos=2)