diff --git a/backend/app/services/ml/suggestions.py b/backend/app/services/ml/suggestions.py index 7c71b0d..9e570a8 100644 --- a/backend/app/services/ml/suggestions.py +++ b/backend/app/services/ml/suggestions.py @@ -4,7 +4,7 @@ threshold-filtered, category-grouped, ranked suggestions for one image. from dataclasses import dataclass, field -from sqlalchemy import select +from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession from ...models import ( @@ -16,6 +16,7 @@ from ...models import ( from ...models.tag import image_tag from .aliases import AliasService from .centroids import CentroidService +from .tag_name import normalize as normalize_tag_name from .tagger import SURFACED_CATEGORIES @@ -84,7 +85,12 @@ class SuggestionService: ) # --- Camie predictions --- - candidates: list[tuple[str, str, float]] = [] + # candidates carry (raw_name, display_name, category, confidence). + # raw_name = the booru-formatted vocab key, kept for alias_map + # lookup since alias rows are hand-curated against raw keys. + # display_name = normalize_tag_name(raw_name) — what the operator + # sees AND what gets written to tag.name on Accept. + candidates: list[tuple[str, str, str, float]] = [] for name, p in predictions.items(): category = p.get("category", "general") if category not in SURFACED_CATEGORIES: @@ -92,10 +98,14 @@ class SuggestionService: conf = float(p.get("confidence", 0.0)) if conf < self._threshold_for(settings, category): continue - candidates.append((name, category, conf)) + display = normalize_tag_name(name) + if display is None: + # emoticon / pure-punctuation vocab entry — drop entirely + continue + candidates.append((name, display, category, conf)) alias_map = await self.aliases.resolve_many( - [(n, c) for n, c, _ in candidates] + [(raw, c) for raw, _disp, c, _conf in candidates] ) merged: dict[object, Suggestion] = {} @@ -116,8 +126,8 @@ class SuggestionService: creates_new_tag=existing.creates_new_tag, ) - for name, category, conf in candidates: - canonical = alias_map.get((name, category)) + for raw, display, category, conf in candidates: + canonical = alias_map.get((raw, category)) if canonical is not None: if canonical.id in applied or canonical.id in rejected: continue @@ -133,9 +143,17 @@ class SuggestionService: ), ) else: + # Case-insensitive match on BOTH the raw camie key AND + # the normalized form — covers legacy underscore-named + # Tag rows accepted before normalization shipped, AND + # any tag the operator created with the human form. existing_tag = ( await self.session.execute( - select(Tag).where(Tag.name == name) + select(Tag).where( + func.lower(Tag.name).in_( + [raw.lower(), display.lower()] + ) + ) ) ).scalars().first() if existing_tag is not None: @@ -157,10 +175,10 @@ class SuggestionService: ) else: _merge( - f"raw:{name}:{category}", + f"raw:{display}:{category}", Suggestion( canonical_tag_id=None, - display_name=name, + display_name=display, category=category, score=conf, source="tagger", diff --git a/backend/app/services/ml/tag_name.py b/backend/app/services/ml/tag_name.py new file mode 100644 index 0000000..3dc9d6a --- /dev/null +++ b/backend/app/services/ml/tag_name.py @@ -0,0 +1,61 @@ +"""Camie vocabulary -> human-readable tag-name normalization. + +Camie v2's ~57k tag vocabulary is booru-derived and arrives as raw +strings like `uchiha_sasuke_(naruto)`, `#unicus_(idolmaster)`, +`1000-nen_ikiteru_(vocaloid)`, or `:/`. We want the operator to see +"Uchiha Sasuke", "Unicus", "1000-Nen Ikiteru", or to never see the +emoticon at all — and we want the same clean string to be what lands +in `tag.name` when the suggestion is accepted, so Accept matches the +existing-tag convention (`tag_service.find_or_create`). + +Rules (operator-approved 2026-06-03): + 1. Strip leading junk chars (#, ., +, ;, ~, _, whitespace) + 2. Drop trailing `_(disambiguator)` block(s), iteratively + 3. Strip wrapping single/double quotes (after disambig removal so + `"foo_em_up"_(series)` -> `"foo_em_up"` -> `foo_em_up`) + 4. Replace remaining `_` with space; collapse runs of whitespace + 5. Add a space after any `:` (namespace:tag -> namespace: tag) + 6. Preserve hyphens (booru hyphens often carry meaning) + 7. Title-case each space-separated word (first character only — + apostrophes, digits, hyphens stay) + 8. If no letters remain, return None (drop emoticons like `:/`) + 9. No surname/givenname swap — no reliable signal in the vocab +""" + +import re + +_LEADING_JUNK = re.compile(r"^[#.+;~_\s]+") +_TRAILING_DISAMBIG = re.compile(r"_\([^)]*\)\s*$") +_MULTISPACE = re.compile(r"\s+") +_COLON_NOSPACE = re.compile(r":(?=\S)") +_HAS_LETTER = re.compile(r"[A-Za-z]") + + +def _strip_wrapping_quotes(s: str) -> str: + if len(s) >= 2 and s[0] == s[-1] and s[0] in ('"', "'"): + return s[1:-1] + return s + + +def _title_word(w: str) -> str: + return w[:1].upper() + w[1:] if w else w + + +def normalize(raw: str) -> str | None: + """Return the human-readable form of a raw Camie tag, or None if the + string is junk (emoticon, empty after stripping).""" + if not raw: + return None + s = _LEADING_JUNK.sub("", raw) + while True: + new = _TRAILING_DISAMBIG.sub("", s) + if new == s: + break + s = new + s = _strip_wrapping_quotes(s) + s = s.replace("_", " ") + s = _COLON_NOSPACE.sub(": ", s) + s = _MULTISPACE.sub(" ", s).strip() + if not s or not _HAS_LETTER.search(s): + return None + return " ".join(_title_word(w) for w in s.split(" ")) diff --git a/tests/test_ml_suggestions.py b/tests/test_ml_suggestions.py index 1074acd..be1f4fa 100644 --- a/tests/test_ml_suggestions.py +++ b/tests/test_ml_suggestions.py @@ -39,8 +39,9 @@ async def test_threshold_filters_low_confidence_general(db): await db.flush() sl = await SuggestionService(db).for_image(img.id) names = [s.display_name for s in sl.by_category.get("general", [])] - assert "sword" in names - assert "lowconf" not in names + # display_name is normalized (tag_name.normalize) before surfacing. + assert "Sword" in names + assert "Lowconf" not in names @pytest.mark.asyncio @@ -84,7 +85,9 @@ async def test_raw_tag_creates_new(db): await db.flush() sl = await SuggestionService(db).for_image(img.id) chars = sl.by_category["character"] - assert chars[0].display_name == "brand_new_tag" + # display_name is the normalized Camie name (underscores -> spaces, + # title-cased), not the raw vocab key. + assert chars[0].display_name == "Brand New Tag" assert chars[0].creates_new_tag is True assert chars[0].canonical_tag_id is None diff --git a/tests/test_ml_tag_name.py b/tests/test_ml_tag_name.py new file mode 100644 index 0000000..a791cfd --- /dev/null +++ b/tests/test_ml_tag_name.py @@ -0,0 +1,53 @@ +import pytest + +from backend.app.services.ml.tag_name import normalize + + +@pytest.mark.parametrize( + "raw, expected", + [ + # Rule 4: underscores -> spaces; rule 7: title case + ("light_purple_hair", "Light Purple Hair"), + ("no_pants", "No Pants"), + ("year_2005", "Year 2005"), + # Single-word still title-cased + ("sword", "Sword"), + # Rule 3: drop trailing _(disambiguator) + ("uchiha_sasuke_(naruto)", "Uchiha Sasuke"), + ("apple_(fruit)", "Apple"), + ("kirby_(series)", "Kirby"), + # Repeated trailing disambig blocks + ("foo_(bar)_(baz)", "Foo"), + # Rule 1: leading junk chars + ("#unicus_(idolmaster)", "Unicus"), + (".52_gal_(splatoon)", "52 Gal"), + ("+_+_smile_(emote)", "Smile"), + # Rule 5: space after colon + ("nier:automata", "Nier: Automata"), + # Already-spaced colon left alone + ("nier: automata", "Nier: Automata"), + # Rule 6: hyphens preserved + ("1000-nen_ikiteru_(vocaloid)", "1000-nen Ikiteru"), + ("well-known_face", "Well-known Face"), + # Rule 2: wrapping quotes + ('"pile_em_up"_(genshin_impact)', "Pile Em Up"), + ("'foo_bar'", "Foo Bar"), + # Rule 8: emoticons -> None + (":/", None), + (";)", None), + ("+_+", None), + ("^_^", None), + # Empty / whitespace-only + ("", None), + (" ", None), + ("___", None), + # Apostrophe inside word — preserved, not title-cased + ("it's_okay", "It's Okay"), + # Digit-only still surfaces (year tags) + ("2005", "2005"), + # Multi-space collapse + ("foo___bar", "Foo Bar"), + ], +) +def test_normalize(raw, expected): + assert normalize(raw) == expected